From 4e3079c521361e3191f2a823fec7b71af9b85ddc Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Wed, 10 Jan 2024 15:03:52 -0500 Subject: [PATCH] Add archive bytes limit option to config This sets the max length of the archive in bytes. If the archive grows larger than this limit the archive is trimmed until it satisfies the limit. For k8s products this limit is 1MiB or 1048576 bytes. If the limit is set to 0 there is no size limit. The archive can grow as large as the retention period. --- csp_billing_adapter/archive.py | 34 +++++++++++++++---- examples/README.md | 3 ++ ...sp-billing-adapter-example-Rancher_NV.yaml | 1 + .../csp-billing-adapter-example-SUMA.yaml | 1 + tests/data/config_bad.yaml | 1 + tests/data/config_dimensions_gap.yaml | 1 + tests/data/config_dimensions_no_tail.yaml | 1 + tests/data/config_good_average.yaml | 1 + tests/data/config_good_maximum.yaml | 1 + tests/data/config_invalid_consumption.yaml | 1 + tests/data/config_no_usage_metrics.yaml | 1 + tests/data/config_testing_mixed.yaml | 1 + tests/unit/test_archive.py | 23 ++++++++++++- 13 files changed, 62 insertions(+), 8 deletions(-) diff --git a/csp_billing_adapter/archive.py b/csp_billing_adapter/archive.py index bf6b5b6..54f4160 100644 --- a/csp_billing_adapter/archive.py +++ b/csp_billing_adapter/archive.py @@ -18,6 +18,7 @@ import functools import logging +import json from csp_billing_adapter.config import Config from csp_billing_adapter.utils import retry_on_exception @@ -25,17 +26,21 @@ log = logging.getLogger('CSPBillingAdapter') DEFAULT_RETENTION_PERIOD = 6 # in months +DEFAULT_BYTES_LIMIT = 0 def append_metering_records( archive: list, billing_record: dict, - max_length: int + max_length: int, + max_bytes: int ) -> list: """ Append usage and metering records to the archive - If archive is larger than max length, drop the oldest record. + If archive is larger than max length drop the oldest + record. If the archive is larger than the max bytes + limit trim it until it satisfies the limit. :param archive: The list of meterings and usage records to append to. @@ -44,17 +49,31 @@ def append_metering_records( metering and usage records to be archived. :param max_length: The max size of the archive list. + :param max_bytes: + The max size in bytes of the archive. :return: The archive of meterings and usage records with the billing_record appended. If archive ends up greater - than max lengh the first (oldest) record is dropped. + than max lengh or max bytes the archive is trimmed + as necessary to satisfy both max_length and + max_bytes. """ archive.append(billing_record) if len(archive) > max_length: - return archive[1:] - else: - return archive + archive = archive[1:] + + if max_bytes > 0: + while True: + # Trim archive until it is smaller than max bytes + archive_size = len(bytes(json.dumps(archive), 'utf-8')) + + if archive_size > max_bytes: + archive = archive[1:] + else: + break + + return archive def archive_record( @@ -88,7 +107,8 @@ def archive_record( archive = append_metering_records( archive, billing_record, - config.archive_retention_period or DEFAULT_RETENTION_PERIOD + config.archive_retention_period or DEFAULT_RETENTION_PERIOD, + config.archive_bytes_limit or DEFAULT_BYTES_LIMIT ) retry_on_exception( diff --git a/examples/README.md b/examples/README.md index 2663044..c195510 100644 --- a/examples/README.md +++ b/examples/README.md @@ -33,6 +33,9 @@ reporting_interval: archive_retention_period: Sets the time in months that meterings and usage records are retained in the data archive. +archive_bytes_limit: + Sets the max size in bytes for the metering archive. The archive will be trimmed to stay below this limit. + usage_metrics: {metric name}: diff --git a/examples/csp-billing-adapter-example-Rancher_NV.yaml b/examples/csp-billing-adapter-example-Rancher_NV.yaml index 7b90ad3..40f0bcb 100644 --- a/examples/csp-billing-adapter-example-Rancher_NV.yaml +++ b/examples/csp-billing-adapter-example-Rancher_NV.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: CSP_DEPENDENT_SETTING reporting_interval: CSP_DEPENDENT_SETTING archive_retention_period: 6 +archive_bytes_limit: 1048576 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/examples/csp-billing-adapter-example-SUMA.yaml b/examples/csp-billing-adapter-example-SUMA.yaml index 6f8811f..b7f6bb3 100644 --- a/examples/csp-billing-adapter-example-SUMA.yaml +++ b/examples/csp-billing-adapter-example-SUMA.yaml @@ -6,6 +6,7 @@ reporting_api_is_cumulative: CSP_DEPENDENT_SETTING reporting_interval: CSP_DEPENDENT_SETTING support_info_collection_cmd: CSP_DEPENDENT_SETTING archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: ha_systems: consumption_reporting: volume diff --git a/tests/data/config_bad.yaml b/tests/data/config_bad.yaml index 2cc3df5..fb78ca5 100644 --- a/tests/data/config_bad.yaml +++ b/tests/data/config_bad.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_dimensions_gap.yaml b/tests/data/config_dimensions_gap.yaml index a5f411b..f864c6b 100644 --- a/tests/data/config_dimensions_gap.yaml +++ b/tests/data/config_dimensions_gap.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_dimensions_no_tail.yaml b/tests/data/config_dimensions_no_tail.yaml index d6715be..743b79e 100644 --- a/tests/data/config_dimensions_no_tail.yaml +++ b/tests/data/config_dimensions_no_tail.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_good_average.yaml b/tests/data/config_good_average.yaml index c39e7d5..f4378f1 100644 --- a/tests/data/config_good_average.yaml +++ b/tests/data/config_good_average.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_good_maximum.yaml b/tests/data/config_good_maximum.yaml index 117a3c0..b1ee86f 100644 --- a/tests/data/config_good_maximum.yaml +++ b/tests/data/config_good_maximum.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_invalid_consumption.yaml b/tests/data/config_invalid_consumption.yaml index 796379f..21b07d3 100644 --- a/tests/data/config_invalid_consumption.yaml +++ b/tests/data/config_invalid_consumption.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: invalid diff --git a/tests/data/config_no_usage_metrics.yaml b/tests/data/config_no_usage_metrics.yaml index 2ebc13e..9e29b2d 100644 --- a/tests/data/config_no_usage_metrics.yaml +++ b/tests/data/config_no_usage_metrics.yaml @@ -4,4 +4,5 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 version: 1.1.1 diff --git a/tests/data/config_testing_mixed.yaml b/tests/data/config_testing_mixed.yaml index 39bb3b0..6d3dbc9 100644 --- a/tests/data/config_testing_mixed.yaml +++ b/tests/data/config_testing_mixed.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: jobs: # test metric with average aggregation, volume reporting diff --git a/tests/unit/test_archive.py b/tests/unit/test_archive.py index 07db2b1..29cc2fa 100644 --- a/tests/unit/test_archive.py +++ b/tests/unit/test_archive.py @@ -39,12 +39,33 @@ def test_append_metering_records(): } for i in range(10): - archive = append_metering_records(archive, records, 6) + archive = append_metering_records(archive, records, 6, 0) assert len(archive) == 6 assert archive[4] == records +def test_append_metering_records_max_bytes(): + archive = [] + records = { + "bill_time": "2024-01-03T20:06:42.076972+00:00", + "metering_id": "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", + "usage_records": [ + { + "managed_node_count": 9, + "managed_systems": 6, + "reporting_time": "2024-01-03T20:06:42.076972+00:00" + } + ] + } + + for i in range(4): + archive = append_metering_records(archive, records, 6, 456) + + assert len(archive) == 2 + assert archive[0] == records + + def test_archive_record(cba_pm, cba_config): record = { 'billing_time': '2024-02-09T18:11:59.527064+00:00',