diff --git a/csp_billing_adapter/archive.py b/csp_billing_adapter/archive.py index bf6b5b6..54f4160 100644 --- a/csp_billing_adapter/archive.py +++ b/csp_billing_adapter/archive.py @@ -18,6 +18,7 @@ import functools import logging +import json from csp_billing_adapter.config import Config from csp_billing_adapter.utils import retry_on_exception @@ -25,17 +26,21 @@ log = logging.getLogger('CSPBillingAdapter') DEFAULT_RETENTION_PERIOD = 6 # in months +DEFAULT_BYTES_LIMIT = 0 def append_metering_records( archive: list, billing_record: dict, - max_length: int + max_length: int, + max_bytes: int ) -> list: """ Append usage and metering records to the archive - If archive is larger than max length, drop the oldest record. + If archive is larger than max length drop the oldest + record. If the archive is larger than the max bytes + limit trim it until it satisfies the limit. :param archive: The list of meterings and usage records to append to. @@ -44,17 +49,31 @@ def append_metering_records( metering and usage records to be archived. :param max_length: The max size of the archive list. + :param max_bytes: + The max size in bytes of the archive. :return: The archive of meterings and usage records with the billing_record appended. If archive ends up greater - than max lengh the first (oldest) record is dropped. + than max lengh or max bytes the archive is trimmed + as necessary to satisfy both max_length and + max_bytes. """ archive.append(billing_record) if len(archive) > max_length: - return archive[1:] - else: - return archive + archive = archive[1:] + + if max_bytes > 0: + while True: + # Trim archive until it is smaller than max bytes + archive_size = len(bytes(json.dumps(archive), 'utf-8')) + + if archive_size > max_bytes: + archive = archive[1:] + else: + break + + return archive def archive_record( @@ -88,7 +107,8 @@ def archive_record( archive = append_metering_records( archive, billing_record, - config.archive_retention_period or DEFAULT_RETENTION_PERIOD + config.archive_retention_period or DEFAULT_RETENTION_PERIOD, + config.archive_bytes_limit or DEFAULT_BYTES_LIMIT ) retry_on_exception( diff --git a/examples/README.md b/examples/README.md index 2663044..c195510 100644 --- a/examples/README.md +++ b/examples/README.md @@ -33,6 +33,9 @@ reporting_interval: archive_retention_period: Sets the time in months that meterings and usage records are retained in the data archive. +archive_bytes_limit: + Sets the max size in bytes for the metering archive. The archive will be trimmed to stay below this limit. + usage_metrics: {metric name}: diff --git a/examples/csp-billing-adapter-example-Rancher_NV.yaml b/examples/csp-billing-adapter-example-Rancher_NV.yaml index 7b90ad3..40f0bcb 100644 --- a/examples/csp-billing-adapter-example-Rancher_NV.yaml +++ b/examples/csp-billing-adapter-example-Rancher_NV.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: CSP_DEPENDENT_SETTING reporting_interval: CSP_DEPENDENT_SETTING archive_retention_period: 6 +archive_bytes_limit: 1048576 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/examples/csp-billing-adapter-example-SUMA.yaml b/examples/csp-billing-adapter-example-SUMA.yaml index 6f8811f..b7f6bb3 100644 --- a/examples/csp-billing-adapter-example-SUMA.yaml +++ b/examples/csp-billing-adapter-example-SUMA.yaml @@ -6,6 +6,7 @@ reporting_api_is_cumulative: CSP_DEPENDENT_SETTING reporting_interval: CSP_DEPENDENT_SETTING support_info_collection_cmd: CSP_DEPENDENT_SETTING archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: ha_systems: consumption_reporting: volume diff --git a/tests/data/config_bad.yaml b/tests/data/config_bad.yaml index 2cc3df5..fb78ca5 100644 --- a/tests/data/config_bad.yaml +++ b/tests/data/config_bad.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_dimensions_gap.yaml b/tests/data/config_dimensions_gap.yaml index a5f411b..f864c6b 100644 --- a/tests/data/config_dimensions_gap.yaml +++ b/tests/data/config_dimensions_gap.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_dimensions_no_tail.yaml b/tests/data/config_dimensions_no_tail.yaml index d6715be..743b79e 100644 --- a/tests/data/config_dimensions_no_tail.yaml +++ b/tests/data/config_dimensions_no_tail.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_good_average.yaml b/tests/data/config_good_average.yaml index c39e7d5..f4378f1 100644 --- a/tests/data/config_good_average.yaml +++ b/tests/data/config_good_average.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_good_maximum.yaml b/tests/data/config_good_maximum.yaml index 117a3c0..b1ee86f 100644 --- a/tests/data/config_good_maximum.yaml +++ b/tests/data/config_good_maximum.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: volume diff --git a/tests/data/config_invalid_consumption.yaml b/tests/data/config_invalid_consumption.yaml index 796379f..21b07d3 100644 --- a/tests/data/config_invalid_consumption.yaml +++ b/tests/data/config_invalid_consumption.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: managed_node_count: consumption_reporting: invalid diff --git a/tests/data/config_no_usage_metrics.yaml b/tests/data/config_no_usage_metrics.yaml index 2ebc13e..9e29b2d 100644 --- a/tests/data/config_no_usage_metrics.yaml +++ b/tests/data/config_no_usage_metrics.yaml @@ -4,4 +4,5 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 version: 1.1.1 diff --git a/tests/data/config_testing_mixed.yaml b/tests/data/config_testing_mixed.yaml index 39bb3b0..6d3dbc9 100644 --- a/tests/data/config_testing_mixed.yaml +++ b/tests/data/config_testing_mixed.yaml @@ -4,6 +4,7 @@ query_interval: 3600 reporting_api_is_cumulative: true reporting_interval: 3600 archive_retention_period: 6 +archive_bytes_limit: 0 usage_metrics: jobs: # test metric with average aggregation, volume reporting diff --git a/tests/unit/test_archive.py b/tests/unit/test_archive.py index 07db2b1..29cc2fa 100644 --- a/tests/unit/test_archive.py +++ b/tests/unit/test_archive.py @@ -39,12 +39,33 @@ def test_append_metering_records(): } for i in range(10): - archive = append_metering_records(archive, records, 6) + archive = append_metering_records(archive, records, 6, 0) assert len(archive) == 6 assert archive[4] == records +def test_append_metering_records_max_bytes(): + archive = [] + records = { + "bill_time": "2024-01-03T20:06:42.076972+00:00", + "metering_id": "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", + "usage_records": [ + { + "managed_node_count": 9, + "managed_systems": 6, + "reporting_time": "2024-01-03T20:06:42.076972+00:00" + } + ] + } + + for i in range(4): + archive = append_metering_records(archive, records, 6, 456) + + assert len(archive) == 2 + assert archive[0] == records + + def test_archive_record(cba_pm, cba_config): record = { 'billing_time': '2024-02-09T18:11:59.527064+00:00',