Add archive bytes limit option to config

This sets the max length of the archive in bytes. If the archive grows larger than this limit the archive is trimmed until it satisfies the limit. For k8s products this limit is 1MiB or 1048576 bytes. If the limit is set to 0 there is no size limit. The archive can grow as large as the retention period.
SUSE-Enceladus · Jan 10, 2024 · 4e3079c · 4e3079c
1 parent f946719
commit 4e3079c
Show file tree

Hide file tree

Showing 13 changed files with 62 additions and 8 deletions.
diff --git a/csp_billing_adapter/archive.py b/csp_billing_adapter/archive.py
@@ -18,24 +18,29 @@
 
 import functools
 import logging
+import json
 
 from csp_billing_adapter.config import Config
 from csp_billing_adapter.utils import retry_on_exception
 
 log = logging.getLogger('CSPBillingAdapter')
 
 DEFAULT_RETENTION_PERIOD = 6  # in months
+DEFAULT_BYTES_LIMIT = 0
 
 
 def append_metering_records(
     archive: list,
     billing_record: dict,
-    max_length: int
+    max_length: int,
+    max_bytes: int
 ) -> list:
     """
     Append usage and metering records to the archive
 
-    If archive is larger than max length, drop the oldest record.
+    If archive is larger than max length drop the oldest
+    record. If the archive is larger than the max bytes
+    limit trim it until it satisfies the limit.
 
     :param archive:
         The list of meterings and usage records to append to.
@@ -44,17 +49,31 @@ def append_metering_records(
         metering and usage records to be archived.
     :param max_length:
         The max size of the archive list.
+    :param max_bytes:
+        The max size in bytes of the archive.
     :return:
         The archive of meterings and usage records with the
         billing_record appended. If archive ends up greater
-        than max lengh the first (oldest) record is dropped.
+        than max lengh or max bytes the archive is trimmed
+        as necessary to satisfy both max_length and
+        max_bytes.
     """
     archive.append(billing_record)
 
     if len(archive) > max_length:
-        return archive[1:]
-    else:
-        return archive
+        archive = archive[1:]
+
+    if max_bytes > 0:
+        while True:
+            # Trim archive until it is smaller than max bytes
+            archive_size = len(bytes(json.dumps(archive), 'utf-8'))
+
+            if archive_size > max_bytes:
+                archive = archive[1:]
+            else:
+                break
+
+    return archive
 
 
 def archive_record(
@@ -88,7 +107,8 @@ def archive_record(
     archive = append_metering_records(
         archive,
         billing_record,
-        config.archive_retention_period or DEFAULT_RETENTION_PERIOD
+        config.archive_retention_period or DEFAULT_RETENTION_PERIOD,
+        config.archive_bytes_limit or DEFAULT_BYTES_LIMIT
     )
 
     retry_on_exception(

diff --git a/examples/README.md b/examples/README.md
@@ -33,6 +33,9 @@ reporting_interval:
 archive_retention_period:
     Sets the time in months that meterings and usage records are retained in the data archive.
 
+archive_bytes_limit:
+    Sets the max size in bytes for the metering archive. The archive will be trimmed to stay below this limit.
+
 usage_metrics: 
 
   {metric name}:

diff --git a/examples/csp-billing-adapter-example-Rancher_NV.yaml b/examples/csp-billing-adapter-example-Rancher_NV.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: CSP_DEPENDENT_SETTING
 reporting_interval: CSP_DEPENDENT_SETTING
 archive_retention_period: 6
+archive_bytes_limit: 1048576
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/examples/csp-billing-adapter-example-SUMA.yaml b/examples/csp-billing-adapter-example-SUMA.yaml
@@ -6,6 +6,7 @@ reporting_api_is_cumulative: CSP_DEPENDENT_SETTING
 reporting_interval: CSP_DEPENDENT_SETTING
 support_info_collection_cmd: CSP_DEPENDENT_SETTING
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   ha_systems:
     consumption_reporting: volume

diff --git a/tests/data/config_bad.yaml b/tests/data/config_bad.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/tests/data/config_dimensions_gap.yaml b/tests/data/config_dimensions_gap.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/tests/data/config_dimensions_no_tail.yaml b/tests/data/config_dimensions_no_tail.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/tests/data/config_good_average.yaml b/tests/data/config_good_average.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/tests/data/config_good_maximum.yaml b/tests/data/config_good_maximum.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: volume

diff --git a/tests/data/config_invalid_consumption.yaml b/tests/data/config_invalid_consumption.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   managed_node_count:
     consumption_reporting: invalid

diff --git a/tests/data/config_no_usage_metrics.yaml b/tests/data/config_no_usage_metrics.yaml
@@ -4,4 +4,5 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 version: 1.1.1
diff --git a/tests/data/config_testing_mixed.yaml b/tests/data/config_testing_mixed.yaml
@@ -4,6 +4,7 @@ query_interval: 3600
 reporting_api_is_cumulative: true
 reporting_interval: 3600
 archive_retention_period: 6
+archive_bytes_limit: 0
 usage_metrics:
   jobs:
     # test metric with average aggregation, volume reporting

diff --git a/tests/unit/test_archive.py b/tests/unit/test_archive.py
@@ -39,12 +39,33 @@ def test_append_metering_records():
     }
 
     for i in range(10):
-        archive = append_metering_records(archive, records, 6)
+        archive = append_metering_records(archive, records, 6, 0)
 
     assert len(archive) == 6
     assert archive[4] == records
 
 
+def test_append_metering_records_max_bytes():
+    archive = []
+    records = {
+        "bill_time": "2024-01-03T20:06:42.076972+00:00",
+        "metering_id": "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF",
+        "usage_records": [
+            {
+                "managed_node_count": 9,
+                "managed_systems": 6,
+                "reporting_time": "2024-01-03T20:06:42.076972+00:00"
+            }
+        ]
+    }
+
+    for i in range(4):
+        archive = append_metering_records(archive, records, 6, 456)
+
+    assert len(archive) == 2
+    assert archive[0] == records
+
+
 def test_archive_record(cba_pm, cba_config):
     record = {
         'billing_time': '2024-02-09T18:11:59.527064+00:00',