From 7718fa67e7c9b3214b50cbddbc4cd3c8c6ae7b24 Mon Sep 17 00:00:00 2001 From: Sean Marlow Date: Tue, 9 Jan 2024 15:06:25 -0500 Subject: [PATCH] Implement archive record function And integrate archive into the adapter event loop. Each billing cycle is archived with metering and usage info included. With a default archive length of 6 months. --- csp_billing_adapter/adapter.py | 2 + csp_billing_adapter/archive.py | 55 +++++++++++++++++++++++++++ csp_billing_adapter/bill_utils.py | 15 ++++++++ csp_billing_adapter/memory_archive.py | 16 ++++++++ tests/unit/conftest.py | 3 ++ tests/unit/test_archive.py | 45 +++++++++++++++++++++- 6 files changed, 135 insertions(+), 1 deletion(-) diff --git a/csp_billing_adapter/adapter.py b/csp_billing_adapter/adapter.py index 2962030..c207ea4 100644 --- a/csp_billing_adapter/adapter.py +++ b/csp_billing_adapter/adapter.py @@ -49,6 +49,7 @@ csp_hookspecs, hookspecs, storage_hookspecs, + archive_hookspecs, hookimpls ) @@ -71,6 +72,7 @@ def get_plugin_manager() -> pluggy.PluginManager: pm.add_hookspecs(hookspecs) pm.add_hookspecs(csp_hookspecs) pm.add_hookspecs(storage_hookspecs) + pm.add_hookspecs(archive_hookspecs) pm.register(hookimpls) pm.load_setuptools_entrypoints('csp_billing_adapter') return pm diff --git a/csp_billing_adapter/archive.py b/csp_billing_adapter/archive.py index d86ac6a..bf6b5b6 100644 --- a/csp_billing_adapter/archive.py +++ b/csp_billing_adapter/archive.py @@ -16,6 +16,16 @@ """Utility functions for handling a rolling dictionary archive.""" +import functools +import logging + +from csp_billing_adapter.config import Config +from csp_billing_adapter.utils import retry_on_exception + +log = logging.getLogger('CSPBillingAdapter') + +DEFAULT_RETENTION_PERIOD = 6 # in months + def append_metering_records( archive: list, @@ -45,3 +55,48 @@ def append_metering_records( return archive[1:] else: return archive + + +def archive_record( + hook, + config: Config, + billing_record: dict +) -> None: + """ + :param hook: + The Pluggy plugin manager hook that will be + used to call the meter_billing operation. + :param config: + The configuration specifying the metrics that + need to be processed in the usage records list. + :param billing_record: + The dictionary containing the most recent + metering and usage records to be archived. + """ + archive = retry_on_exception( + functools.partial( + hook.get_metering_archive, + config=config, + ), + logger=log, + func_name="hook.get_metering_archive" + ) + + if archive is None: + archive = [] + + archive = append_metering_records( + archive, + billing_record, + config.archive_retention_period or DEFAULT_RETENTION_PERIOD + ) + + retry_on_exception( + functools.partial( + hook.save_metering_archive, + config=config, + archive_data=archive + ), + logger=log, + func_name="hook.save_metering_archive" + ) diff --git a/csp_billing_adapter/bill_utils.py b/csp_billing_adapter/bill_utils.py index d2b3f94..6f3c132 100644 --- a/csp_billing_adapter/bill_utils.py +++ b/csp_billing_adapter/bill_utils.py @@ -39,6 +39,7 @@ string_to_date ) from csp_billing_adapter.config import Config +from csp_billing_adapter.archive import archive_record log = logging.getLogger('CSPBillingAdapter') @@ -655,3 +656,17 @@ def process_metering( ) csp_config['usage'] = billable_usage csp_config['last_billed'] = metering_time + + # Save last usage and metering records to archive + billing_record = { + 'billing_time': metering_time, + 'billing_status': billing_status, + 'billed_usage': billed_dimensions, + 'usage_records': billable_records + } + + archive_record( + hook, + config, + billing_record + ) diff --git a/csp_billing_adapter/memory_archive.py b/csp_billing_adapter/memory_archive.py index 4659c6a..1e5057a 100644 --- a/csp_billing_adapter/memory_archive.py +++ b/csp_billing_adapter/memory_archive.py @@ -20,6 +20,10 @@ import csp_billing_adapter +from csp_billing_adapter.config import Config + +memory_archive = [] + log = logging.getLogger('CSPBillingAdapter') @@ -27,3 +31,15 @@ def get_archive_location(): """Retrieve archive location.""" return '/tmp/fake_archive.json' + + +@csp_billing_adapter.hookimpl(trylast=True) +def get_metering_archive(config: Config): + return memory_archive.copy() + + +@csp_billing_adapter.hookimpl(trylast=True) +def save_metering_archive(config: Config, archive_data: list): + global memory_archive + + memory_archive = archive_data diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index a84b256..753caeb 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -107,6 +107,9 @@ def cba_pm(cba_config): # reset the in-memory csp_config to empty pm.hook.save_csp_config(config=cba_config, csp_config=dict()) + # reset the in-memory archive to empty + pm.hook.save_metering_archive(config=cba_config, archive_data=list()) + return pm diff --git a/tests/unit/test_archive.py b/tests/unit/test_archive.py index 662346d..07db2b1 100644 --- a/tests/unit/test_archive.py +++ b/tests/unit/test_archive.py @@ -18,7 +18,10 @@ for the archive util functions. """ -from csp_billing_adapter.archive import append_metering_records +from csp_billing_adapter.archive import ( + append_metering_records, + archive_record +) def test_append_metering_records(): @@ -40,3 +43,43 @@ def test_append_metering_records(): assert len(archive) == 6 assert archive[4] == records + + +def test_archive_record(cba_pm, cba_config): + record = { + 'billing_time': '2024-02-09T18:11:59.527064+00:00', + 'billing_status': { + 'tier_1': { + 'record_id': 'd92c6e6556b14770994f5b64ebe3d339', + 'status': 'succeeded' + } + }, + 'billed_usage': { + 'tier_1': 10 + }, + 'usage_records': [ + { + 'managed_node_count': 9, + 'reporting_time': '2024-01-09T18:11:59.527673+00:00', + 'base_product': 'cpe:/o:suse:product:v1.2.3' + }, + { + 'managed_node_count': 9, + 'reporting_time': '2024-01-09T18:11:59.529096+00:00', + 'base_product': 'cpe:/o:suse:product:v1.2.3' + }, + { + 'managed_node_count': 10, + 'reporting_time': '2024-01-09T18:11:59.531424+00:00', + 'base_product': 'cpe:/o:suse:product:v1.2.3' + } + ] + } + archive_record( + cba_pm.hook, + cba_config, + record + ) + archive = cba_pm.hook.get_metering_archive(config=cba_config) + assert len(archive) == 1 + assert archive[0] == record