From ffff397ecbde2521497f1147fe0236ce0995294e Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Thu, 17 Dec 2020 08:43:47 +0200 Subject: [PATCH 01/10] Travis Update --- .travis.yml | 5 +++-- tests/test_settings.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e5e6e4..91ce28d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,9 @@ python: - "3.8" env: - - DJANGO=django==2.2.12 - - DJANGO=django==3.0.5 + - DJANGO=django==2.2.17 + - DJANGO=django==3.0.11 + - DJANGO=django==3.1.4 # command to install dependencies diff --git a/tests/test_settings.py b/tests/test_settings.py index ec28c7d..c5f5d58 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -10,6 +10,9 @@ 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'TEST': { + 'NAME': 'tst_db.sqlite3', + }, }, } @@ -25,7 +28,6 @@ 'django.contrib.sessions', 'django.contrib.messages', - # 'django.contrib.admin.apps.SimpleAdminConfig', 'django.contrib.staticfiles', 'slick_reporting', From a1cf8a60081d64667e960e447fe508c6db572e06 Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Thu, 17 Dec 2020 09:26:31 +0200 Subject: [PATCH 02/10] Travis fail debugging --- tests/test_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_settings.py b/tests/test_settings.py index c5f5d58..18f0a65 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -12,6 +12,7 @@ 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 'TEST': { 'NAME': 'tst_db.sqlite3', + 'MIGRATE': False }, }, } From 512767d09c7988b4187c2566999bbb2c2591368b Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Thu, 17 Dec 2020 09:59:37 +0200 Subject: [PATCH 03/10] Travis fail debugging 2 --- tests/test_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_settings.py b/tests/test_settings.py index 18f0a65..1bf3cf0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -56,3 +56,5 @@ }, ] STATIC_URL = '/static/' + +MIGRATION_MODULES = {'contenttypes': None, 'auth': None} From d544677fed343cc5cc348488bef04b5a09f41e84 Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Sun, 20 Dec 2020 08:16:18 +0200 Subject: [PATCH 04/10] init_preparation init --- slick_reporting/fields.py | 27 +++++++++++++++++++-------- slick_reporting/generator.py | 5 ++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/slick_reporting/fields.py b/slick_reporting/fields.py index 17731a9..99852ba 100644 --- a/slick_reporting/fields.py +++ b/slick_reporting/fields.py @@ -110,6 +110,21 @@ def apply_aggregation(self, queryset, group_by=''): queryset = queryset.aggregate(annotation) return queryset + def init_preparation(self, q_filters=None, kwargs_filters=None, **kwargs): + """ + Called by the generator to preparet he calculation of this field + it's requirements + :param q_filters: + :param kwargs_filters: + :param kwargs: + :return: + """ + kwargs_filters = kwargs_filters or {} + + dep_values = self._prepare_dependencies(q_filters, kwargs_filters.copy()) + + debit_results, credit_results = self.prepare(q_filters, kwargs_filters, **kwargs) + self._cache = debit_results, credit_results, dep_values + def prepare(self, q_filters=None, kwargs_filters=None, **kwargs): """ This is the first hook where you can customize the calculation away from the Django Query aggregation method @@ -122,10 +137,6 @@ def prepare(self, q_filters=None, kwargs_filters=None, **kwargs): :param kwargs: :return: """ - kwargs_filters = kwargs_filters or {} - - dep_values = self._prepare_dependencies(q_filters, kwargs_filters.copy()) - queryset = self.get_queryset() if q_filters: queryset = queryset.filter(*q_filters) @@ -148,8 +159,7 @@ def prepare(self, q_filters=None, kwargs_filters=None, **kwargs): credit_results = self.apply_aggregation(queryset, self.group_by) - self._cache = debit_results, credit_results, dep_values - return debit_results, credit_results, dep_values + return debit_results, credit_results def get_queryset(self): queryset = self.report_model.objects @@ -171,11 +181,12 @@ def _prepare_dependencies(self, q_filters=None, extra_filters=None, ): 'instance': dep} return values - def resolve(self, current_obj): + def resolve(self, current_obj, current_row=None): ''' Reponsible for getting the exact data from the prepared value :param cached: the returned data from prepare - :param current_obj: + :param current_obj: he value of group by id + :param current_row: the row in iteration :return: a solid number or value ''' cached = self._cache diff --git a/slick_reporting/generator.py b/slick_reporting/generator.py index 20de4d6..ce16afa 100644 --- a/slick_reporting/generator.py +++ b/slick_reporting/generator.py @@ -322,8 +322,7 @@ def _prepare_report_dependencies(self): if window == 'crosstab': q_filters = self._construct_crosstab_filter(col_data) - # print(f'preparing {report_class} for {window}') - report_class.prepare(q_filters, date_filter) + report_class.init_preparation(q_filters, date_filter) self.report_fields_classes[name] = report_class def _get_record_data(self, obj, columns): @@ -359,7 +358,7 @@ def _get_record_data(self, obj, columns): computation_class = self.report_fields_classes[name] except KeyError: continue - value = computation_class.resolve(group_by_val) + value = computation_class.resolve(group_by_val, data) if self.swap_sign: value = -value data[name] = value From d9c83afaf98f55922dcbc9e7baf3d67253e4fdc7 Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Sun, 20 Dec 2020 13:44:18 +0200 Subject: [PATCH 05/10] - Enhanced Field prepare flow - Add traversing for group_by --- CHANGELOG.md | 6 ++++++ slick_reporting/fields.py | 2 +- slick_reporting/generator.py | 18 +++++++++++------- tests/test_generator.py | 10 ++++++++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5baed07..c17440d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [0.5.3] + +- Enhanced Field prepare flow +- Add traversing for group_by + + ## [0.5.2] - Enhanced Time Series Plot total HighChart by accenting the categories diff --git a/slick_reporting/fields.py b/slick_reporting/fields.py index 99852ba..fab5031 100644 --- a/slick_reporting/fields.py +++ b/slick_reporting/fields.py @@ -177,7 +177,7 @@ def _prepare_dependencies(self, q_filters=None, extra_filters=None, ): for dep_class in self._require_classes: dep = dep_class(self.plus_side_q, self.minus_side_q, self.report_model, date_field=self.date_field, group_by=self.group_by) - values[dep.name] = {'results': dep.prepare(q_filters, extra_filters), + values[dep.name] = {'results': dep.init_preparation(q_filters, extra_filters), 'instance': dep} return values diff --git a/slick_reporting/generator.py b/slick_reporting/generator.py index ce16afa..ddf82ea 100644 --- a/slick_reporting/generator.py +++ b/slick_reporting/generator.py @@ -193,14 +193,18 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d # todo validate columns is not empty (if no time series / cross tab) if self.group_by: + search_field = self.group_by.split('__')[0] try: - self.group_by_field = [x for x in self.report_model._meta.fields if x.name == self.group_by][0] + self.group_by_field = [x for x in self.report_model._meta.fields if x.name == search_field][0] + except IndexError: raise ImproperlyConfigured( f'Can not find group_by field:{self.group_by} in report_model {self.report_model} ') - - self.focus_field_as_key = self.group_by - self.group_by_field_attname = self.group_by_field.attname + self.focus_field_as_key = self.group_by_field + if '__' not in self.group_by: + self.group_by_field_attname = self.group_by_field.attname + else: + self.group_by_field_attname = self.group_by else: self.focus_field_as_key = None self.group_by_field_attname = None @@ -237,10 +241,10 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d else: self.main_queryset = self._apply_queryset_options(main_queryset) if type(self.group_by_field) is ForeignKey: - ids = self.main_queryset.values_list(self.group_by_field.attname).distinct() + ids = self.main_queryset.values_list(self.group_by_field_attname).distinct() self.main_queryset = self.group_by_field.related_model.objects.filter(pk__in=ids).values() else: - self.main_queryset = self.main_queryset.distinct().values(self.group_by_field.attname) + self.main_queryset = self.main_queryset.distinct().values(self.group_by_field_attname) else: if self.time_series_pattern: self.main_queryset = [{}] @@ -401,7 +405,7 @@ def check_columns(cls, columns, group_by, report_model, ): """ group_by_model = None if group_by: - group_by_field = [x for x in report_model._meta.fields if x.name == group_by][0] + group_by_field = [x for x in report_model._meta.fields if x.name == group_by.split('__')[0]][0] if group_by_field.is_relation: group_by_model = group_by_field.related_model else: diff --git a/tests/test_generator.py b/tests/test_generator.py index e135843..93694b0 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -138,6 +138,16 @@ def test_gather_dependencies_for_time_series(self): self.assertTrue(report._report_fields_dependencies) + def test_group_by_traverse(self): + report = ReportGenerator(report_model=SimpleSales, group_by='client__slug', + columns=['slug', 'name'], + time_series_pattern='monthly', + date_field='doc_date', + time_series_columns=['__debit__', '__credit__', '__balance__', '__total__'] + ) + + self.assertTrue(report._report_fields_dependencies) + def test_db_field_column_verbose_name(self): report = GenericGenerator() field_list = report.get_list_display_columns() From ab387a8a5fba1bd728d8c6545506c92746399abc Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Mon, 21 Dec 2020 09:55:50 +0200 Subject: [PATCH 06/10] - Enhanced Test suit to accept individual tests - Add traversing for group_by --- runtests.py | 15 ++++++++++++++- slick_reporting/generator.py | 9 +++++---- tests/models.py | 7 +++++++ tests/test_generator.py | 31 +++++++++++++++++++++++++------ tests/tests.py | 7 +++---- 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/runtests.py b/runtests.py index 001be57..5f753e1 100644 --- a/runtests.py +++ b/runtests.py @@ -2,15 +2,28 @@ import os import sys +import argparse import django from django.conf import settings from django.test.utils import get_runner if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run the Django Slick Reporting test suite.") + parser.add_argument( + 'modules', nargs='*', metavar='module', + help='Optional path(s) to test modules; e.g. "i18n" or ' + '"i18n.tests.TranslationTests.test_lazy_objects".', + ) + options = parser.parse_args() + + options.modules = [os.path.normpath(labels) for labels in options.modules] + + os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_settings' django.setup() TestRunner = get_runner(settings) test_runner = TestRunner() - failures = test_runner.run_tests(["tests"]) + failures = test_runner.run_tests(options.modules) + # failures = test_runner.run_tests(["tests"]) sys.exit(bool(failures)) diff --git a/slick_reporting/generator.py b/slick_reporting/generator.py index ddf82ea..e259e6a 100644 --- a/slick_reporting/generator.py +++ b/slick_reporting/generator.py @@ -193,7 +193,8 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d # todo validate columns is not empty (if no time series / cross tab) if self.group_by: - search_field = self.group_by.split('__')[0] + group_by_split = self.group_by.split('__') + search_field = group_by_split[0] try: self.group_by_field = [x for x in self.report_model._meta.fields if x.name == search_field][0] @@ -205,6 +206,7 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d self.group_by_field_attname = self.group_by_field.attname else: self.group_by_field_attname = self.group_by + else: self.focus_field_as_key = None self.group_by_field_attname = None @@ -240,7 +242,7 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d else: self.main_queryset = self._apply_queryset_options(main_queryset) - if type(self.group_by_field) is ForeignKey: + if type(self.group_by_field) is ForeignKey and '__' not in self.group_by: ids = self.main_queryset.values_list(self.group_by_field_attname).distinct() self.main_queryset = self.group_by_field.related_model.objects.filter(pk__in=ids).values() else: @@ -250,7 +252,6 @@ def __init__(self, report_model=None, main_queryset=None, start_date=None, end_d self.main_queryset = [{}] else: self.main_queryset = self._apply_queryset_options(main_queryset, self.get_database_columns()) - self._prepare_report_dependencies() def _apply_queryset_options(self, query, fields=None): @@ -453,7 +454,7 @@ def check_columns(cls, columns, group_by, report_model, ): } else: # A database field - model_to_use = group_by_model if group_by else report_model + model_to_use = group_by_model if group_by and '__' not in group_by else report_model try: if '__' in col: # A traversing link order__client__email diff --git a/tests/models.py b/tests/models.py index bfc2f0e..7577c57 100644 --- a/tests/models.py +++ b/tests/models.py @@ -6,9 +6,16 @@ class Product(models.Model): + CATEGORY_CHOICES = ( + ('tiny', 'tiny'), + ('small', 'small'), + ('medium', 'medium'), + ('big', 'big'), + ) slug = models.CharField(max_length=200, verbose_name=_('Slug')) name = models.CharField(max_length=200, verbose_name=_('Name')) sku = models.CharField(max_length=200, default='', blank=True) + category = models.CharField(max_length=10, choices=CATEGORY_CHOICES) notes = models.TextField() class Meta: diff --git a/tests/test_generator.py b/tests/test_generator.py index 93694b0..ffc9d58 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,7 +1,10 @@ from datetime import datetime import pytz +from django.db.models import Sum from django.test import TestCase + +from slick_reporting.fields import SlickReportField from slick_reporting.generator import ReportGenerator from slick_reporting.helpers import get_foreign_keys from .models import OrderLine @@ -9,7 +12,7 @@ from .report_generators import GeneratorWithAttrAsColumn, CrosstabOnClient, GenericGenerator, GroupByCharField, \ TimeSeriesCustomDates -from .tests import BaseTestData +from .tests import BaseTestData, year from .models import SimpleSales @@ -44,7 +47,16 @@ def test_get_crosstab_parsed_columns(self): self.assertTrue('is_summable' in col.keys(), col) -class GeneratorReportStructureTest(TestCase): +class GeneratorReportStructureTest(BaseTestData, TestCase): + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + SimpleSales.objects.create( + doc_date=datetime(year, 3, 2), client=cls.client3, + product=cls.product3, quantity=30, price=10) + + def test_time_series_columns_inclusion(self): x = ReportGenerator(OrderLine, date_field='order__date_placed', group_by='client', columns=['name', '__time_series__'], @@ -115,6 +127,7 @@ def test_attr_as_column(self): self.assertEqual(columns_data[0]['verbose_name'], 'get_data_verbose_name') data = report.get_report_data() + self.assertIsNot(data, []) # todo def test_improper_group_by(self): @@ -139,14 +152,20 @@ def test_gather_dependencies_for_time_series(self): self.assertTrue(report._report_fields_dependencies) def test_group_by_traverse(self): - report = ReportGenerator(report_model=SimpleSales, group_by='client__slug', - columns=['slug', 'name'], - time_series_pattern='monthly', + report = ReportGenerator(report_model=SimpleSales, group_by='product__category', + columns=['product__category', SlickReportField.create(Sum, 'value'), '__total__'], + # time_series_pattern='monthly', date_field='doc_date', - time_series_columns=['__debit__', '__credit__', '__balance__', '__total__'] + # time_series_columns=['__debit__', '__credit__', '__balance__', '__total__'] ) self.assertTrue(report._report_fields_dependencies) + data = report.get_report_data() + # import pdb; + # pdb.set_trace() + self.assertNotEqual(data, []) + self.assertEqual(data[0]['product__category'], 'small') + self.assertEqual(data[1]['product__category'], 'big') def test_db_field_column_verbose_name(self): report = GenericGenerator() diff --git a/tests/tests.py b/tests/tests.py index e9179e4..481f03a 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -40,9 +40,9 @@ def setUpTestData(cls): cls.client3 = Client.objects.create(name='Client 3') cls.clientIdle = Client.objects.create(name='Client Idle') - cls.product1 = Product.objects.create(name='Product 1') - cls.product2 = Product.objects.create(name='Product 2') - cls.product3 = Product.objects.create(name='Product 3') + cls.product1 = Product.objects.create(name='Product 1', category='small') + cls.product2 = Product.objects.create(name='Product 2', category='medium') + cls.product3 = Product.objects.create(name='Product 3', category='big') SimpleSales.objects.create( doc_date=datetime.datetime(year, 1, 2), client=cls.client1, @@ -126,7 +126,6 @@ def test_productclientsalesmatrix(self): self.assertEqual(data[0]['__total__CT%s' % self.client2.pk], 600) self.assertEqual(data[0]['__total__CT----'], 900) - def test_show_empty_records(self): report = report_generators.ClientTotalBalance() data = report.get_report_data() From 2f1bff2522678e7b7156425ff1e2e4920e38ba15 Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Mon, 21 Dec 2020 21:38:41 +0200 Subject: [PATCH 07/10] - Move select2 to base.html to be easily replaceable --- slick_reporting/templates/slick_reporting/base.html | 3 +++ slick_reporting/templates/slick_reporting/simple_report.html | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/slick_reporting/templates/slick_reporting/base.html b/slick_reporting/templates/slick_reporting/base.html index b3bcd0c..4087a27 100644 --- a/slick_reporting/templates/slick_reporting/base.html +++ b/slick_reporting/templates/slick_reporting/base.html @@ -45,6 +45,9 @@ + {% block js_script %} {% endblock %} diff --git a/slick_reporting/templates/slick_reporting/simple_report.html b/slick_reporting/templates/slick_reporting/simple_report.html index 345196b..d326973 100644 --- a/slick_reporting/templates/slick_reporting/simple_report.html +++ b/slick_reporting/templates/slick_reporting/simple_report.html @@ -167,7 +167,6 @@

Results

} - $('select').select2(); $('table').DataTable(); $('.nav-charts').find('a:first').trigger('click'); From 3d480ee826d3860d2ecf8c43d963e13cf34d11ad Mon Sep 17 00:00:00 2001 From: Ramez Issac Date: Mon, 21 Dec 2020 22:00:42 +0200 Subject: [PATCH 08/10] - Better arrange so resources are better placed together --- slick_reporting/templates/slick_reporting/base.html | 3 --- slick_reporting/templates/slick_reporting/simple_report.html | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/slick_reporting/templates/slick_reporting/base.html b/slick_reporting/templates/slick_reporting/base.html index 4087a27..8c60515 100644 --- a/slick_reporting/templates/slick_reporting/base.html +++ b/slick_reporting/templates/slick_reporting/base.html @@ -42,9 +42,6 @@ integrity="JnbsSLBmv2/R0fUmF2XYIcAEMPHEAO51Gitn9IjL4l89uFTIgtLF1+jqIqqd9FSk" crossorigin="anonymous"> - - - diff --git a/slick_reporting/templates/slick_reporting/simple_report.html b/slick_reporting/templates/slick_reporting/simple_report.html index d326973..68cbc1d 100644 --- a/slick_reporting/templates/slick_reporting/simple_report.html +++ b/slick_reporting/templates/slick_reporting/simple_report.html @@ -53,6 +53,9 @@

Results

{% endblock %} {% block js_script %} + + +