Detect extension locations from arches applications #10288

arches/app/const.py

@@ -13,3 +13,10 @@ class IntegrityCheck(Enum):
 
     def __str__(self):
         return IntegrityCheckDescriptions[self.value]
+
+
+class ExtensionType(Enum):
+    DATATYPES = "datatypes"
+    ETL_MODULES = "etl_modules"
+    FUNCTIONS = "functions"
+    SEARCH_COMPONENTS = "search_components"

arches/app/datatypes/datatypes.py

@@ -16,6 +16,8 @@
 
 from django.core.files.images import get_image_dimensions
 from django.db.models import fields
+
+from arches.app.const import ExtensionType
 from arches.app.datatypes.base import BaseDataType
 from arches.app.models import models
 from arches.app.models.system_settings import settings
@@ -98,7 +100,7 @@ def get_instance(self, datatype):
         try:
             datatype_instance = DataTypeFactory._datatype_instances[d_datatype.classname]
         except KeyError:
-            class_method = get_class_from_modulename(d_datatype.modulename, d_datatype.classname, settings.DATATYPE_LOCATIONS)
+            class_method = get_class_from_modulename(d_datatype.modulename, d_datatype.classname, ExtensionType.DATATYPES)
             datatype_instance = class_method(d_datatype)
             DataTypeFactory._datatype_instances[d_datatype.classname] = datatype_instance
             self.datatype_instances = DataTypeFactory._datatype_instances

arches/app/models/models.py

@@ -18,6 +18,7 @@
 import traceback
 import django.utils.timezone
 
+from arches.app.const import ExtensionType
 from arches.app.utils.module_importer import get_class_from_modulename
 from arches.app.utils.thumbnail_factory import ThumbnailGeneratorInstance
 from arches.app.models.fields.i18n import I18n_TextField, I18n_JSONField
@@ -460,7 +461,7 @@ def defaultconfig_json(self):
         return json_string
 
     def get_class_module(self):
-        return get_class_from_modulename(self.modulename, self.classname, settings.FUNCTION_LOCATIONS)
+        return get_class_from_modulename(self.modulename, self.classname, ExtensionType.FUNCTIONS)
 
 
 class FunctionXGraph(models.Model):
@@ -1045,7 +1046,7 @@ class Meta:
         db_table = "search_component"
 
     def get_class_module(self):
-        return get_class_from_modulename(self.modulename, self.classname, settings.SEARCH_COMPONENT_LOCATIONS)
+        return get_class_from_modulename(self.modulename, self.classname, ExtensionType.SEARCH_COMPONENTS)
 
     def toJSON(self):
         from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
@@ -1752,7 +1753,7 @@ class Meta:
         db_table = "etl_modules"
 
     def get_class_module(self):
-        return get_class_from_modulename(self.modulename, self.classname, settings.ETL_MODULE_LOCATIONS)
+        return get_class_from_modulename(self.modulename, self.classname, ExtensionType.ETL_MODULES)
 
 
 class LoadEvent(models.Model):

arches/app/search/components/base.py

@@ -1,3 +1,4 @@
+from arches.app.const import ExtensionType
 from arches.app.models import models
 from arches.app.models.system_settings import settings
 from arches.app.utils.module_importer import get_class_from_modulename
@@ -60,7 +61,7 @@ def get_filter(self, componentname):
             except:
                 filter_instance = None
                 class_method = get_class_from_modulename(
-                    search_filter.modulename, search_filter.classname, settings.SEARCH_COMPONENT_LOCATIONS
+                    search_filter.modulename, search_filter.classname, ExtensionType.SEARCH_COMPONENTS
                 )
                 if class_method:
                     filter_instance = class_method(self.request)

arches/app/utils/module_importer.py

@@ -1,17 +1,39 @@
 import importlib
 
+from arches.app.const import ExtensionType
+from arches.app.models.system_settings import settings
+
 
 def get_module(path, modulename=""):
     module = importlib.machinery.SourceFileLoader(modulename, path).load_module()
     return module
 
 
-def get_class_from_modulename(modulename, classname, directory_list):
+def get_directories(extension_type: ExtensionType):
+    core_root_dir = f"arches.app.{extension_type.value}"
+    if extension_type is ExtensionType.SEARCH_COMPONENTS:
+        core_root_dir = core_root_dir.replace("search_components", "search.components")
+
+    core_and_arches_app_dirs = [core_root_dir]
+    for arches_app in settings.ARCHES_APPLICATIONS:
+        core_and_arches_app_dirs.append(f"{arches_app}.{extension_type.value}")
+        core_and_arches_app_dirs.append(f"{arches_app}.pkg.extensions.{extension_type.value}")
+
+    filtered_settings_dirs = [
+        setting_dir for setting_dir in
+        getattr(settings, extension_type.value.upper()[:-1] + "_LOCATIONS")
+        if setting_dir not in core_and_arches_app_dirs
+    ]
+
+    return core_and_arches_app_dirs + filtered_settings_dirs
+
+
+def get_class_from_modulename(modulename, classname, extension_type: ExtensionType):
     mod_path = modulename.replace(".py", "")
     module = None
     import_success = False
     import_error = None
-    for directory in directory_list:
+    for directory in get_directories(extension_type):
         try:
             module = importlib.import_module(directory + ".%s" % mod_path)
             import_success = True

tests/base_test.py

@@ -17,6 +17,8 @@
 """
 
 import os
+from contextlib import contextmanager
+
 from django.test import TestCase
 from arches.app.models.graph import Graph
 from arches.app.models.models import Ontology
@@ -134,3 +136,20 @@ def setUp(self):
 
     def tearDown(self):
         pass
+
+
+@contextmanager
+def sync_overridden_test_settings_to_arches(*args, **kwargs):
+    """Django's @override_settings test util acts on django.conf.settings,
+    which is not enough for us, because we use SystemSettings at runtime.
+
+    This context manager swaps in the overridden django.conf.settings for SystemSettings.
+    """
+    from django.conf import settings as patched_settings
+
+    original_settings_wrapped = settings._wrapped
+    try:
+        settings._wrapped = patched_settings._wrapped
+        yield True
+    finally:
+        settings._wrapped = original_settings_wrapped

tests/test_settings.py

@@ -32,6 +32,8 @@
 TEST_ROOT = os.path.normpath(os.path.join(ROOT_DIR, "..", "tests"))
 APP_ROOT = ""
 
+ARCHES_APPLICATIONS = []
+
 MIN_ARCHES_VERSION = arches.__version__
 MAX_ARCHES_VERSION = arches.__version__
 

tests/utils/test_module_importer.py

@@ -0,0 +1,88 @@
+from django.test import TestCase, override_settings
+
+from arches.app.const import ExtensionType
+from arches.app.utils.module_importer import get_directories
+from tests.base_test import sync_overridden_test_settings_to_arches
+
+# these tests can be run from the command line via
+# python manage.py test tests.utils.test_module_importer --settings="tests.test_settings"
+
+class ModuleImporterTests(TestCase):
+    @override_settings(
+        APP_NAME="hiphop",
+        ARCHES_APPLICATIONS=["arches_for_music", "arches_for_dance"],
+        FUNCTION_LOCATIONS=[
+            "arches.app.functions",
+            # Include an example where one of the installed apps is explicitly given
+            "arches_for_music.pkg.extensions.functions",
+            "hiphop.functions",
+        ],
+    )
+    def test_arches_application_extension_explicit(self):
+        with sync_overridden_test_settings_to_arches():
+            function_dirs = get_directories(ExtensionType.FUNCTIONS)
+        self.assertEqual(function_dirs, [
+            "arches.app.functions",
+            "arches_for_music.functions",
+            "arches_for_music.pkg.extensions.functions",
+            "arches_for_dance.functions",
+            "arches_for_dance.pkg.extensions.functions",
+            "hiphop.functions",
+        ])
+
+    @override_settings(
+        APP_NAME="hiphop",
+        ARCHES_APPLICATIONS=["arches_for_music", "arches_for_dance"],
+        SEARCH_COMPONENT_LOCATIONS=[
+            # Same, but poorly ordered.
+            "hiphop.search_components",
+            "arches_for_music.search_components",
+            "arches.app.search.components",
+        ]
+    )
+    def test_arches_application_extension_explicit_poorly_ordered(self):
+        with sync_overridden_test_settings_to_arches():
+            search_dirs = get_directories(ExtensionType.SEARCH_COMPONENTS)
+        self.assertEqual(search_dirs, [
+            "arches.app.search.components",
+            "arches_for_music.search_components",
+            "arches_for_music.pkg.extensions.search_components",
+            "arches_for_dance.search_components",
+            "arches_for_dance.pkg.extensions.search_components",
+            "hiphop.search_components",
+        ])
+
+    @override_settings(
+        APP_NAME="hiphop",
+        ARCHES_APPLICATIONS=["arches_for_music", "arches_for_dance"],
+        ETL_MODULE_LOCATIONS=[
+            # App not given.
+            "arches.app.etl_modules",
+        ]
+    )
+    def test_arches_application_extension_implicit(self):
+        with sync_overridden_test_settings_to_arches():
+            etl_modules = get_directories(ExtensionType.ETL_MODULES)
+        self.assertEqual(etl_modules, [
+            "arches.app.etl_modules",
+            "arches_for_music.etl_modules",
+            "arches_for_music.pkg.extensions.etl_modules",
+            "arches_for_dance.etl_modules",
+            "arches_for_dance.pkg.extensions.etl_modules",
+        ])
+
+    @override_settings(
+        APP_NAME="hiphop",
+        ARCHES_APPLICATIONS=["arches_for_music", "arches_for_dance"],
+        DATATYPE_LOCATIONS=[],  # Nothing given.
+    )
+    def test_arches_application_extension_core_arches_implicit(self):
+        with sync_overridden_test_settings_to_arches():
+            datatypes = get_directories(ExtensionType.DATATYPES)
+        self.assertEqual(datatypes, [
+            "arches.app.datatypes",
+            "arches_for_music.datatypes",
+            "arches_for_music.pkg.extensions.datatypes",
+            "arches_for_dance.datatypes",
+            "arches_for_dance.pkg.extensions.datatypes",
+        ])