support airflow dag restructure

pelotoncycle · Dec 3, 2023 · d3b3154 · d3b3154
1 parent b29bd45
commit d3b3154
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/dagfactory/dagfactory.py b/dagfactory/dagfactory.py
@@ -109,9 +109,9 @@ def from_directory(cls, config_dir, globals: Dict[str, Any], parent_default_conf
             if os.path.isdir(sub_fpath):
                 cls.from_directory(sub_fpath, globals, default_config)
             elif os.path.isfile(sub_fpath) and sub_fpath.split('.')[-1] in ALLOWED_CONFIG_FILE_SUFFIX:
-                if 'git/repo/dags/data_engineering' in sub_fpath:
-                    print("sub_fpath=" + sub_fpath)
-                    if CONFIG_FILENAME_REGEX.match(sub_fpath.split("/")[-1]):
+                if 'owner' not in default_config['default_args']:
+                    if 'git/repo/dags/data_engineering' in sub_fpath:
+                        print("sub_fpath=" + sub_fpath)
                         print("config_filename=" + sub_fpath.split("/")[-1])
                         if 'owner' not in default_config['default_args']:
                             print("owner="+sub_fpath.split("/")[4])
@@ -139,7 +139,6 @@ def from_directory(cls, config_dir, globals: Dict[str, Any], parent_default_conf
 
         # in the end we want to surface the error messages if there's any
         if import_failures:
-            print("import_failure="+import_failures)
             # reformat import_failures so they are reader friendly
             import_failures_reformatted = ''
             for import_loc, import_trc in import_failures.items():