Merge pull request #178 from weecology/schema_field

Primary event detection

Snakefile

@@ -26,7 +26,7 @@ def flights_in_year_site(wildcards):
         flight_path = os.path.join(basepath, year, site, f"{flight}_projected.shp")
         # Assuming there is a tools.get_event function
         event = tools.get_event(flight_path)
-        if site == wildcards.site and year == wildcards.year and event == "primary":
+        if site == wildcards.site and year == wildcards.year and event[0] == "primary":
             flights_in_year_site.append(flight_path)
     return flights_in_year_site
 

combine_birds_site_year.py

@@ -19,7 +19,7 @@ def combine_files(bird_detection_files, year, site, score_thresh, savedir):
             eventdf["Site"] = tools.get_site(x)
             eventdf["Date"] = tools.get_date(x)
             eventdf["Year"] = tools.get_year(x)
-            eventdf["event"] = tools.get_event(x)
+            eventdf["event"], eventdf["file_postscript"] = tools.get_event(x)
             eventdf = eventdf[eventdf.score > score_thresh]
             df.append(eventdf)
         except IndexError as e:

everglades_workflow.sh

@@ -2,8 +2,8 @@
 #SBATCH --job-name=everglades_workflow 
 #SBATCH [email protected] 
 #SBATCH --mail-type=FAIL
-#SBATCH --gpus=a100:1
-#SBATCH --cpus-per-task=30
+#SBATCH --gpus=a100:4
+#SBATCH --cpus-per-task=60
 #SBATCH --mem=1200gb
 #SBATCH --time=80:00:00
 #SBATCH --partition=gpu
@@ -22,5 +22,5 @@ cd /blue/ewhite/everglades/EvergladesTools/Zooniverse
 
 snakemake --unlock
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] Starting Snakemake pipeline"
-snakemake --printshellcmds --keep-going --cores 30 --resources gpu=1 --rerun-incomplete --latency-wait 10 --use-conda
+snakemake --printshellcmds --keep-going --cores 60 --resources gpu=4 --rerun-incomplete --latency-wait 10 --use-conda
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] End"

mbtile.py

@@ -12,9 +12,6 @@ def create_mbtile(path, year, site, force_upload=False):
     basename = os.path.splitext(os.path.basename(path))[0]
     flight = basename.replace("_projected", "")
 
-    if tools.get_event(basename) == "primary":
-        flight = "_".join(basename.split('_')[0:4])
-
     working_dir = tools.get_working_dir()
     mbtiles_dir = os.path.join(working_dir, "mapbox", year, site)
     if not os.path.exists(mbtiles_dir):

tools.py

@@ -23,13 +23,18 @@ def get_event(path):
     This function returns "primary" for no event and events with the following values:
     "A", "a", "primary", "PRIMARY", or mixed case versions of "primary"
     """
-    path = os.path.basename(path)
-    regex = re.compile('\\w+_\\d+_\\d+_\\d+_(\\w+)_projected')
-    match = regex.match(path)
-    if match and match.group(1).upper() != "A" and match.group(1).upper() != "PRIMARY":
-        return match.group(1)
+    filename = os.path.basename(path)
+    regex = re.compile(r'\w+_\d+_\d+_\d+_(\w+)_projected')
+    match = regex.match(filename)
+
+    if match:
+        event = match.group(1).upper()
+        if event not in {"A", "PRIMARY"}:
+            return (event, "_" + event)
+        else:
+            return ("primary", "_" + event)
     else:
-        return "primary"
+        return ("primary", "")
 
 
 def get_site(path):