Merge pull request #2 from henrykironde/new_workflow

Limit field names to ten characters
weecology · Apr 25, 2024 · 8d6c8ae · 8d6c8ae
2 parents af218c8 + a123183
commit 8d6c8ae
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 30 deletions.
diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml
@@ -30,11 +30,11 @@ jobs:
       - uses: mamba-org/provision-with-micromamba@v15
         with:
           environment-name: Zooniverse
-          environment-file: Zooniverse/dev_environment.yml
+          environment-file: dev_environment.yml
           python-version: "${{ matrix.python-version }}"
           cache-env: true
           cache-downloads: true
 
       - name: "Run tests"
         run: |
-          yapf -d --recursive ./Zooniverse/ --style=.style.yapf 2>&1
+          yapf -d ./*.py --style=.style.yapf 2>&1
diff --git a/Snakefile b/Snakefile
@@ -38,7 +38,7 @@ rule all:
                zip, site=SITES, year=YEARS, flight=FLIGHTS),
         expand(f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp",
                zip, site=SITES, year=YEARS),
-        expand(f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles",
+        expand(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles",
                zip, site=SITES, year=YEARS, flight=FLIGHTS)
 
 
@@ -131,7 +131,7 @@ rule upload_mapbox:
     input:
         f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles"
     output:
-        f"{working_dir}/mapbox/{{year}}/{{site}}/{{flight}}.mbtiles"
+        touch(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles")
     conda:
         "EvergladesTools"
     shell:

diff --git a/combine_nests.py b/combine_nests.py
@@ -31,8 +31,8 @@ def load_shapefile(x):
         'last_obs': 'str',
         'num_obs': 'int',
         'species': 'str',
-        'sum_top1_s': 'float',
-        'num_obs_to': 'int',
+        'sum_top1': 'float',
+        'num_top1': 'int',
         'bird_match': 'str'
     })
     shp["site"] = get_site(x)

diff --git a/everglades_workflow.sh b/everglades_workflow.sh
@@ -4,7 +4,7 @@
 #SBATCH --mail-type=FAIL
 #SBATCH --gpus=a100:4
 #SBATCH --cpus-per-task=60
-#SBATCH --mem=1200gb
+#SBATCH --mem=600gb
 #SBATCH --time=80:00:00
 #SBATCH --partition=gpu
 #SBATCH --output=/blue/ewhite/everglades/EvergladesTools/logs/everglades_workflow.out

diff --git a/nest_detection.py b/nest_detection.py
@@ -68,12 +68,12 @@ def compare_site(gdf):
         # add target info to match
         row = geopandas.GeoDataFrame(pd.DataFrame(row).transpose(), crs=matches.crs)
         matches = geopandas.GeoDataFrame(pd.concat([matches, row], ignore_index=True))
-        matches["target_index"] = index
+        matches["target_ind"] = index
         matches = matches.rename(columns={
-            "xmin": "matched_xmin",
-            "max": "matched_xmax",
-            "ymin": "matched_ymin",
-            "ymax": "matched_ymax"
+            "xmin": "match_xmin",
+            "xmax": "match_xmax",
+            "ymin": "match_ymin",
+            "ymax": "match_ymax"
         })
 
         results.append(matches)
@@ -82,8 +82,8 @@ def compare_site(gdf):
         results = pd.concat(results)
     else:
         results = pd.DataFrame(columns=[
-            'matched_xmin', 'matched_ymin', 'xmax', 'matched_ymax', 'label', 'score', 'image_path', 'Date', 'bird_id',
-            'target_index', 'geometry'
+            'match_xmin', 'match_ymin', 'match_xmax', 'match_ymax', 'label', 'score', 'image_path', 'Site', 'Date',
+            'Year', 'event', 'file_posts', 'bird_id', 'target_ind'
         ])
 
     return results
@@ -102,19 +102,20 @@ def detect_nests(bird_detection_file, year, site, savedir):
     schema = {
         "geometry": "Polygon",
         "properties": {
-            'matched_xmin': 'float',
-            'matched_ymin': 'float',
-            'xmax': 'float',
-            'matched_ymax': 'float',
+            'match_xmin': 'float',
+            'match_ymin': 'float',
+            'match_xmax': 'float',
+            'match_ymax': 'float',
             'label': 'str',
             'score': 'float',
             'image_path': 'str',
             'Site': 'str',
             'Date': 'str',
             'Year': 'str',
-            'bird_id': 'int',
             'event': 'str',
-            'target_index': 'int'
+            'file_posts': 'str',
+            'bird_id': 'int',
+            'target_ind': 'int'
         }
     }
     if not results.empty:

diff --git a/process_nests.py b/process_nests.py
@@ -55,13 +55,13 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
             top_score_data = summed_scores[summed_scores['sum'] == max(summed_scores['sum'])].reset_index()
             nest_info = nest_data.groupby(['Site', 'Year', 'target_ind']).agg({
                 'Date': ['min', 'max', 'count'],
-                'matched_xm': ['mean'],
-                'matched_ym': ['mean'],
-                'xmax': ['mean'],
-                'matched__1': ['mean']
+                'match_xmin': ['mean'],
+                'match_ymin': ['mean'],
+                'match_xmax': ['mean'],
+                'match_ymax': ['mean']
             }).reset_index()
-            xmean = (nest_info['matched_xm']['mean'][0] + nest_info['xmax']['mean']) / 2
-            ymean = (nest_info['matched_ym']['mean'][0] + nest_info['matched__1']['mean']) / 2
+            xmean = (nest_info['match_xmin']['mean'][0] + nest_info['match_xmax']['mean']) / 2
+            ymean = (nest_info['match_ymin']['mean'][0] + nest_info['match_ymax']['mean']) / 2
             bird_match = ",".join([str(x) for x in nest_data["bird_id"]])
             nests.append([
                 target_ind, nest_info['Site'][0], nest_info['Year'][0], xmean[0], ymean[0], nest_info['Date']['min'][0],
@@ -77,7 +77,7 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
         nests = pd.DataFrame(nests,
                              columns=[
                                  'nest_id', 'Site', 'Year', 'xmean', 'ymean', 'first_obs', 'last_obs', 'num_obs',
-                                 'species', 'sum_top1_score', 'num_obs_top1', 'bird_match'
+                                 'species', 'sum_top1', 'num_top1', 'bird_match'
                              ])
         nests_shp = geopandas.GeoDataFrame(nests, geometry=geopandas.points_from_xy(nests.xmean, nests.ymean))
         nests_shp.crs = nests_data.crs
@@ -95,8 +95,8 @@ def process_nests(nest_file, year, site, savedir, min_score=0.3, min_detections=
                 'last_obs': 'str',
                 'num_obs': 'int',
                 'species': 'str',
-                'sum_top1_score': 'float',
-                'num_obs_top1': 'int',
+                'sum_top1': 'float',
+                'num_top1': 'int',
                 'bird_match': 'str'
             }
         }

diff --git a/upload_mapbox.py b/upload_mapbox.py
@@ -56,7 +56,7 @@ def retrieve_upload_status(self, upload_id):
 
 def get_credentials():
     """Get credentials from mapbox.ini"""
-    with open("/blue/ewhite/everglades/mapbox/mapbox.ini", "rb") as f:
+    with open("/blue/ewhite/everglades/mapbox.ini", "rb") as f:
         toml_dict = tomli.load(f)
         access_token = toml_dict['mapbox']['access-token']
     return access_token