Update working dir to everwatch-workflow

weecology · May 8, 2024 · a663f3f · a663f3f
1 parent a503b60
commit a663f3f
Show file tree

Hide file tree

Showing 11 changed files with 42 additions and 41 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,11 @@
-*tokens.py
-.snakemake/**
-logs/**
-lightning_logs
 *.DS_Store*
 *.idea*
 *.pytest_cache*
 *__pycache__*
 *_cache*
+*tokens.py
+.snakemake/**
 .vscode/launch.json
+App/Zooniverse/*
+lightning_logs
+logs/**
diff --git a/README.md b/README.md
@@ -69,5 +69,5 @@ The output shapefiles for (4) contain the predicted nest polygon, site, date and
 
 ## Logs
 
-The logs are located in `/blue/ewhite/everglades/EvergladesTools/logs`
-Checkout the current cronjob in `/blue/ewhite/everglades/EvergladesTools/everglades_workflow.sh`
+The logs are located in `/blue/ewhite/everglades/everwatch-workflow/logs`
+Checkout the current cronjob in `/blue/ewhite/everglades/everwatch-workflow/everglades_workflow.sh`
diff --git a/SLURM/mapbox_upload.sbatch b/SLURM/mapbox_upload.sbatch
@@ -12,5 +12,5 @@
 
 conda activate DeepTreeAttention
 
-cd /home/b.weinstein/EvergladesTools/Zooniverse
+cd /home/b.weinstein/everwatch-workflow
 python upload_mapbox.py
diff --git a/Snakefile b/Snakefile
@@ -2,7 +2,7 @@ import glob
 import os
 import tools
 
-configfile: "/blue/ewhite/everglades/EvergladesTools/snakemake_config.yml"
+configfile: "/blue/ewhite/everglades/everwatch-workflow/snakemake_config.yml"
 
 # Check if the test environment variable exists
 test_env_name = "TEST_ENV"
@@ -32,8 +32,8 @@ def flights_in_year_site(wildcards):
 
 rule all:
     input:
-        f"{working_dir}/EvergladesTools/App/Zooniverse/data/PredictedBirds.zip",
-        f"{working_dir}/EvergladesTools/App/Zooniverse/data/nest_detections_processed.zip",
+        f"{working_dir}/everwatch-workflow/App/Zooniverse/data/PredictedBirds.zip",
+        f"{working_dir}/everwatch-workflow/App/Zooniverse/data/nest_detections_processed.zip",
         expand(f"{working_dir}/predictions/{{year}}/{{site}}/{{flight}}_projected.shp",
                zip, site=SITES, year=YEARS, flight=FLIGHTS),
         expand(f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp",
@@ -49,7 +49,7 @@ rule project_mosaics:
         projected=f"{working_dir}/projected_mosaics/{{year}}/{{site}}/{{flight}}_projected.tif",
         webmercator=f"{working_dir}/projected_mosaics/webmercator/{{year}}/{{site}}/{{flight}}_projected.tif"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python project_orthos.py {input.orthomosaic}"
 
@@ -59,7 +59,7 @@ rule predict_birds:
     output:
         f"{working_dir}/predictions/{{year}}/{{site}}/{{flight}}_projected.shp"
     conda:
-        "EvergladesTools"
+        "everwatch"
     resources:
         gpu=1
     shell:
@@ -71,7 +71,7 @@ rule combine_birds_site_year:
     output:
         f"{working_dir}/predictions/{{year}}/{{site}}/{{site}}_{{year}}_combined.shp"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python combine_birds_site_year.py {input}"
 
@@ -80,9 +80,9 @@ rule combine_predicted_birds:
         expand(f"{working_dir}/predictions/{{year}}/{{site}}/{{site}}_{{year}}_combined.shp",
                zip, site=SITES_SY, year=YEARS_SY)
     output:
-        f"{working_dir}/EvergladesTools/App/Zooniverse/data/PredictedBirds.zip"
+        f"{working_dir}/everwatch-workflow/App/Zooniverse/data/PredictedBirds.zip"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python combine_bird_predictions.py {input}"
 
@@ -92,7 +92,7 @@ rule detect_nests:
     output:
         f"{working_dir}/detected_nests/{{year}}/{{site}}/{{site}}_{{year}}_detected_nests.shp"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python nest_detection.py {input}"
 
@@ -102,7 +102,7 @@ rule process_nests:
     output:
         f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python process_nests.py {input}"
 
@@ -111,9 +111,9 @@ rule combine_nests:
         expand(f"{working_dir}/processed_nests/{{year}}/{{site}}/{{site}}_{{year}}_processed_nests.shp",
                zip, site=SITES_SY, year=YEARS_SY)
     output:
-        f"{working_dir}/EvergladesTools/App/Zooniverse/data/nest_detections_processed.zip"
+        f"{working_dir}/everwatch-workflow/App/Zooniverse/data/nest_detections_processed.zip"
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python combine_nests.py {input}"
 
@@ -133,6 +133,6 @@ rule upload_mapbox:
     output:
         touch(f"{working_dir}/mapbox/last_uploaded/{{year}}/{{site}}/{{flight}}.mbtiles")
     conda:
-        "EvergladesTools"
+        "everwatch"
     shell:
         "python upload_mapbox.py {input}"
diff --git a/combine_bird_predictions.py b/combine_bird_predictions.py
@@ -19,7 +19,7 @@ def combine(paths):
 if __name__ == "__main__":
     working_dir = tools.get_working_dir()
     predictions_path = f"{working_dir}/predictions/"
-    output_path = f"{working_dir}/EvergladesTools/App/Zooniverse/data"
+    output_path = f"{working_dir}/everwatch-workflow/App/Zooniverse/data"
 
     predictions = sys.argv[1:]
     # write output to zooniverse app

diff --git a/combine_nests.py b/combine_nests.py
@@ -55,7 +55,7 @@ def combine(paths):
 if __name__ == "__main__":
     working_dir = tools.get_working_dir()
     nests_path = f"{working_dir}/processed_nests/"
-    output_path = f"{working_dir}/EvergladesTools/App/Zooniverse/data/"
+    output_path = f"{working_dir}/everwatch-workflow/App/Zooniverse/data/"
 
     nest_files = sys.argv[1:]
     # write output to zooniverse app

diff --git a/environment.yml b/environment.yml
@@ -1,4 +1,4 @@
-name: EvergladesTools
+name: everwatch
 channels: 
   - defaults
   - conda-forge

diff --git a/everglades_dryrun_workflow.sh b/everglades_dryrun_workflow.sh
@@ -1,25 +1,25 @@
 #!/bin/bash
-#SBATCH --job-name=everglades_workflow 
-#SBATCH [email protected] 
+#SBATCH --job-name=everwatch_workflow_dryrun
+#SBATCH [email protected]
 #SBATCH --mail-type=FAIL
 #SBATCH --gpus=a100:1
 #SBATCH --cpus-per-task=3
 #SBATCH --mem=200gb
 #SBATCH --time=01:30:00
 #SBATCH --partition=gpu
-#SBATCH --output=/blue/ewhite/everglades/EvergladesTools/logs/everglades_dryrun_workflow.out
-#SBATCH --error=/blue/ewhite/everglades/EvergladesTools/logs/everglades_dryrun_workflow.err
+#SBATCH --output=/blue/ewhite/everglades/everwatch-workflow/logs/everglades_dryrun_workflow.out
+#SBATCH --error=/blue/ewhite/everglades/everwatch-workflow/logs/everglades_dryrun_workflow.err
 
 echo "INFO: [$(date "+%Y-%m-%d %H:%M:%S")] Starting everglades workflow on $(hostname) in $(pwd)"
 
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] Loading required modules"
 source /etc/profile.d/modules.sh
 
 ml conda
-conda activate EvergladesTools
+conda activate everwatch
 export TEST_ENV=True
 
-cd /blue/ewhite/everglades/EvergladesTools/Zooniverse
+cd /blue/ewhite/everglades/everwatch-workflow/
 
 snakemake --unlock
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] Starting Snakemake pipeline"

diff --git a/everglades_workflow.sh b/everglades_workflow.sh
@@ -7,18 +7,18 @@
 #SBATCH --mem=600gb
 #SBATCH --time=80:00:00
 #SBATCH --partition=gpu
-#SBATCH --output=/blue/ewhite/everglades/EvergladesTools/logs/everglades_workflow.out
-#SBATCH --error=/blue/ewhite/everglades/EvergladesTools/logs/everglades_workflow.err
+#SBATCH --output=/blue/ewhite/everglades/everwatch-workflow/logs/everglades_workflow.out
+#SBATCH --error=/blue/ewhite/everglades/everwatch-workflow/logs/everglades_workflow.err
 
 echo "INFO: [$(date "+%Y-%m-%d %H:%M:%S")] Starting everglades workflow on $(hostname) in $(pwd)"
 
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] Loading required modules"
 source /etc/profile.d/modules.sh
 
 ml conda
-conda activate EvergladesTools
+conda activate everwatch
 
-cd /blue/ewhite/everglades/EvergladesTools/Zooniverse
+cd /blue/ewhite/everglades/everwatch-workflow/
 
 snakemake --unlock
 echo "INFO [$(date "+%Y-%m-%d %H:%M:%S")] Starting Snakemake pipeline"

diff --git a/nest_detection.sh b/nest_detection.sh
@@ -8,11 +8,11 @@ sbatch <<EOT
 #SBATCH --cpus-per-task=8
 #SBATCH --mem=62GB
 #SBATCH --time=72:00:00       #Time limit hrs:min:sec
-#SBATCH --output=/blue/ewhite/everglades/EvergladesTools/Zooniverse/logs/nest_detector_%j.out   # Standard output and error log
-#SBATCH --error=/blue/ewhite/everglades/EvergladesTools/Zooniverse/logs/nest_detector_%j.err
+#SBATCH --output=/blue/ewhite/everglades/everwatch-workflow/logs/nest_detector_%j.out   # Standard output and error log
+#SBATCH --error=/blue/ewhite/everglades/everwatch-workflow/logs/nest_detector_%j.err
 #SBATCH --partition=hpg-default
 ulimit -c 0
-cd /blue/ewhite/everglades/EvergladesTools/Zooniverse
-source activate EvergladesTools
+cd /blue/ewhite/everglades/everwatch-workflow
+source activate everwatch
 python nest_detection.py
 EOT
diff --git a/predict.sh b/predict.sh
@@ -8,12 +8,12 @@ sbatch <<EOT
 #SBATCH --cpus-per-task=8
 #SBATCH --mem=62GB
 #SBATCH --time=96:00:00       #Time limit hrs:min:sec
-#SBATCH --output=/blue/ewhite/everglades/EvergladesTools/Zooniverse/logs/bird_detector_%j.out   # Standard output and error log
-#SBATCH --error=/blue/ewhite/everglades/EvergladesTools/Zooniverse/logs/bird_detector_%j.err
+#SBATCH --output=/blue/ewhite/everglades/everwatch-workflow/logs/bird_detector_%j.out   # Standard output and error log
+#SBATCH --error=/blue/ewhite/everglades/everwatch-workflow/logs/bird_detector_%j.err
 #SBATCH --partition=gpu
 #SBATCH --gpus=1
 ulimit -c 0
-cd /blue/ewhite/everglades/EvergladesTools/Zooniverse
-source activate EvergladesTools
+cd /blue/ewhite/everglades/everwatch-workflow
+source activate everwatch
 python predict.py
 EOT