Skip to content

Commit

Permalink
removed unnecessary comments, removed nodes specified for old jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
@julietcohen
julietcohen committed May 9, 2023
1 parent 727c8c4 commit 03c715a
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 49 deletions.
12 changes: 4 additions & 8 deletions IN_PROGRESS_VIZ_WORKFLOW.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import json

import logging
#import logging.config # removed this because trying to get logger to work with special config to log.log rather than Kastan's file
#import logging.config
import logging.handlers

import os
Expand All @@ -22,16 +22,13 @@
import shlex

import pdgraster
import pdgstaging # For staging
import pdgstaging
import ray
import viz_3dtiles # import Cesium3DTile, Cesium3DTileset

# define user on Delta, avoid writing files to other user's dir
user = subprocess.check_output("whoami").strip().decode("ascii")

# help flag provides flag help
# store_true actions stores argument as True

#import lake_change_config
#IWP_CONFIG = lake_change_config.IWP_CONFIG

Expand Down Expand Up @@ -97,7 +94,7 @@ def main():

# (optionally) Comment out steps you don't need 😁
# todo: sync footprints to nodes.
# step0_staging()
step0_staging()
# todo: rsync staging to /scratch
# todo: merge staged files in /scratch # ./merge_staged_vector_tiles.py
# DO NOT RUN 3d-tiling UNTIL WORKFLOW CAN ACCOMODATE FILE HIERARCHY:step1_3d_tiles() # default parameter batch_size = 300
Expand All @@ -107,7 +104,7 @@ def main():
# step3_raster_lower(batch_size_geotiffs=100) # rasterize all LOWER Z levels
# todo: immediately after initiating above step, start rsync script to continuously sync geotiff files,
# or immediately after the above step is done, rsync all files at once if there is time left in job
step4_webtiles(batch_size_web_tiles=250) # convert to web tiles.
# step4_webtiles(batch_size_web_tiles=250) # convert to web tiles.

# mem_testing = False
# if mem_testing:
Expand Down Expand Up @@ -674,7 +671,6 @@ def step4_webtiles(batch_size_web_tiles=300):

for i in range(0, len(app_futures)):
ready, not_ready = ray.wait(app_futures)
print(f"✅ Finished {ray.get(ready)}")
print(f"📌 Completed {i+1} of {len(geotiff_batches)}")
print(f"⏰ Running total of elapsed time: {(time.time() - start)/60:.2f} minutes\n")
app_futures = not_ready
Expand Down
16 changes: 7 additions & 9 deletions PRODUCTION_IWP_CONFIG.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@

# define user on Delta, avoid writing files to other user's dir
user = subprocess.check_output("whoami").strip().decode("ascii")
head_node = 'cn005'
#head_node = 'gpub___'
#head_node = 'cn___/'
head_node = 'gpub___/'
# define desired location for output files within user dir
# ensures a new subfolder every run as long as new run is not started within same day as last run
# following path is the output subdir for test run, using just on subdir of the alaska files that is only ~8% of the Alaska dir, 23.5 GB
output_subdir = 'IWP/output/iwp_testRun_20230502'
output_subdir = 'IWP/output/iwp_testRun_2023____'
#output_subdir = datetime.now().strftime("%b-%d-%y")
# don't use subprocess to retrieve date for subdir because runs might span over 2 days if they go overnight

Expand All @@ -22,8 +22,7 @@

# input path for all data:
#INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/' # The output data of MAPLE. Which is the input data for STAGING.
#INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/russia/226_227_iwp/'
INPUT = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/iwp/'
INPUT = '/scratch/bbou/julietcohen/IWP/input/2023-01-19/iwp_files/high/russia/226_227_iwp/'

# following path is the OUTPUT for test run, using just on subdir of the alaska files that is only 7.78% of the Alaska dir, 45.57 GB
OUTPUT = f'/scratch/bbou/{user}/{output_subdir}/' # Dir for results. High I/O is good.
Expand All @@ -33,8 +32,7 @@
# footprints paths for all data:
FOOTPRINTS_LOCAL = '/tmp/staged_footprints/'
#FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/'
#FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/russia/226_227_iwp/'
FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/input/few_adjacent_russia/footprints/'
FOOTPRINTS_REMOTE = '/scratch/bbou/julietcohen/IWP/footprint_files_with_date_20230119/high/russia/226_227_iwp/'

STAGING_LOCAL = '/tmp/staged/'
STAGING_REMOTE = OUTPUT + 'staged/'
Expand Down Expand Up @@ -89,8 +87,8 @@
1
],
"palette": [
"#66339952",
"#ffcc00"
"#f8ff1f1A", # 10% alpha yellow
"#f8ff1f" # solid yellow
],
"nodata_val": 0,
"nodata_color": "#ffffff00"
Expand Down
47 changes: 16 additions & 31 deletions merge_staged_vector_tiles.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,6 @@
assert ray.is_initialized() == True
print("🎯 Ray initialized.")

# todo: see if I can delete all of these constants....
# These don't matter much in this workflow.
# ALWAYS include the tailing slash "/"
# BASE_DIR_OF_INPUT = '/tmp/'
# FOOTPRINTS_PATH = BASE_DIR_OF_INPUT + 'staged_footprints/'

# OUTPUT = '/tmp/' # Dir for results. High I/O is good.
# Output dirs for each sub-step:
# OUTPUT_OF_STAGING = OUTPUT + 'staged/'
# GEOTIFF_PATH = OUTPUT + 'geotiff/'
# WEBTILE_PATH = OUTPUT + 'web_tiles/'
# THREE_D_PATH = OUTPUT + '3d_tiles/'
# IWP_CONFIG = {"dir_input": BASE_DIR_OF_INPUT,"ext_input": ".shp", "dir_footprints": FOOTPRINTS_PATH,"dir_geotiff": GEOTIFF_PATH,"dir_web_tiles": WEBTILE_PATH,"dir_staged": OUTPUT_OF_STAGING,"filename_staging_summary": OUTPUT_OF_STAGING + "staging_summary.csv","filename_rasterization_events": GEOTIFF_PATH + "raster_events.csv","filename_rasters_summary": GEOTIFF_PATH + "raster_summary.csv","version": datetime.now().strftime("%B%d,%Y"),"simplify_tolerance": 0.1,"tms_id": "WorldCRS84Quad","z_range": [0, 16],"geometricError": 57,"z_coord": 0,"statistics": [ { "name": "iwp_count", "weight_by": "count", "property": "centroids_per_pixel", "aggregation_method": "sum", "resampling_method": "sum", "val_range": [0, None], "palette": ["#66339952", "#d93fce", "#ffcc00"], "nodata_val": 0, "nodata_color": "#ffffff00" }, { "name": "iwp_coverage", "weight_by": "area", "property": "area_per_pixel_area", "aggregation_method": "sum", "resampling_method": "average", "val_range": [0, 1], "palette": ["#66339952", "#ffcc00"], "nodata_val": 0, "nodata_color": "#ffffff00" },],"deduplicate_at": ["raster", "3dtiles"],"deduplicate_keep_rules": [["Date", "larger"]],"deduplicate_method": "footprints",}

# Juliet's edit: commented out Kastan's manual config settings to instead import config for workflow
import PRODUCTION_IWP_CONFIG
IWP_CONFIG = PRODUCTION_IWP_CONFIG.IWP_CONFIG
# update the config for the current context
Expand All @@ -84,23 +69,23 @@ def main():
#######################
# todo -- get files from dirs automatically, using os.lsdir().
#BASE_DIR = '/scratch/bbou/julietcohen/IWP/output/...'
merged_dir_path = f"{IWP_CONFIG['dir_staged']}gpub020" # this path SHOULD NOT be in the `staged_dir_paths_list`
merged_dir_path = f"{IWP_CONFIG['dir_staged']}cn___" # this path SHOULD NOT be in the `staged_dir_paths_list`
staged_dir_paths_list = [
f"{IWP_CONFIG['dir_staged']}gpub030",
f"{IWP_CONFIG['dir_staged']}gpub031",
# f"{IWP_CONFIG['dir_staged']}cn071",
# f"{IWP_CONFIG['dir_staged']}cn072",
# f"{IWP_CONFIG['dir_staged']}cn073",
# f"{IWP_CONFIG['dir_staged']}cn074",
# f"{IWP_CONFIG['dir_staged']}cn075",
# f"{IWP_CONFIG['dir_staged']}cn076",
# f"{IWP_CONFIG['dir_staged']}cn077",
# f"{IWP_CONFIG['dir_staged']}cn078",
# f"{IWP_CONFIG['dir_staged']}gpub094",
# f"{IWP_CONFIG['dir_staged']}gpub095",
# f"{IWP_CONFIG['dir_staged']}gpub096",
# f"{IWP_CONFIG['dir_staged']}gpub097",
# f"{IWP_CONFIG['dir_staged']}gpub098",
f"{IWP_CONFIG['dir_staged']}cn___",
f"{IWP_CONFIG['dir_staged']}cn___",
f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
# f"{IWP_CONFIG['dir_staged']}cn___",
]
##############################
#### END OF Change me 😁 ####
Expand Down
2 changes: 1 addition & 1 deletion slurm/BEST_cpu_ray_double_srun.slurm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
#SBATCH --job-name=pdg_viz
#SBATCH --partition=cpu
#SBATCH --account=
#SBATCH --time=04:00:00
#SBATCH --time=01:00:00

#SBATCH --export=ALL,RAY_worker_register_timeout_seconds=120

Expand Down

0 comments on commit 03c715a

Please sign in to comment.