From 0c3d8bbde011ea1fd45b77b545b8152c03932f9e Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:01:42 -0700 Subject: [PATCH 01/19] invert quartic fit --- create_quartic_fit_coeffs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_quartic_fit_coeffs.py b/create_quartic_fit_coeffs.py index 55024ba..6c40454 100644 --- a/create_quartic_fit_coeffs.py +++ b/create_quartic_fit_coeffs.py @@ -18,8 +18,8 @@ wfi_quartic = create_constant_quartic_coefficients((2048, 2048)) nfi_quartic = create_constant_quartic_coefficients((2048, 2048)) -wfi_quartic[-2, :, :] /= wfi_vignette -nfi_quartic[-2, :, :] /= nfi_vignette +wfi_quartic[-2, :, :] = 1/wfi_vignette +nfi_quartic[-2, :, :] = 1/nfi_vignette meta = NormalizedMetadata.load_template("FQ1", "1") meta['DATE-OBS'] = str(datetime.now()) From 7f747efa4f7e476d52d5f42a3702e318e189d6e6 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:40:05 -0700 Subject: [PATCH 02/19] fix quartic fit --- create_quartic_fit_coeffs.py | 23 +++++++++++++++++++++++ simpunch/level0.py | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/create_quartic_fit_coeffs.py b/create_quartic_fit_coeffs.py index 6c40454..26c985b 100644 --- a/create_quartic_fit_coeffs.py +++ b/create_quartic_fit_coeffs.py @@ -9,6 +9,29 @@ from punchbowl.data.io import load_ndcube_from_fits from punchbowl.level1.quartic_fit import create_constant_quartic_coefficients +# backward +wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits" +nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits" + +wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...] + 1E-8 +nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...] + 1E-8 + +wfi_quartic = create_constant_quartic_coefficients((2048, 2048)) +nfi_quartic = create_constant_quartic_coefficients((2048, 2048)) + +wfi_quartic[-2, :, :] = wfi_vignette +nfi_quartic[-2, :, :] = nfi_vignette + +meta = NormalizedMetadata.load_template("FQ1", "1") +meta['DATE-OBS'] = str(datetime.now()) + +wfi_cube = NDCube(data=wfi_quartic, meta=meta, wcs=WCS(naxis=3)) +nfi_cube = NDCube(data=nfi_quartic, meta=meta, wcs=WCS(naxis=3)) + +write_ndcube_to_fits(wfi_cube, "wfi_quartic_backward_coeffs.fits") +write_ndcube_to_fits(nfi_cube, "nfi_quartic_backward_coeffs.fits") + +# forward wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits" nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits" diff --git a/simpunch/level0.py b/simpunch/level0.py index f1ad9db..0217be2 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -26,8 +26,8 @@ def perform_photometric_uncalibration(input_data: NDCube, coefficient_array: np.ndarray) -> NDCube: """Undo quartic fit calibration.""" num_coefficients = coefficient_array.shape[0] - new_data = np.sum( - [coefficient_array[i, ...] / np.power(input_data.data, num_coefficients - i - 1) + new_data = np.nansum( + [coefficient_array[i, ...] * np.power(input_data.data, num_coefficients - i - 1) for i in range(num_coefficients)], axis=0) input_data.data[...] = new_data[...] return input_data From baf3ba1e95cd2fcb0f9080748c93c5f0fa7eb8fd Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:44:02 -0700 Subject: [PATCH 03/19] allow backward coeffs --- simpunch/flow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 41acc88..2521431 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -17,6 +17,8 @@ def generate_flow(gamera_directory: str, output_directory: str, psf_model_path: str, + wfi_quartic_backward_model_path: str, + nfi_quartic_backward_model_path: str, wfi_quartic_model_path: str, nfi_quartic_model_path: str, num_repeats: int = 1, @@ -35,8 +37,8 @@ def generate_flow(gamera_directory: str, generate_l1_all(gamera_directory) generate_l0_all(gamera_directory, psf_model_path, - wfi_quartic_model_path, - nfi_quartic_model_path, + wfi_quartic_backward_model_path, + nfi_quartic_backward_model_path, transient_probability=transient_probability) # duplicate the psf model to all required versions From a1024652a45d44d8ee4267ac271102acf0d2e81d Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:53:10 -0700 Subject: [PATCH 04/19] reduce psf model --- create_synthetic_psf.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/create_synthetic_psf.py b/create_synthetic_psf.py index 1974fb8..860e3cb 100644 --- a/create_synthetic_psf.py +++ b/create_synthetic_psf.py @@ -2,6 +2,7 @@ from pathlib import Path import numpy as np +from matplotlib import pyplot as plt from regularizepsf import (ArrayPSFTransform, simple_functional_psf, varied_functional_psf) from regularizepsf.util import calculate_covering @@ -53,7 +54,7 @@ def target_psf(row, @varied_functional_psf(target_psf) def synthetic_psf(row, col): return {"tail_angle": -np.arctan2(row - img_size//2, col - img_size//2), - "tail_separation": np.sqrt((row - img_size//2) ** 2 + (col - img_size//2) ** 2)/500 * 2 + 1E-3, + "tail_separation": np.sqrt((row - img_size//2) ** 2 + (col - img_size//2) ** 2)/1200 * 2 + 1E-3, "core_sigma_x": initial_sigma, "core_sigma_y": initial_sigma} @@ -61,6 +62,9 @@ def synthetic_psf(row, col): initial = baked_in_initial_psf.as_array_psf(coords, psf_size) synthetic = synthetic_psf.as_array_psf(coords, psf_size) +synthetic.visualize_psfs() +plt.show() + backward_corrector = ArrayPSFTransform.construct(initial, synthetic, alpha=3.0, epsilon=0.2) backward_corrector.save(Path("synthetic_backward_psf.fits")) From 04a050df993ddbef15ab555cfbdb55b2204dd4e9 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:54:24 -0700 Subject: [PATCH 05/19] remove visualize --- create_synthetic_psf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/create_synthetic_psf.py b/create_synthetic_psf.py index 860e3cb..6d34bd6 100644 --- a/create_synthetic_psf.py +++ b/create_synthetic_psf.py @@ -62,9 +62,6 @@ def synthetic_psf(row, col): initial = baked_in_initial_psf.as_array_psf(coords, psf_size) synthetic = synthetic_psf.as_array_psf(coords, psf_size) -synthetic.visualize_psfs() -plt.show() - backward_corrector = ArrayPSFTransform.construct(initial, synthetic, alpha=3.0, epsilon=0.2) backward_corrector.save(Path("synthetic_backward_psf.fits")) From 4502882cf038b224e509fcbac33beb17439335ca Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 21:57:32 -0700 Subject: [PATCH 06/19] use both forward and backward psf --- simpunch/flow.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 2521431..e3d6683 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -16,7 +16,8 @@ @flow(log_prints=True) def generate_flow(gamera_directory: str, output_directory: str, - psf_model_path: str, + forward_psf_model_path: str, + backward_psf_model_path: str, wfi_quartic_backward_model_path: str, nfi_quartic_backward_model_path: str, wfi_quartic_model_path: str, @@ -36,7 +37,7 @@ def generate_flow(gamera_directory: str, generate_l2_all(gamera_directory) generate_l1_all(gamera_directory) generate_l0_all(gamera_directory, - psf_model_path, + backward_psf_model_path, wfi_quartic_backward_model_path, nfi_quartic_backward_model_path, transient_probability=transient_probability) @@ -45,7 +46,7 @@ def generate_flow(gamera_directory: str, for type_code in ["RM", "RZ", "RP", "RC"]: for obs_code in ["1", "2", "3", "4"]: new_name = f"PUNCH_L1_{type_code}{obs_code}_{time_str}_v1.fits" - shutil.copy(psf_model_path, os.path.join(gamera_directory, f"synthetic_l0/{new_name}")) + shutil.copy(forward_psf_model_path, os.path.join(gamera_directory, f"synthetic_l0/{new_name}")) # duplicate the quartic model type_code = "FQ" From bb6067e6161404e204f6d80820498ffbed6322c9 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 22:10:58 -0700 Subject: [PATCH 07/19] make sure models are older than data --- simpunch/flow.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index e3d6683..1eff8b7 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -29,8 +29,7 @@ def generate_flow(gamera_directory: str, update_database: bool = True) -> None: """Generate all the products in the reverse pipeline.""" if start_time is None: - start_time = datetime.now() - timedelta(days=3) # noqa: DTZ005 - time_str = start_time.strftime("%Y%m%d%H%M%S") + start_time = datetime.now() # noqa: DTZ005 if generate_new: generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) @@ -42,19 +41,22 @@ def generate_flow(gamera_directory: str, nfi_quartic_backward_model_path, transient_probability=transient_probability) + model_time = start_time - timedelta(days=3) + model_time_str = model_time.strftime("%Y%m%d%H%M%S") + # duplicate the psf model to all required versions for type_code in ["RM", "RZ", "RP", "RC"]: for obs_code in ["1", "2", "3", "4"]: - new_name = f"PUNCH_L1_{type_code}{obs_code}_{time_str}_v1.fits" + new_name = f"PUNCH_L1_{type_code}{obs_code}_{model_time_str}_v1.fits" shutil.copy(forward_psf_model_path, os.path.join(gamera_directory, f"synthetic_l0/{new_name}")) # duplicate the quartic model type_code = "FQ" for obs_code in ["1", "2", "3"]: - new_name = f"PUNCH_L1_{type_code}{obs_code}_{time_str}_v1.fits" + new_name = f"PUNCH_L1_{type_code}{obs_code}_{model_time_str}_v1.fits" shutil.copy(wfi_quartic_model_path, os.path.join(gamera_directory, f"synthetic_l0/{new_name}")) obs_code = "4" - new_name = f"PUNCH_L1_{type_code}{obs_code}_{time_str}_v1.fits" + new_name = f"PUNCH_L1_{type_code}{obs_code}_{model_time_str}_v1.fits" shutil.copy(nfi_quartic_model_path, os.path.join(gamera_directory, f"synthetic_l0/{new_name}")) if update_database: From a21531cb7594ef32ae7c6d84acfcc414626b2145 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 22:16:41 -0700 Subject: [PATCH 08/19] change model age --- simpunch/flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 1eff8b7..4907d9f 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -41,7 +41,7 @@ def generate_flow(gamera_directory: str, nfi_quartic_backward_model_path, transient_probability=transient_probability) - model_time = start_time - timedelta(days=3) + model_time = start_time - timedelta(days=5) model_time_str = model_time.strftime("%Y%m%d%H%M%S") # duplicate the psf model to all required versions From c6a3d9a5d8261b20519c25953c2fe4c4988ba41e Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 22:29:15 -0700 Subject: [PATCH 09/19] make quartic fit not have large values --- create_quartic_fit_coeffs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/create_quartic_fit_coeffs.py b/create_quartic_fit_coeffs.py index 26c985b..55a2c9f 100644 --- a/create_quartic_fit_coeffs.py +++ b/create_quartic_fit_coeffs.py @@ -1,6 +1,7 @@ # ruff: noqa from datetime import datetime +import numpy as np from astropy.io.fits import CompImageHDU, HDUList, ImageHDU, PrimaryHDU from astropy.wcs import WCS from astropy.wcs.docstrings import naxis @@ -35,14 +36,18 @@ wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits" nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits" -wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...] + 1E-8 -nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...] + 1E-8 +wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...] +nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...] wfi_quartic = create_constant_quartic_coefficients((2048, 2048)) nfi_quartic = create_constant_quartic_coefficients((2048, 2048)) wfi_quartic[-2, :, :] = 1/wfi_vignette nfi_quartic[-2, :, :] = 1/nfi_vignette +wfi_quartic[np.isinf(wfi_quartic)] = 0 +wfi_quartic[np.isnan(wfi_quartic)] = 0 +nfi_quartic[np.isinf(nfi_quartic)] = 0 +nfi_quartic[np.isnan(nfi_quartic)] = 0 meta = NormalizedMetadata.load_template("FQ1", "1") meta['DATE-OBS'] = str(datetime.now()) From 7196070b0485d017eca12d557711cdccc8937527 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 22:34:28 -0700 Subject: [PATCH 10/19] remove offset in quartic coeffs --- create_quartic_fit_coeffs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_quartic_fit_coeffs.py b/create_quartic_fit_coeffs.py index 55a2c9f..f4611b8 100644 --- a/create_quartic_fit_coeffs.py +++ b/create_quartic_fit_coeffs.py @@ -14,8 +14,8 @@ wfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM1_20240817174727_v2.fits" nfi_vignetting_model_path = "./build_3_review_files/PUNCH_L1_GM4_20240819045110_v1.fits" -wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...] + 1E-8 -nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...] + 1E-8 +wfi_vignette = load_ndcube_from_fits(wfi_vignetting_model_path).data[...] +nfi_vignette = load_ndcube_from_fits(nfi_vignetting_model_path).data[...] wfi_quartic = create_constant_quartic_coefficients((2048, 2048)) nfi_quartic = create_constant_quartic_coefficients((2048, 2048)) From 7d62d6f92f54f1119f27d8de3f77a426a0593b0f Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Tue, 12 Nov 2024 23:31:42 -0700 Subject: [PATCH 11/19] turn off distortion for now --- simpunch/level1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/simpunch/level1.py b/simpunch/level1.py index 11a5d4a..776aafb 100644 --- a/simpunch/level1.py +++ b/simpunch/level1.py @@ -281,9 +281,9 @@ def generate_l1_pmzp(input_file: str, path_output: str, rotation_stage: int, spa output_pdata.meta["POLAR"] = 60 # Add distortion - output_mdata = add_distortion(output_mdata) - output_zdata = add_distortion(output_zdata) - output_pdata = add_distortion(output_pdata) + # output_mdata = add_distortion(output_mdata) # noqa: ERA001 + # output_zdata = add_distortion(output_zdata) # noqa: ERA001 + # output_pdata = add_distortion(output_pdata) # noqa: ERA001 output_mdata = add_starfield_clear(output_mdata) output_zdata = add_starfield_clear(output_zdata) @@ -322,7 +322,7 @@ def generate_l1_cr(input_file: str, path_output: str, rotation_stage: int, space # Quality marking output_data = mark_quality(output_data) - output_data = add_distortion(output_data) + # output_data = add_distortion(output_data) # noqa: ERA001 output_data = add_starfield_clear(output_data) From 2d74ba9ef699b26cbcdf8c1d0b6ce51fa4281f5f Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Wed, 13 Nov 2024 03:17:25 -0700 Subject: [PATCH 12/19] add pointing toggle --- simpunch/flow.py | 2 ++ simpunch/level0.py | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 4907d9f..d29381f 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -25,6 +25,7 @@ def generate_flow(gamera_directory: str, num_repeats: int = 1, start_time: datetime | None = None, transient_probability: float = 0.03, + shift_pointing: bool = False, generate_new: bool = True, update_database: bool = True) -> None: """Generate all the products in the reverse pipeline.""" @@ -39,6 +40,7 @@ def generate_flow(gamera_directory: str, backward_psf_model_path, wfi_quartic_backward_model_path, nfi_quartic_backward_model_path, + shift_pointing=shift_pointing, transient_probability=transient_probability) model_time = start_time - timedelta(days=5) diff --git a/simpunch/level0.py b/simpunch/level0.py index 0217be2..360a07d 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -114,7 +114,8 @@ def generate_l0_pmzp(input_file: NDCube, psf_model: ArrayPSFTransform, wfi_quartic_coefficients: np.ndarray, nfi_quartic_coefficients: np.ndarray, - transient_probability: float=0.03) -> None: + transient_probability: float=0.03, + shift_pointing: bool=False) -> None: """Generate level 0 polarized synthetic data.""" input_data = load_ndcube_from_fits(input_file) @@ -134,7 +135,11 @@ def generate_l0_pmzp(input_file: NDCube, output_meta[key] = input_data.meta[key].value input_data = NDCube(data=input_data.data, meta=output_meta, wcs=input_data.wcs) - output_data, original_wcs = starfield_misalignment(input_data) + if shift_pointing: + output_data, original_wcs = starfield_misalignment(input_data) + else: + output_data = input_data + original_wcs = input_data.wcs.copy() output_data, transient = add_transients(output_data, transient_probability=transient_probability) output_data = uncorrect_psf(output_data, psf_model) @@ -189,7 +194,8 @@ def generate_l0_cr(input_file: NDCube, path_output: str, psf_model: ArrayPSFTransform, wfi_quartic_coefficients: np.ndarray, nfi_quartic_coefficients: np.ndarray, - transient_probability: float = 0.03) -> None: + transient_probability: float = 0.03, + shift_pointing: bool=False) -> None: """Generate level 0 clear synthetic data.""" input_data = load_ndcube_from_fits(input_file) @@ -209,7 +215,11 @@ def generate_l0_cr(input_file: NDCube, path_output: str, output_meta[key] = input_data.meta[key].value input_data = NDCube(data=input_data.data, meta=output_meta, wcs=input_data.wcs) - output_data, original_wcs = starfield_misalignment(input_data) + if shift_pointing: + output_data, original_wcs = starfield_misalignment(input_data) + else: + output_data = input_data + original_wcs = input_data.wcs.copy() output_data, transient = add_transients(output_data, transient_probability=transient_probability) output_data = uncorrect_psf(output_data, psf_model) output_data = add_stray_light(output_data) @@ -262,7 +272,8 @@ def generate_l0_cr(input_file: NDCube, path_output: str, )) def generate_l0_all(datadir: str, psf_model_path: str, wfi_quartic_coeffs_path: str, nfi_quartic_coeffs_path: str, - transient_probability: float = 0.03) -> None: + transient_probability: float = 0.03, + shift_pointing: bool = False) -> None: """Generate all level 0 synthetic data.""" print(f"Running from {datadir}") outdir = os.path.join(datadir, "synthetic_l0/") @@ -282,9 +293,9 @@ def generate_l0_all(datadir: str, psf_model_path: str, futures = [] for file_l1 in tqdm(files_l1, total=len(files_l1)): futures.append(generate_l0_pmzp.submit(file_l1, outdir, psf_model, # noqa: PERF401 - wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability)) + wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) for file_cr in tqdm(files_cr, total=len(files_cr)): futures.append(generate_l0_cr.submit(file_cr, outdir, psf_model, # noqa: PERF401 - wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability)) + wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) wait(futures) From 177301d87bd2f4aa531d17143da4ad8d7eddc2be Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Fri, 15 Nov 2024 00:49:34 -0600 Subject: [PATCH 13/19] server improvements --- simpunch/cli.py | 13 ++++++++----- simpunch/flow.py | 6 +++--- simpunch/level0.py | 45 +++++++++++++++++++++++++++------------------ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/simpunch/cli.py b/simpunch/cli.py index 3b829fe..b98d0cc 100644 --- a/simpunch/cli.py +++ b/simpunch/cli.py @@ -4,21 +4,21 @@ import toml from prefect import serve -from .flow import generate_flow +from simpunch.flow import generate_flow -@click.group() +#@click.group() def main(): """Simulate PUNCH data with simpunch.""" -@main.command() -@click.argument("configuration_path", type=click.Path(exists=True)) +#@main.command() +#@click.argument("configuration_path", type=click.Path(exists=True)) def generate(configuration_path): """Run a single instance of the pipeline.""" configuration = load_configuration(configuration_path) generate_flow(**configuration) -@main.command() +#@main.command() def automate(): """Automate the data generation using Prefect.""" serve(generate_flow.to_deployment(name="simulator-deployment", @@ -28,3 +28,6 @@ def automate(): def load_configuration(configuration_path: str) -> dict: """Load a configuration file.""" return toml.load(configuration_path) + +if __name__ == "__main__": + generate("/home/marcus.hughes/build4/punch190_simpunch_config.toml") diff --git a/simpunch/flow.py b/simpunch/flow.py index d29381f..cb48bcf 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -33,9 +33,9 @@ def generate_flow(gamera_directory: str, start_time = datetime.now() # noqa: DTZ005 if generate_new: - generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) - generate_l2_all(gamera_directory) - generate_l1_all(gamera_directory) + #generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) + #generate_l2_all(gamera_directory) + #generate_l1_all(gamera_directory) generate_l0_all(gamera_directory, backward_psf_model_path, wfi_quartic_backward_model_path, diff --git a/simpunch/level0.py b/simpunch/level0.py index 360a07d..da8cf6a 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -7,7 +7,7 @@ import astropy.units as u import numpy as np from ndcube import NDCube -from prefect import flow, task +from prefect import flow, task, unmapped from prefect.futures import wait from prefect_dask import DaskTaskRunner from punchbowl.data import (NormalizedMetadata, get_base_file_name, @@ -111,13 +111,16 @@ def starfield_misalignment(input_data: NDCube, @task def generate_l0_pmzp(input_file: NDCube, path_output: str, - psf_model: ArrayPSFTransform, - wfi_quartic_coefficients: np.ndarray, - nfi_quartic_coefficients: np.ndarray, + psf_model_path: str, # ArrayPSFTransform, + wfi_quartic_coeffs_path: str, # np.ndarray, + nfi_quartic_coeffs_path: str, # np.ndarray, transient_probability: float=0.03, shift_pointing: bool=False) -> None: """Generate level 0 polarized synthetic data.""" input_data = load_ndcube_from_fits(input_file) + psf_model = ArrayPSFTransform.load(Path(psf_model_path)) + wfi_quartic_coefficients = load_ndcube_from_fits(wfi_quartic_coeffs_path).data + nfi_quartic_coefficients = load_ndcube_from_fits(nfi_quartic_coeffs_path).data # Define the output data product product_code = input_data.meta["TYPECODE"].value + input_data.meta["OBSCODE"].value @@ -191,13 +194,16 @@ def generate_l0_pmzp(input_file: NDCube, @task def generate_l0_cr(input_file: NDCube, path_output: str, - psf_model: ArrayPSFTransform, - wfi_quartic_coefficients: np.ndarray, - nfi_quartic_coefficients: np.ndarray, + psf_model_path: str, # ArrayPSFTransform, + wfi_quartic_coeffs_path: str, # np.ndarray, + nfi_quartic_coeffs_path: str, # np.ndarray, transient_probability: float = 0.03, shift_pointing: bool=False) -> None: """Generate level 0 clear synthetic data.""" input_data = load_ndcube_from_fits(input_file) + psf_model = ArrayPSFTransform.load(Path(psf_model_path)) + wfi_quartic_coefficients = load_ndcube_from_fits(wfi_quartic_coeffs_path).data + nfi_quartic_coefficients = load_ndcube_from_fits(nfi_quartic_coeffs_path).data # Define the output data product product_code = input_data.meta["TYPECODE"].value + input_data.meta["OBSCODE"].value @@ -268,7 +274,7 @@ def generate_l0_cr(input_file: NDCube, path_output: str, original_wcs.to_header().tofile(path_output + get_base_file_name(output_data) + "_original_wcs.txt") @flow(log_prints=True, - task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 4, "threads_per_worker": 2}, + task_runner=DaskTaskRunner(cluster_kwargs={"n_workers": 32, "threads_per_worker": 2}, )) def generate_l0_all(datadir: str, psf_model_path: str, wfi_quartic_coeffs_path: str, nfi_quartic_coeffs_path: str, @@ -286,16 +292,19 @@ def generate_l0_all(datadir: str, psf_model_path: str, print(f"Generating based on {len(files_l1)} files.") files_l1.sort() - psf_model = ArrayPSFTransform.load(Path(psf_model_path)) - wfi_quartic_coeffs = load_ndcube_from_fits(wfi_quartic_coeffs_path).data - nfi_quartic_coeffs = load_ndcube_from_fits(nfi_quartic_coeffs_path).data + #psf_model = ArrayPSFTransform.load(Path(psf_model_path)) + #wfi_quartic_coeffs = load_ndcube_from_fits(wfi_quartic_coeffs_path).data + #nfi_quartic_coeffs = load_ndcube_from_fits(nfi_quartic_coeffs_path).data + + futures = generate_l0_pmzp.map(files_l1, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing) + wait(futures) + #for file_l1 in tqdm(files_l1, total=len(files_l1)): + # futures.append(generate_l0_pmzp.submit(file_l1, outdir, psf_model, # noqa: PERF401 + # wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) - futures = [] - for file_l1 in tqdm(files_l1, total=len(files_l1)): - futures.append(generate_l0_pmzp.submit(file_l1, outdir, psf_model, # noqa: PERF401 - wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) - for file_cr in tqdm(files_cr, total=len(files_cr)): - futures.append(generate_l0_cr.submit(file_cr, outdir, psf_model, # noqa: PERF401 - wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) + #for file_cr in tqdm(files_cr, total=len(files_cr)): + # futures.append(generate_l0_cr.submit(file_cr, outdir, psf_model, # noqa: PERF401 + # wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) + futures = generate_l0_cr.map(files_cr, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing) wait(futures) From 460bb06a91a39f66dbd1d7366bd8db7a1fa84aff Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Fri, 15 Nov 2024 00:11:00 -0700 Subject: [PATCH 14/19] add fixed month generation --- simpunch/cli.py | 2 +- simpunch/flow.py | 22 +++++++++++++++++----- simpunch/level0.py | 17 ++++------------- simpunch/level1.py | 3 +-- simpunch/level2.py | 12 +++--------- simpunch/level3.py | 23 ++++++++++++++++++++++- 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/simpunch/cli.py b/simpunch/cli.py index b98d0cc..b1cf4f6 100644 --- a/simpunch/cli.py +++ b/simpunch/cli.py @@ -1,6 +1,5 @@ """Command line interface.""" -import click import toml from prefect import serve @@ -29,5 +28,6 @@ def load_configuration(configuration_path: str) -> dict: """Load a configuration file.""" return toml.load(configuration_path) + if __name__ == "__main__": generate("/home/marcus.hughes/build4/punch190_simpunch_config.toml") diff --git a/simpunch/flow.py b/simpunch/flow.py index cb48bcf..8110bc4 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -4,13 +4,14 @@ import shutil from datetime import datetime +import numpy as np from asyncpg.pgproto.pgproto import timedelta from prefect import flow from simpunch.level0 import generate_l0_all from simpunch.level1 import generate_l1_all from simpunch.level2 import generate_l2_all -from simpunch.level3 import generate_l3_all +from simpunch.level3 import generate_l3_all_fixed @flow(log_prints=True) @@ -31,11 +32,22 @@ def generate_flow(gamera_directory: str, """Generate all the products in the reverse pipeline.""" if start_time is None: start_time = datetime.now() # noqa: DTZ005 + start_time = datetime(2024, 11, 13, 15, 20, 23) if generate_new: - #generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) - #generate_l2_all(gamera_directory) - #generate_l1_all(gamera_directory) + time_delta = timedelta(days=1) + files_tb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_TB.fits")) + files_pb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_PB.fits")) + + previous_month = np.arange(1, -30, -1) * time_delta + start_time + generate_l3_all_fixed(gamera_directory, previous_month, files_pb[0], files_tb[0]) + + next_month = np.arange(1, 30) * time_delta + start_time + generate_l3_all_fixed(gamera_directory, next_month, files_pb[-1], files_tb[-1]) + + # generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) + generate_l2_all(gamera_directory) + generate_l1_all(gamera_directory) generate_l0_all(gamera_directory, backward_psf_model_path, wfi_quartic_backward_model_path, @@ -43,7 +55,7 @@ def generate_flow(gamera_directory: str, shift_pointing=shift_pointing, transient_probability=transient_probability) - model_time = start_time - timedelta(days=5) + model_time = start_time - timedelta(days=35) model_time_str = model_time.strftime("%Y%m%d%H%M%S") # duplicate the psf model to all required versions diff --git a/simpunch/level0.py b/simpunch/level0.py index da8cf6a..4f69daa 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -7,7 +7,7 @@ import astropy.units as u import numpy as np from ndcube import NDCube -from prefect import flow, task, unmapped +from prefect import flow, task from prefect.futures import wait from prefect_dask import DaskTaskRunner from punchbowl.data import (NormalizedMetadata, get_base_file_name, @@ -17,7 +17,6 @@ from punchbowl.level1.initial_uncertainty import compute_noise from punchbowl.level1.sqrt import encode_sqrt from regularizepsf import ArrayPSFTransform -from tqdm import tqdm from simpunch.spike import generate_spike_image from simpunch.util import update_spacecraft_location, write_array_to_fits @@ -296,15 +295,7 @@ def generate_l0_all(datadir: str, psf_model_path: str, #wfi_quartic_coeffs = load_ndcube_from_fits(wfi_quartic_coeffs_path).data #nfi_quartic_coeffs = load_ndcube_from_fits(nfi_quartic_coeffs_path).data - futures = generate_l0_pmzp.map(files_l1, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing) - wait(futures) - #for file_l1 in tqdm(files_l1, total=len(files_l1)): - # futures.append(generate_l0_pmzp.submit(file_l1, outdir, psf_model, # noqa: PERF401 - # wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) - - - #for file_cr in tqdm(files_cr, total=len(files_cr)): - # futures.append(generate_l0_cr.submit(file_cr, outdir, psf_model, # noqa: PERF401 - # wfi_quartic_coeffs, nfi_quartic_coeffs, transient_probability, shift_pointing)) - futures = generate_l0_cr.map(files_cr, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing) + futures = [] + futures.extend(generate_l0_pmzp.map(files_l1, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing)) + futures.extend(generate_l0_cr.map(files_cr, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing)) wait(futures) diff --git a/simpunch/level1.py b/simpunch/level1.py index 776aafb..02ea98d 100644 --- a/simpunch/level1.py +++ b/simpunch/level1.py @@ -338,7 +338,6 @@ def generate_l1_cr(input_file: str, path_output: str, rotation_stage: int, space output_cdata.meta["POLAR"] = 9999 - output_cdata = update_spacecraft_location(output_cdata, output_cdata.meta.astropy_time) # Write out @@ -346,7 +345,7 @@ def generate_l1_cr(input_file: str, path_output: str, rotation_stage: int, space @flow(log_prints=True, task_runner=DaskTaskRunner( - cluster_kwargs={"n_workers": 8, "threads_per_worker": 2}, + cluster_kwargs={"n_workers": 64, "threads_per_worker": 2}, )) def generate_l1_all(datadir: str) -> None: """Generate all level 1 synthetic data. diff --git a/simpunch/level2.py b/simpunch/level2.py index 70db247..73ed301 100644 --- a/simpunch/level2.py +++ b/simpunch/level2.py @@ -20,7 +20,6 @@ from punchbowl.data import (NormalizedMetadata, get_base_file_name, load_ndcube_from_fits, write_ndcube_to_fits) from punchbowl.data.wcs import calculate_celestial_wcs_from_helio -from tqdm import tqdm from simpunch.stars import (filter_for_visible_stars, find_catalog_in_image, load_raw_hipparcos_catalog) @@ -271,7 +270,7 @@ def generate_l2_ctm(input_file: str, path_output: str) -> None: @flow(log_prints=True, task_runner=DaskTaskRunner( - cluster_kwargs={"n_workers": 8, "threads_per_worker": 2}, + cluster_kwargs={"n_workers": 64, "threads_per_worker": 2}, )) def generate_l2_all(datadir: str) -> None: """Generate all level 2 synthetic data. @@ -292,11 +291,6 @@ def generate_l2_all(datadir: str) -> None: files_ptm.sort() futures = [] - # Run individual generators - for file_ptm in tqdm(files_ptm): - futures.append(generate_l2_ptm.submit(file_ptm, outdir)) # noqa: PERF401 - - for file_ctm in tqdm(files_ctm): - futures.append(generate_l2_ctm.submit(file_ctm, outdir)) # noqa: PERF401 - + futures.extend(generate_l2_ptm.map(files_ptm, outdir)) + futures.extend(generate_l2_ctm.map(files_ctm, outdir)) wait(futures) diff --git a/simpunch/level3.py b/simpunch/level3.py index cb15702..6f4c3a8 100644 --- a/simpunch/level3.py +++ b/simpunch/level3.py @@ -215,7 +215,7 @@ def generate_l3_ctm(input_tb: str, @flow(log_prints=True, task_runner=DaskTaskRunner( - cluster_kwargs={"n_workers": 8, "threads_per_worker": 2}, + cluster_kwargs={"n_workers": 64, "threads_per_worker": 2}, )) def generate_l3_all(datadir: str, start_time: datetime, num_repeats: int = 1) -> None: """Generate all level 3 synthetic data.""" @@ -248,3 +248,24 @@ def generate_l3_all(datadir: str, start_time: datetime, num_repeats: int = 1) -> runs.append(generate_l3_ptm.submit(file_tb, file_pb, outdir, time_obs, time_delta, rotation_indices[i % 8])) runs.append(generate_l3_ctm.submit(file_tb, outdir, time_obs, time_delta, rotation_indices[i % 8])) wait(runs) + + +@flow(log_prints=True, task_runner=DaskTaskRunner( + cluster_kwargs={"n_workers": 64, "threads_per_worker": 2}, +)) +def generate_l3_all_fixed(datadir: str, times: list[datetime], file_pb, file_tb) -> None: + """Generate level 3 synthetic data at specified times with a fixed GAMERA model.""" + # Set file output path + print(f"Running from {datadir}") + outdir = os.path.join(datadir, "synthetic_l3/") + os.makedirs(outdir, exist_ok=True) + print(f"Outputting to {outdir}") + + rotation_indices = np.array([0, 0, 1, 1, 2, 2, 3, 3]) + + runs = [] + for i, time_obs in tqdm(enumerate(times), total=len(times)): + runs.append(generate_l3_ptm.submit(file_tb, file_pb, outdir, time_obs, timedelta(minutes=4), + rotation_indices[i % 8])) + runs.append(generate_l3_ctm.submit(file_tb, outdir, time_obs, timedelta(minutes=4), rotation_indices[i % 8])) + wait(runs) From 49c45eff5d9b76ad6c65b76fba365c808becc371 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Fri, 15 Nov 2024 00:32:54 -0700 Subject: [PATCH 15/19] correct length estimate for L0 --- simpunch/level0.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/simpunch/level0.py b/simpunch/level0.py index 4f69daa..0cb64ba 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -288,12 +288,9 @@ def generate_l0_all(datadir: str, psf_model_path: str, # Parse list of level 1 model data files_l1 = glob.glob(datadir + "/synthetic_l1/*L1_P*_v1.fits") files_cr = glob.glob(datadir + "/synthetic_l1/*CR*_v1.fits") - print(f"Generating based on {len(files_l1)} files.") + print(f"Generating based on {len(files_l1)+len(files_cr)} files.") files_l1.sort() - - #psf_model = ArrayPSFTransform.load(Path(psf_model_path)) - #wfi_quartic_coeffs = load_ndcube_from_fits(wfi_quartic_coeffs_path).data - #nfi_quartic_coeffs = load_ndcube_from_fits(nfi_quartic_coeffs_path).data + files_cr.sort() futures = [] futures.extend(generate_l0_pmzp.map(files_l1, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing)) From de29ba0e12daecf6552f6da1a29dcd6c6001198e Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Fri, 15 Nov 2024 02:44:15 -0700 Subject: [PATCH 16/19] change time delta to 0.25 --- simpunch/flow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 8110bc4..52def01 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -35,14 +35,14 @@ def generate_flow(gamera_directory: str, start_time = datetime(2024, 11, 13, 15, 20, 23) if generate_new: - time_delta = timedelta(days=1) + time_delta = timedelta(days=0.25) files_tb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_TB.fits")) files_pb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_PB.fits")) - previous_month = np.arange(1, -30, -1) * time_delta + start_time + previous_month = np.linspace(1, -30, timedelta(days=30)/time_delta) * time_delta + start_time generate_l3_all_fixed(gamera_directory, previous_month, files_pb[0], files_tb[0]) - next_month = np.arange(1, 30) * time_delta + start_time + next_month = np.linspace(1, 30, timedelta(days=30)/time_delta) * time_delta + start_time generate_l3_all_fixed(gamera_directory, next_month, files_pb[-1], files_tb[-1]) # generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) From 00fb7b9512e97c6f520777b5dcdbd402a100832f Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Fri, 15 Nov 2024 02:47:45 -0700 Subject: [PATCH 17/19] fix linspace needing to be int --- simpunch/flow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index 52def01..f752751 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -39,10 +39,10 @@ def generate_flow(gamera_directory: str, files_tb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_TB.fits")) files_pb = sorted(glob.glob(gamera_directory + "/synthetic_cme/*_PB.fits")) - previous_month = np.linspace(1, -30, timedelta(days=30)/time_delta) * time_delta + start_time + previous_month = np.linspace(1, -30, int(timedelta(days=30)/time_delta)) * time_delta + start_time generate_l3_all_fixed(gamera_directory, previous_month, files_pb[0], files_tb[0]) - next_month = np.linspace(1, 30, timedelta(days=30)/time_delta) * time_delta + start_time + next_month = np.linspace(1, 30, int(timedelta(days=30)/time_delta)) * time_delta + start_time generate_l3_all_fixed(gamera_directory, next_month, files_pb[-1], files_tb[-1]) # generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) From 550e58ba28236664ab525942d15555515575ca41 Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Sun, 17 Nov 2024 20:09:27 -0700 Subject: [PATCH 18/19] adds distortion creation --- create_distortion.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 create_distortion.py diff --git a/create_distortion.py b/create_distortion.py new file mode 100644 index 0000000..6b40647 --- /dev/null +++ b/create_distortion.py @@ -0,0 +1,46 @@ +"""Creates disortion models.""" +import os +from datetime import datetime + +import numpy as np +from astropy.io import fits +from astropy.time import Time +from astropy.wcs import WCS, DistortionLookupTable + +from simpunch.level1 import generate_spacecraft_wcs + +CURRENT_DIR = os.path.dirname(__file__) + +now = datetime(2024, 1, 1, 1, 1, 1) +now_str = now.strftime("%Y%m%d%H%M%S") + +for spacecraft_id in ["1", "2", "3", "4"]: + + filename_distortion = ( + os.path.join(CURRENT_DIR, "simpunch/data/distortion_NFI.fits") + if spacecraft_id == "4" + else os.path.join(CURRENT_DIR, "simpunch/data/distortion_WFI.fits") + ) + + spacecraft_wcs = generate_spacecraft_wcs(spacecraft_id, 0, Time.now()) + + with fits.open(filename_distortion) as hdul: + err_x = hdul[1].data + err_y = hdul[2].data + + crpix = err_x.shape[1] / 2 + 0.5, err_x.shape[0] / 2 + 0.5 + crval = 1024.5, 1024.5 + cdelt = (spacecraft_wcs.wcs.cdelt[1] * err_x.shape[1] / 2048, + spacecraft_wcs.wcs.cdelt[0] * err_x.shape[0] / 2048) + + cpdis1 = DistortionLookupTable( + -err_x.astype(np.float32), crpix, crval, cdelt, + ) + cpdis2 = DistortionLookupTable( + -err_y.astype(np.float32), crpix, crval, cdelt, + ) + + w = WCS(naxis=2) + w.cpdis1 = cpdis1 + w.cpdis2 = cpdis2 + w.to_fits().writeto(f"PUNCH_DD{spacecraft_id}_{now_str}_v1.fits", overwrite=True) From 22e009a0d903fee6e5453d47238eda03d4997fbb Mon Sep 17 00:00:00 2001 From: Marcus Hughes Date: Sun, 17 Nov 2024 20:11:35 -0700 Subject: [PATCH 19/19] fixes ruff --- simpunch/flow.py | 8 ++++---- simpunch/level0.py | 8 ++++++-- simpunch/level3.py | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/simpunch/flow.py b/simpunch/flow.py index f752751..8988503 100644 --- a/simpunch/flow.py +++ b/simpunch/flow.py @@ -11,7 +11,7 @@ from simpunch.level0 import generate_l0_all from simpunch.level1 import generate_l1_all from simpunch.level2 import generate_l2_all -from simpunch.level3 import generate_l3_all_fixed +from simpunch.level3 import generate_l3_all, generate_l3_all_fixed @flow(log_prints=True) @@ -45,7 +45,7 @@ def generate_flow(gamera_directory: str, next_month = np.linspace(1, 30, int(timedelta(days=30)/time_delta)) * time_delta + start_time generate_l3_all_fixed(gamera_directory, next_month, files_pb[-1], files_tb[-1]) - # generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) + generate_l3_all(gamera_directory, start_time, num_repeats=num_repeats) generate_l2_all(gamera_directory) generate_l1_all(gamera_directory) generate_l0_all(gamera_directory, @@ -75,8 +75,8 @@ def generate_flow(gamera_directory: str, if update_database: from punchpipe import __version__ - from punchpipe.controlsegment.db import File - from punchpipe.controlsegment.util import get_database_session + from punchpipe.control.db import File + from punchpipe.control.util import get_database_session db_session = get_database_session() for file_path in sorted(glob.glob(os.path.join(gamera_directory, "synthetic_l0/*v[0-9].fits")), key=lambda s: os.path.basename(s)[13:27]): diff --git a/simpunch/level0.py b/simpunch/level0.py index 0cb64ba..9292760 100644 --- a/simpunch/level0.py +++ b/simpunch/level0.py @@ -293,6 +293,10 @@ def generate_l0_all(datadir: str, psf_model_path: str, files_cr.sort() futures = [] - futures.extend(generate_l0_pmzp.map(files_l1, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing)) - futures.extend(generate_l0_cr.map(files_cr, outdir, psf_model_path, wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, transient_probability, shift_pointing)) + futures.extend(generate_l0_pmzp.map(files_l1, outdir, psf_model_path, + wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, + transient_probability, shift_pointing)) + futures.extend(generate_l0_cr.map(files_cr, outdir, psf_model_path, + wfi_quartic_coeffs_path, nfi_quartic_coeffs_path, + transient_probability, shift_pointing)) wait(futures) diff --git a/simpunch/level3.py b/simpunch/level3.py index 6f4c3a8..d228758 100644 --- a/simpunch/level3.py +++ b/simpunch/level3.py @@ -253,7 +253,7 @@ def generate_l3_all(datadir: str, start_time: datetime, num_repeats: int = 1) -> @flow(log_prints=True, task_runner=DaskTaskRunner( cluster_kwargs={"n_workers": 64, "threads_per_worker": 2}, )) -def generate_l3_all_fixed(datadir: str, times: list[datetime], file_pb, file_tb) -> None: +def generate_l3_all_fixed(datadir: str, times: list[datetime], file_pb: str, file_tb: str) -> None: """Generate level 3 synthetic data at specified times with a fixed GAMERA model.""" # Set file output path print(f"Running from {datadir}")