From d0b58e8d469acb2fcda8dc699875e32017a950ba Mon Sep 17 00:00:00 2001 From: kfanning Date: Mon, 25 Nov 2024 15:44:54 -0300 Subject: [PATCH 1/2] DM-47791: adding make_lsstcam_calibrations.py --- .../maintel/make_lsstcam_calibrations.py | 26 +++ .../ts/externalscripts/maintel/__init__.py | 1 + .../maintel/make_lsstcam_calibrations.py | 189 ++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100755 python/lsst/ts/externalscripts/data/scripts/maintel/make_lsstcam_calibrations.py create mode 100644 python/lsst/ts/externalscripts/maintel/make_lsstcam_calibrations.py diff --git a/python/lsst/ts/externalscripts/data/scripts/maintel/make_lsstcam_calibrations.py b/python/lsst/ts/externalscripts/data/scripts/maintel/make_lsstcam_calibrations.py new file mode 100755 index 000000000..b8214d870 --- /dev/null +++ b/python/lsst/ts/externalscripts/data/scripts/maintel/make_lsstcam_calibrations.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# This file is part of ts_externalscripts +# +# Developed for the LSST Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +import asyncio + +from lsst.ts.externalscripts.maintel import MakeLSSTCamCalibrations + +asyncio.run(MakeLSSTCamCalibrations.amain()) diff --git a/python/lsst/ts/externalscripts/maintel/__init__.py b/python/lsst/ts/externalscripts/maintel/__init__.py index 548abdaa3..d21b0dff1 100644 --- a/python/lsst/ts/externalscripts/maintel/__init__.py +++ b/python/lsst/ts/externalscripts/maintel/__init__.py @@ -20,6 +20,7 @@ # along with this program. If not, see . from .make_comcam_calibrations import * +from .make_lsstcam_calibrations import * from .parameter_march_comcam import * from .parameter_march_lsstcam import * from .take_comcam_guider_image import * diff --git a/python/lsst/ts/externalscripts/maintel/make_lsstcam_calibrations.py b/python/lsst/ts/externalscripts/maintel/make_lsstcam_calibrations.py new file mode 100644 index 000000000..b1ba1d3ef --- /dev/null +++ b/python/lsst/ts/externalscripts/maintel/make_lsstcam_calibrations.py @@ -0,0 +1,189 @@ +# This file is part of ts_externalscripts +# +# Developed for the LSST Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + +__all__ = ["MakeLSSTCamCalibrations"] + +import yaml +from lsst.ts.observatory.control import RemoteGroup +from lsst.ts.observatory.control.maintel.lsstcam import LSSTCam + +from ..base_make_calibrations import BaseMakeCalibrations + + +class MakeLSSTCamCalibrations(BaseMakeCalibrations): + """Class for taking images, constructing, verifying, and + certifying combined calibrations with LSSTCam. + + This class takes bias, darks, and flat exposures with LSSTCam, + constructs a combined calibration for each image type by calling + the appropiate pipetask via OCPS, and then verifies and certifies + each combined calibration. It also optionally produces defects and + Photon Transfer Curves. + """ + + def __init__(self, index=1): + super().__init__( + index=index, + descr="Takes series of bias, darks and flat-field exposures" + "with LSSTCam, and constructs combined " + "calibrations, verify and certify the results.", + ) + self._lsstcam = LSSTCam(domain=self.domain, log=self.log) + self._ocps_group = RemoteGroup( + domain=self.domain, components=["OCPS:2"], log=self.log + ) + + @property + def camera(self): + return self._lsstcam + + @property + def ocps_group(self): + """Access the Remote OCPS Groups in the constructor. + + The OCPS index will be 2 for LSSTCam: OCPS:2. + """ + return self._ocps_group + + @property + def ocps(self): + return self.ocps_group.rem.ocps_2 + + @property + def instrument_name(self): + """String with instrument name for pipeline task""" + return "LSSTCam" + + @property + def pipeline_instrument(self): + """String with instrument name for pipeline yaml file""" + return "LSSTCam" + + @property + def detectors(self): + """Array with detector IDs""" + return self.config.detectors if self.config is not None else [] + + @property + def n_detectors(self): + """Number of detectors""" + return ( + len(self.config.detectors) + if self.config is not None and self.config.detectors + else 197 + ) + + @property + def image_in_oods(self): + """OODS imageInOODS event.""" + return self.camera.rem.ccoods.evt_imageInOODS + + @classmethod + def get_schema(cls): + url = "https://github.com/lsst-ts/" + path = ( + "ts_externalscripts/blob/main/python/lsst/ts/externalscripts/" + "/make_lsstcam_calibrations.py" + ) + schema = f""" + $schema: http://json-schema.org/draft-07/schema# + $id: {url}/{path} + title: MakeLSSTCamCalibrations v1 + description: Configuration for making a LSSTCam combined calibrations SAL Script. + type: object + properties: + detectors: + description: Detector IDs. If omitted, all 197 (wavefronts are 2, without guiders) \ + LSSTCam detectors will be used. + type: array + items: + - type: integer + minContains: 0 + maxContains: 204 + minItems: 0 + maxItems: 205 + uniqueItems: true + default: [] + filter: + description: Filter name or ID; if omitted the filter is not changed. + anyOf: + - type: string + - type: integer + minimum: 1 + - type: "null" + default: null + input_collections_bias: + type: string + descriptor: Additional comma-separated input collections to pass to the bias pipetask. + default: "LSSTCam/calib" + input_collections_verify_bias: + type: string + descriptor: Additional comma-separated input collections to pass to \ + the verify (bias) pipetask. + default: "LSSTCam/calib" + input_collections_dark: + type: string + descriptor: Additional comma-separarted input collections to pass to the dark pipetask. + default: "LSSTCam/calib" + input_collections_verify_dark: + type: string + descriptor: Additional comma-separated input collections to pass to \ + the verify (dark) pipetask. + default: "LSSTCam/calib" + input_collections_flat: + type: string + descriptor: Additional comma-separated input collections to pass to the flat pipetask. + default: "LSSTCam/calib" + input_collections_verify_flat: + type: string + descriptor: Additional comma-separated input collections to pass to \ + the verify (flat) pipetask. + default: "LSSTCam/calib" + input_collections_defects: + type: string + descriptor: Additional comma-separated input collections to pass to the defects pipetask. + default: "LSSTCam/calib" + input_collections_ptc: + type: string + descriptor: Additional comma-separated input collections to pass to the \ + Photon Transfer Curve pipetask. + default: "LSSTCam/calib" + calib_collection: + type: string + descriptor: Calibration collection where combined calibrations will be certified into. + default: "LSSTCam/calib/daily" + repo: + type: string + descriptor: Butler repository. + default: "/repo/LSSTCam/butler+sasquatch.yaml" + additionalProperties: false + """ + schema_dict = yaml.safe_load(schema) + base_schema_dict = super().get_schema() + + for properties in base_schema_dict["properties"]: + schema_dict["properties"][properties] = base_schema_dict["properties"][ + properties + ] + + return schema_dict + + def get_instrument_configuration(self): + return dict(filter=self.config.filter) From f91a4d0f5f88f019443b1309276f6b4dcbeac132 Mon Sep 17 00:00:00 2001 From: kfanning Date: Mon, 25 Nov 2024 15:46:18 -0300 Subject: [PATCH 2/2] DM-47791: add test case and news fragment. --- doc/news/DM-47791.feature.rst | 1 + .../maintel/test_make_lsstcam_calibrations.py | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 doc/news/DM-47791.feature.rst create mode 100644 tests/maintel/test_make_lsstcam_calibrations.py diff --git a/doc/news/DM-47791.feature.rst b/doc/news/DM-47791.feature.rst new file mode 100644 index 000000000..d5f5017e9 --- /dev/null +++ b/doc/news/DM-47791.feature.rst @@ -0,0 +1 @@ +DM-47791: adding make_lsstcam_calibrations.py in line with ComCam and LATISS versions. \ No newline at end of file diff --git a/tests/maintel/test_make_lsstcam_calibrations.py b/tests/maintel/test_make_lsstcam_calibrations.py new file mode 100644 index 000000000..9fad441df --- /dev/null +++ b/tests/maintel/test_make_lsstcam_calibrations.py @@ -0,0 +1,120 @@ +# This file is part of ts_standardscripts +# +# Developed for the LSST Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License + + +import logging +import unittest + +import pytest +from lsst.ts import externalscripts, salobj, standardscripts +from lsst.ts.externalscripts.maintel import MakeLSSTCamCalibrations + +logger = logging.getLogger(__name__) +logger.propagate = True + + +class TestMakeLSSTCamCalibrations( + standardscripts.BaseScriptTestCase, unittest.IsolatedAsyncioTestCase +): + async def basic_make_script(self, index): + logger.debug("Starting basic_make_script") + self.script = MakeLSSTCamCalibrations(index=index) + + logger.debug("Finished initializing from basic_make_script") + # Return a single element tuple + return (self.script,) + + @unittest.mock.patch( + "lsst.ts.standardscripts.BaseBlockScript.obs_id", "202306060001" + ) + async def test_configure(self): + async with self.make_script(): + # Try configure with minimum set of parameters declared + # Note that all are scalars and should be converted to arrays + n_bias = 2 + n_dark = 2 + exp_times_dark = 10 + n_flat = 4 + exp_times_flat = [10, 10, 50, 50] + detectors = [0, 1, 2, 3, 4, 5, 6, 7, 8] + n_processes = 4 + program = "BLOCK-123" + reason = "SITCOM-321" + + self.script.get_obs_id = unittest.mock.AsyncMock( + side_effect=["202306060001"] + ) + + await self.configure_script( + n_bias=n_bias, + n_dark=n_dark, + n_flat=n_flat, + exp_times_dark=exp_times_dark, + exp_times_flat=exp_times_flat, + detectors=detectors, + n_processes=n_processes, + program=program, + reason=reason, + ) + + assert self.script.config.n_bias == n_bias + assert self.script.config.n_dark == n_dark + assert self.script.config.n_flat == n_flat + assert self.script.config.exp_times_dark == exp_times_dark + assert self.script.config.exp_times_flat == exp_times_flat + assert self.script.config.n_processes == n_processes + assert self.script.config.detectors == detectors + assert self.script.program == program + assert self.script.reason == reason + assert ( + self.script.checkpoint_message + == "MakeLSSTCamCalibrations BLOCK-123 202306060001 SITCOM-321" + ) + + async def test_configure_invalid_program_name(self): + async with self.make_script(): + n_bias = 2 + n_dark = 2 + exp_times_dark = 10 + n_flat = 4 + exp_times_flat = [10, 10, 50, 50] + detectors = [0, 1, 2, 3, 4, 5, 6, 7, 8] + n_processes = 4 + program = "BLOCK_123" + reason = "SITCOM-321" + + with pytest.raises(salobj.ExpectedError): + await self.configure_script( + n_bias=n_bias, + n_dark=n_dark, + n_flat=n_flat, + exp_times_dark=exp_times_dark, + exp_times_flat=exp_times_flat, + detectors=detectors, + n_processes=n_processes, + program=program, + reason=reason, + ) + + async def test_executable(self): + scripts_dir = externalscripts.get_scripts_dir() + script_path = scripts_dir / "maintel" / "make_lsstcam_calibrations.py" + logger.debug(f"Checking for script in {script_path}") + await self.check_executable(script_path)