diff --git a/CHANGES.rst b/CHANGES.rst index ff7453565c..c43eb166a3 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,6 +27,10 @@ assign_wcs - Update default parameters to increase the accuracy of the SIP approximation in the output FITS WCS. [#8529] +- Added handling for fixed slit sources defined in a MSA metadata file, for combined + NIRSpec MOS and fixed slit observations. Slits are now appended to the data + product in the order they appear in the MSA file. [#8467] + associations ------------ @@ -83,6 +87,10 @@ exp_to_source in order to support changes in `source_id` handling for NIRSpec MOS exposures that contain background and virtual slits. [#8442] +- Update the top-level model exposure type from the slit exposure type, + to support processing for combined NIRSpec MOS and fixed slit + observations. [#8467] + extract_1d ---------- @@ -108,11 +116,19 @@ extract_1d - Fix error in application of aperture correction to variance arrays. [#8530] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + extract_2d ---------- - Added handling for NIRCam GRISM time series pointing offsets. [#8449] +- Added support for slit names that have string values instead of integer + values, necessary for processing combined NIRSpec MOS and fixed slit + data products. [#8467] + flat_field ---------- @@ -122,9 +138,13 @@ flat_field - Update NIRSpec flatfield code for all modes to ensure SCI=ERR=NaN wherever the DO_NOT_USE flag is set in the DQ array. [#8463] -- Updated the NIRSpec flatfield code to use the new format of the ``wavecorr`` +- Updated the NIRSpec flatfield code to use the new format of the ``wavecorr`` wavelength zero-point corrections for point sources. [#8376] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + general ------- @@ -139,9 +159,22 @@ lib --- - Updated the ``wcs_utils.get_wavelength`` to use the new format - of the ``wavecorr`` wavelength zero-point corrections for point + of the ``wavecorr`` wavelength zero-point corrections for point sources in NIRSpec slit data. [#8376] +master_background +----------------- + +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + +- Disabled support for master background correction for NIRSpec MOS + slits in the ``master_background``, called in ``calwebb_spec3``. + Master background correction for MOS mode should be performed + via ``master_background_mos``, called in ``calwebb_spec2``. [#8467] + + master_background_mos --------------------- @@ -156,6 +189,10 @@ nsclean sparse matrix to perform linear algebra operations with a diagonal weight matrix. [#8547] +- Added a check for combined NIRSpec MOS and fixed slit products: if fixed + slits are defined in a MOS product, the central fixed slit quadrant + is not automatically masked. [#8467] + outlier_detection ----------------- @@ -198,6 +235,10 @@ photom - Ensure that NaNs in MRS photom files are not replaced with ones by pipeline code for consistency with other modes [#8453] +- Removed a check for the primary slit for NIRSpec fixed slit mode: + all slits containing point sources are now handled consistently, + whether they are marked primary or not. [#8467] + pipeline -------- @@ -215,6 +256,12 @@ pipeline - Added new optional step ``badpix_selfcal`` to the ``calwebb_spec2`` to self-calibrate bad pixels in IFU data. [#8500] +- Added ``calwebb_spec2`` pipeline handling for combined NIRSpec MOS and + fixed slit observations. Steps that require different reference files + for MOS and FS are run twice, first for all MOS slits, then for all + FS slits. Final output products (``cal``, ``s2d``, ``x1d``) contain the + combined products. [#8467] + pixel_replace ------------- @@ -303,7 +350,12 @@ wavecorr - Changed the NIRSpec wavelength correction algorithm to include it in slit WCS models and resampling. Fixed the sign of the wavelength corrections. [#8376] - + +- Added a check for fixed slits that already have source position information, + assigned via a MSA metafile, for combined NIRSpec MOS and fixed slit processing. + Point source position is calculated from dither offsets only for standard + fixed slit processing. [#8467] + wfss_contam ----------- @@ -615,7 +667,6 @@ resample - Changed deprecated ``stpipe.extern.configobj`` to ``astropy.extern.configobj``. [#8320] - residual_fringe --------------- diff --git a/docs/jwst/data_products/msa_metadata.rst b/docs/jwst/data_products/msa_metadata.rst index b251402c80..5173000aa0 100644 --- a/docs/jwst/data_products/msa_metadata.rst +++ b/docs/jwst/data_products/msa_metadata.rst @@ -43,7 +43,7 @@ The overall structure of the MSA FITS file is as follows: +-----+---------------+----------+-----------+--------------------+ | 1 | SHUTTER_IMAGE | IMAGE | float32 | 342 x 730 | +-----+---------------+----------+-----------+--------------------+ -| 2 | SHUTTER_INFO | BINTABLE | N/A | variable x 12 cols | +| 2 | SHUTTER_INFO | BINTABLE | N/A | variable x 13 cols | +-----+---------------+----------+-----------+--------------------+ | 3 | SOURCE_INFO | BINTABLE | N/A | variable x 8 cols | +-----+---------------+----------+-----------+--------------------+ @@ -89,6 +89,8 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: +-------------------------------+-----------+----------------------+ | PRIMARY_SOURCE | string | Primary source flag | +-------------------------------+-----------+----------------------+ +| FIXED_SLIT | string | Fixed slit name | ++-------------------------------+-----------+----------------------+ - SLITLET_ID: integer ID of each slitlet consisting of one or more open shutters. @@ -102,8 +104,8 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: quadrant. - SOURCE_ID: unique integer ID for each source in each slitlet, used for matching to an entry in the ``SOURCE_INFO`` table. -- BACKGROUND: boolean indicating whether the shutter is open to background - only or contains a known source. +- BACKGROUND: Y or N. Y indicates that the shutter is open to background + only. - SHUTTER_STATE: OPEN or CLOSED. Generally will always be OPEN, unless the shutter is part of a long slit that is not contiguous. - ESTIMATED_SOURCE_IN_SHUTTER_X/Y: the position of the source within the @@ -111,8 +113,11 @@ The structure of the ``SHUTTER_INFO`` table extension is as follows: and 1,1 is upper-right, as planned in the MPT. - DITHER_POINT_INDEX: integer index of the nod sequence; matches to header keyword `PATT_NUM`. -- PRIMARY_SOURCE: boolean indicating whether the shutter contains the +- PRIMARY_SOURCE: Y or N. Y indicates that the shutter contains the primary science source. +- FIXED_SLIT: string name of a fixed slit containing the source; set to + NONE for MSA slitlets. This column may not appear in older versions of + the MSA metadata files. It is the :ref:`assign_wcs ` step in the :ref:`calwebb_spec2 ` pipeline that opens and loads all @@ -140,29 +145,29 @@ slitlet 2 is comprised of 3 shutters. Because a 3-point nod pattern has been use there are 3 different sets of metadata for each slitlet (one set for each dither/nod position) and hence a total of 9 entries (3 shutters x 3 dithers). -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| Slit | Meta | | | | Src | | | X | Y | Dith | Pri | -| | | | | | | | | | | | | -| ID | ID | Quad | Row | Col | ID | Bkg | State | pos | pos | Pt | Src | -+======+======+======+=====+=====+========+=====+=======+=======+=======+======+=====+ -| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 1 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 42 | N | OPEN | 0.399 | 0.702 | 1 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 1 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 154 | 42 | N | OPEN | 0.410 | 0.710 | 2 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 2 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 2 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 3 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 3 | N | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ -| 2 | 1 | 2 | 10 | 156 | 42 | N | OPEN | 0.389 | 0.718 | 3 | Y | -+------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+ ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| Slit | Meta | | | | Src | | | X | Y | Dith | Pri | Fxd | +| | | | | | | | | | | | | | +| ID | ID | Quad | Row | Col | ID | Bkg | State | pos | pos | Pt | Src | Slit | ++======+======+======+=====+=====+========+=====+=======+=======+=======+======+=====+======+ +| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 1 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 42 | N | OPEN | 0.399 | 0.702 | 1 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 1 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 154 | 42 | N | OPEN | 0.410 | 0.710 | 2 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 2 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 0 | Y | OPEN | NaN | NaN | 2 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 154 | 0 | Y | OPEN | NaN | NaN | 3 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 155 | 0 | Y | OPEN | NaN | NaN | 3 | N | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ +| 2 | 1 | 2 | 10 | 156 | 42 | N | OPEN | 0.389 | 0.718 | 3 | Y | NONE | ++------+------+------+-----+-----+--------+-----+-------+-------+-------+------+-----+------+ The values in the `slitlet_id` column show that we're only looking at table rows for slitlet 2, all of which come from MSA configuration (`msa_metadata_id`) 1. @@ -229,6 +234,20 @@ from a virtual slitlet with `source_id` = -42 will look like: jw12345-o066_v000000042_nirspec_f170lp_g235m_x1d.fits +Fixed Slits +~~~~~~~~~~~ + +It is possible to plan fixed slit sources alongside standard MOS targets. In this case, +a unique `slitlet_id` is not assigned in the MSA file. Instead, the slit is identified +by the value of the `fixed_slit` column. This value may be set to any of the NIRSpec +fixed slit names used for science: S200A1, S200A2, S400A1, or S1600A1. + +Fixed slit targets must always have `primary_source` = "Y" and `background` = "N". +They will never be extracted as background sources. + +The `shutter_quadrant`, `shutter_row`, and `shutter_column` fields are set to placeholder +values. All other values have the same meaning and values as for MSA slitlets. + The SOURCE_INFO Metadata ------------------------ diff --git a/docs/jwst/exp_to_source/main.rst b/docs/jwst/exp_to_source/main.rst index 0216274adf..2e862079d2 100644 --- a/docs/jwst/exp_to_source/main.rst +++ b/docs/jwst/exp_to_source/main.rst @@ -34,14 +34,21 @@ configuration of MSA slitlets with a source assigned to each slitlet. The source-to-slitlet linkage is carried along in the information contained in the MSA metadata file used during :ref:`calwebb_spec2 ` processing. Each slitlet instance created by the :ref:`extract_2d ` -step stores the source ID (a simple integer number) in the SOURCEID keyword of +step stores the source name in the SRCNAME keyword of the SCI extension header for the slitlet. The ``exp_to_source`` tool uses -the SOURCEID values to sort the data from each input product into an +the SRCNAME values to sort the data from each input product into an appropriate source-based output product. -NIRSpec Fixed-Slit +If fixed slit targets are planned as part of a NIRSpec MOS exposure, they +will also have sources identified by the MSA metadata file. The associated +SRCNAME values are used to sort the data from these slits in the same way the +MSA slitlets are sorted. In this combined mode, the products containing +MOS slitlets are marked with EXP_TYPE = "NRS_MSASPEC" after sorting; the models +containing fixed slits are marked with EXP_TYPE = "NRS_FIXEDSLIT". + +NIRSpec Fixed Slit ^^^^^^^^^^^^^^^^^^ -NIRSpec fixed-slit observations do not have sources identified with each +NIRSpec fixed slit observations do not have sources identified with each slit, so the slit names, e.g. S200A1, S1600A1, etc., are mapped to predefined source ID values, as follows: diff --git a/docs/jwst/flatfield/main.rst b/docs/jwst/flatfield/main.rst index 3cd4111764..f6592b2105 100644 --- a/docs/jwst/flatfield/main.rst +++ b/docs/jwst/flatfield/main.rst @@ -78,12 +78,12 @@ pixel. This interpolation requires knowledge of the dispersion direction, which is read from keyword "DISPAXIS." See the Reference File section for further details. -For NIRSpec Fixed-Slit and MOS exposures, an on-the-fly flat-field is +For NIRSpec Fixed Slit (FS) and MOS exposures, an on-the-fly flat-field is constructed to match each of the slits/slitlets contained in the science exposure. For NIRSpec IFU exposures, a single full-frame flat-field is constructed, which is applied to the entire science image. -NIRSpec NRS_BRIGHTOBJ data are processed just like NIRSpec Fixed-Slit +NIRSpec NRS_BRIGHTOBJ data are processed just like NIRSpec fixed slit data, except that NRS_BRIGHTOBJ data are stored in a CubeModel, rather than a MultiSlitModel. A 2-D flat-field image is constructed on-the-fly as usual, but this image is then divided into each plane of @@ -92,9 +92,9 @@ the 3-D science data arrays. In all cases, there is a step option that allows for saving the on-the-fly flatfield to a file, if desired. -NIRSpec Fixed-Slit Primary Slit +NIRSpec Fixed Slit Primary Slit ------------------------------- -The primary slit in a NIRSpec fixed-slit exposure receives special handling. +The primary slit in a NIRSpec fixed slit exposure receives special handling. If the primary slit, as given by the "FXD_SLIT" keyword value, contains a point source, as given by the "SRCTYPE" keyword, it is necessary to know the flatfield conversion factors for both a point source and a uniform source @@ -104,7 +104,7 @@ is applied to the slit data, but that correction is not appropriate for the background signal contained in the slit, and hence corrections must be applied later in the :ref:`master background ` step. -So in this case the `flatfield` step will compute 2D arrays of conversion +In this case, the `flatfield` step will compute 2D arrays of conversion factors that are appropriate for a uniform source and for a point source, and store those correction factors in the "FLATFIELD_UN" and "FLATFIELD_PS" extensions, respectively, of the output data product. The point source @@ -116,8 +116,15 @@ applied by the :ref:`wavecorr ` step to account for any source mis-centering in the slit and the flatfield conversion factors are wavelength-dependent. A uniform source does not require wavelength corrections and hence the flatfield conversions will differ for point and uniform -sources. Any secondary slits that may be included in a fixed-slit exposure +sources. Any secondary slits that may be included in a fixed slit exposure do not have source centering information available, so the -:ref:`wavecorr ` step is not applied, and hence there's no +:ref:`wavecorr ` step is not applied, and there is no difference between the point source and uniform source flatfield conversions for those slits. + +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are treated in the same manner as the primary fixed slit in standard +FS observations. diff --git a/docs/jwst/master_background/description.rst b/docs/jwst/master_background/description.rst index a436ecdc1d..6a5d39b10b 100644 --- a/docs/jwst/master_background/description.rst +++ b/docs/jwst/master_background/description.rst @@ -295,27 +295,25 @@ source centering within the slit, hence slits containing uniform sources receive the same flat-field and photometric calibrations as background spectra and therefore don't require corrections for those two calibrations. Furthermore, the source position in the slit is only known for the primary slit in an exposure, so -even if the secondary slits contain point sources, no wavelength correction can -be applied, and therefore again the flat-field and photometric calibrations are -the same as for background spectra. This means only the pathloss correction -difference between uniform and point sources needs to be accounted for in the -secondary slits. +secondary slits are always handled as extended sources, no wavelength correction is +applied, and therefore again the flat-field, photometric, and pathloss calibrations +are the same as for background spectra. -Therefore if the primary slit (as given by the FXD_SLIT keyword) contains a point source -(as given by the SRCTYPE keyword) the corrections that need to be applied to the 2-D -master background for that slit are: +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are treated in the same manner as the primary fixed slit in standard +FS observations. + +Therefore, if a fixed slit contains a point source (as given by the SRCTYPE keyword) +the corrections that need to be applied to the 2-D master background for that slit are: .. math:: bkg(corr) = bkg &* [flatfield(uniform) / flatfield(point)]\\ &* [pathloss(uniform) / pathloss(point)]\\ &* [photom(point) / photom(uniform)] -For secondary slits that contain a point source, the correction applied to the -2-D master background is simply: - -.. math:: - bkg(corr) = bkg * pathloss(uniform) / pathloss(point) - The uniform and point source versions of the flat-field, pathloss, and photom corrections are retrieved from the input :ref:`cal ` product. They are computed and stored there during the execution of each of those steps diff --git a/docs/jwst/nsclean/main.rst b/docs/jwst/nsclean/main.rst index 48a9c615d9..c61e406094 100644 --- a/docs/jwst/nsclean/main.rst +++ b/docs/jwst/nsclean/main.rst @@ -103,6 +103,10 @@ processing MOS and IFU images. The masked region is currently hardwired in the step to image indexes [1:2048, 923:1116], where the indexes are in x, y order and in 1-indexed values. +Note, however, that it is possible to plan one or more fixed slit targets +alongside MSA slitlets in MOS observations. In this situation, the fixed +slit region is not automatically masked. + Left/Right Reference Pixel Columns ---------------------------------- Full-frame images contain 4 columns of reference pixels on the left and diff --git a/docs/jwst/photom/main.rst b/docs/jwst/photom/main.rst index 1d98734cb5..0e0b478b64 100644 --- a/docs/jwst/photom/main.rst +++ b/docs/jwst/photom/main.rst @@ -84,9 +84,9 @@ conversion factor. If the time-dependent coefficients are present in the reference file, the photom step will apply the correction based on the observation date of the exposure being processed. -NIRSpec Fixed-Slit Primary Slit +NIRSpec Fixed Slit Primary Slit ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The primary slit in a NIRSpec fixed-slit exposure receives special handling. +The primary slit in a NIRSpec fixed slit exposure receives special handling. If the primary slit, as given by the "FXD_SLIT" keyword value, contains a point source, as given by the "SRCTYPE" keyword, it is necessary to know the photometric conversion factors for both a point source and a uniform source @@ -96,7 +96,7 @@ is applied to the slit data, but that correction is not appropriate for the background signal contained in the slit, and hence corrections must be applied later in the :ref:`master background ` step. -So in this case the `photom` step will compute 2D arrays of conversion +In this case, the `photom` step will compute 2D arrays of conversion factors that are appropriate for a uniform source and for a point source, and store those correction factors in the "PHOTOM_UN" and "PHOTOM_PS" extensions, respectively, of the output data product. The point source @@ -105,7 +105,7 @@ correction array is also applied to the slit data. Note that this special handling is only needed when the slit contains a point source, because in that case corrections to the wavelength grid are applied by the :ref:`wavecorr ` step to account for any -source mis-centering in the slit and the photometric conversion factors are +source offsets in the slit and the photometric conversion factors are wavelength-dependent. A uniform source does not require wavelength corrections and hence the photometric conversions will differ for point and uniform sources. Any secondary slits that may be included in a fixed-slit exposure @@ -114,6 +114,13 @@ do not have source centering information available, so the difference between the point source and uniform source photometric conversions for those slits. +Fixed slits planned as part of a combined MOS and FS observation are an +exception to this rule. These targets may each be identified as +point sources, with location information for each given in the +:ref:`MSA metadata file `. Point sources in fixed slits planned +this way are all treated in the same manner as the primary fixed slit in standard +FS observations. + Pixel Area Data ^^^^^^^^^^^^^^^ For all instrument modes other than NIRSpec the photom step loads a 2-D pixel diff --git a/docs/jwst/pipeline/calwebb_spec2.rst b/docs/jwst/pipeline/calwebb_spec2.rst index af3284c6c6..d2ad75724b 100644 --- a/docs/jwst/pipeline/calwebb_spec2.rst +++ b/docs/jwst/pipeline/calwebb_spec2.rst @@ -116,6 +116,48 @@ performed on the original (unresampled) data. The :ref:`cube_build ` step. +Combined NIRSpec MOS and FS Exposures +------------------------------------- + +It is possible to observe one or more fixed slit sources as part of a NIRSpec MOS +observation. Fixed slits observed this way are mostly handled as if they were +MOS slitlets: they are assigned a WCS, extracted from the full-frame image, calibrated, +and appended to the output data products. + +However, since FS and MOS modes require different pipeline steps and reference files +at various points in the ``calwebb_spec2`` pipeline, the processing path is not as +straightforward as for a standard MOS exposure. Internal to the pipeline, the data +product is sorted into MOS slits and FS slits. MOS slits are processed together first, +then FS slits are processed through the same steps. At the end of the processing, +the calibrated images and spectra are recombined into final data products containing +all observed slits. + +The detailed processing flow is as follows: + +- Fixed slits containing primary sources are identified in the input + :ref:`MSA metadata file ` in the :ref:`assign_wcs ` + step, via the "fixed_slit" column in the MSA SHUTTER_INFO table. +- All slits (MOS and FS) are processed together through the :ref:`srctype ` step. +- MOS slits are processed together through the :ref:`master_background `, + :ref:`wavecorr `, :ref:`flat_field `, + :ref:`pathloss `, :ref:`barshadow `, and + :ref:`photom ` steps. +- FS slits are processed together through the :ref:`wavecorr `, + :ref:`flat_field `, :ref:`pathloss `, and + :ref:`photom ` steps. If intermediate products from these + steps are saved, they will have an additional "_fs" suffix appended to + their file names. +- MOS and FS slits are recombined and processed together through the + :ref:`resample_spec ` step. +- MOS slits are processed through the :ref:`extract_1d ` step, + then FS slits are processed through the same step. The extracted spectra are + recombined into a final data product. + +The combined, calibrated output product for this mode may be used as input for +the :ref:`calwebb_spec3 ` pipeline. Since that pipeline sorts and +separates the data by source, the fixed slit and MOS targets are independently handled +through all pipeline steps with no further accommodation necessary. + NIRSpec Lamp Exposures ---------------------- diff --git a/docs/jwst/pipeline/calwebb_spec3.rst b/docs/jwst/pipeline/calwebb_spec3.rst index 4331170368..307311bc58 100644 --- a/docs/jwst/pipeline/calwebb_spec3.rst +++ b/docs/jwst/pipeline/calwebb_spec3.rst @@ -51,6 +51,9 @@ to observations of a moving target (TARGTYPE='moving'). :sup:`2`\ The master background subtraction step is applied to NIRSpec MOS exposures in the :ref:`calwebb_spec2 ` pipeline. +WFSS and SOSS Processing +------------------------ + Notice that NIRCam and NIRISS WFSS, as well as NIRISS SOSS data, receive only minimal processing by ``calwebb_spec3``. WFSS 2D input data are reorganized into source-based products by the @@ -66,6 +69,19 @@ obtained in TSO mode. TSO mode NIRISS SOSS exposures should be processed with the :ref:`calwebb_tso3 ` pipeline. +Combined NIRSpec MOS and FS Exposures +------------------------------------- + +It is possible to observe NIRSpec fixed slit targets alongside MSA +slitlets in NIRSpec MOS exposures. In this case, the input files produced by the +:ref:`calwebb_spec2 ` pipeline contain both MOS and FS data. Any +:ref:`master_background ` corrections applied in this pipeline +will operate on the fixed slits only, since background corrections for MOS slitlets should +be applied in the :ref:`calwebb_spec2 ` pipeline. After the sources are +separated in the :ref:`exp_to_source ` step, each data product will contain +only a fixed slit or a MOS target, so all further processing follows the standard flow +for each exposure type. + Arguments --------- diff --git a/docs/jwst/wavecorr/description.rst b/docs/jwst/wavecorr/description.rst index fa7e311d5a..fcaba1f7fe 100644 --- a/docs/jwst/wavecorr/description.rst +++ b/docs/jwst/wavecorr/description.rst @@ -49,7 +49,7 @@ estimate the fractional location of the source within the given slit. Note that this computation can only be performed for the primary slit in the exposure, which is given in the "FXD_SLIT" keyword. The positions of sources in any additional slits cannot be estimated and therefore -the wavelength correction is only applied to the primary slit. +the wavelength correction is only applied to the primary slit.\ :sup:`1` The estimated position of the source within the primary slit (in the dispersion direction) is then used in the same manner as described above @@ -58,3 +58,9 @@ for the primary slit. Upon successful completion of the step, the status keyword "S_WAVCOR" is set to "COMPLETE". + +:sup:`1`\ Note that fixed slits that are planned as part of a combined +MOS and FS observation do have *a priori* estimates of their source +locations, via the :ref:`MSA metadata file`. When available, +these source locations are directly used, instead of recomputing the source +position in the ``wavecorr`` step. diff --git a/jwst/assign_wcs/nirspec.py b/jwst/assign_wcs/nirspec.py index 3577a84d57..740782a099 100644 --- a/jwst/assign_wcs/nirspec.py +++ b/jwst/assign_wcs/nirspec.py @@ -35,6 +35,8 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) +FIXED_SLIT_NUMS = {'NONE': 0, 'S200A1': 1, 'S200A2': 2, + 'S400A1': 3, 'S1600A1': 4, 'S200B1': 5} __all__ = ["create_pipeline", "imaging", "ifu", "slits_wcs", "get_open_slits", "nrs_wcs_set_input", "nrs_ifu_wcs", "get_spectral_order_wrange"] @@ -446,12 +448,11 @@ def get_open_fixed_slits(input_model, slit_y_range=[-.55, .55]): if input_model.meta.instrument.fixed_slit is None: input_model.meta.instrument.fixed_slit = 'NONE' - slit_nums = {'NONE':0, 'S200A1':1, 'S200A2':2, 'S400A1':3, 'S1600A1':4, 'S200B1':5} primary_slit = input_model.meta.instrument.fixed_slit ylow, yhigh = slit_y_range # Slits are defined with hardwired source ID's, based on the assignments - # in the "slit_nums" dictionary. Exact assignments depend on whether the + # in the "FIXED_SLIT_NUMS" dictionary. Exact assignments depend on whether the # slit is the "primary" and hence contains the target of interest. The # source_id for the primary slit is always 1, while source_ids for secondary # slits is a two-digit value, where the first (tens) digit corresponds to @@ -461,15 +462,20 @@ def get_open_fixed_slits(input_model, slit_y_range=[-.55, .55]): # # Slit(Name, ShutterID, DitherPos, Xcen, Ycen, Ymin, Ymax, Quad, SourceID) s2a1 = Slit('S200A1', 0, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200A1' else 10*slit_nums[primary_slit] + 1) + 1 if primary_slit == 'S200A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 1) s2a2 = Slit('S200A2', 1, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200A2' else 10*slit_nums[primary_slit] + 2) + 1 if primary_slit == 'S200A2' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 2) s4a1 = Slit('S400A1', 2, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S400A1' else 10*slit_nums[primary_slit] + 3) + 1 if primary_slit == 'S400A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 3) s16a1 = Slit('S1600A1', 3, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S1600A1' else 10*slit_nums[primary_slit] + 4) + 1 if primary_slit == 'S1600A1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 4) s2b1 = Slit('S200B1', 4, 0, 0, 0, ylow, yhigh, 5, - 1 if primary_slit=='S200B1' else 10*slit_nums[primary_slit] + 5) + 1 if primary_slit == 'S200B1' + else 10 * FIXED_SLIT_NUMS[primary_slit] + 5) # Decide which slits need to be added to this exposure subarray = input_model.meta.subarray.name.upper() @@ -597,37 +603,127 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, log.info(f'Retrieving open MSA slitlets for msa_metadata_id = {msa_metadata_id} ' f'and dither_index = {dither_position}') - # Get the unique slitlet_ids - slitlet_ids_unique = list(set([x['slitlet_id'] for x in msa_data])) + # Sort the MSA rows by slitlet_id + slitlet_sets = {} + for row in msa_data: + # Check for fixed slit: if set, then slitlet_id is null + is_fs = False + try: + fixed_slit = row['fixed_slit'] + if (fixed_slit in FIXED_SLIT_NUMS.keys() + and fixed_slit != 'NONE'): + is_fs = True + except (IndexError, ValueError, KeyError): + # May be old-style MSA file without a fixed_slit column + fixed_slit = None + + if is_fs: + # Fixed slit - use the slit name as the ID + slitlet_id = fixed_slit + else: + # MSA - use the slitlet ID + slitlet_id = row['slitlet_id'] + + # Append the row for the slitlet + if slitlet_id in slitlet_sets: + slitlet_sets[slitlet_id].append(row) + else: + slitlet_sets[slitlet_id] = [row] - # add a margin to the slit y limits + # Add a margin to the slit y limits margin = 0.5 - # Now lets look at each unique slitlet id - for slitlet_id in slitlet_ids_unique: - # Get the rows of shutter info for the current slitlet_id - slitlets_sid = [x for x in msa_data if x['slitlet_id'] == slitlet_id] - open_shutters = [x['shutter_column'] for x in slitlets_sid] + # Now let's look at each unique slitlet id + for slitlet_id, slitlet_rows in slitlet_sets.items(): + # Get the open shutter information from the slitlet rows + open_shutters = [x['shutter_column'] for x in slitlet_rows] # How many shutters in the slitlet are labeled as "main" or "primary"? - n_main_shutter = len([s for s in slitlets_sid if s['primary_source'] == 'Y']) + n_main_shutter = len([s for s in slitlet_rows if s['primary_source'] == 'Y']) + + # Check for fixed slit sources defined in the MSA file + is_fs = [False] * len(slitlet_rows) + for i, slitlet in enumerate(slitlet_rows): + try: + if (slitlet['fixed_slit'] in FIXED_SLIT_NUMS.keys() + and slitlet['fixed_slit'] != 'NONE'): + is_fs[i] = True + except (IndexError, ValueError, KeyError): + # May be old-style MSA file without a fixed_slit column + pass # In the next part we need to calculate, find, or determine 5 things for each slit: # quadrant, xcen, ycen, ymin, ymax - # There are no main shutters: all are background - if n_main_shutter == 0: + # First, check for a fixed slit + if all(is_fs) and len(slitlet_rows) == 1: + # One fixed slit open for the source + slitlet = slitlet_rows[0] + + # Use a standard number for fixed slit shutter id + shutter_id = FIXED_SLIT_NUMS[slitlet_id] - 1 + xcen = ycen = 0 + quadrant = 5 + + # No additional margin for fixed slit bounding boxes + ymin = ylow + ymax = yhigh + + # Source position and id + if n_main_shutter == 1: + # Source is marked primary + source_id = slitlet['source_id'] + source_xpos = slitlet['estimated_source_in_shutter_x'] + source_ypos = slitlet['estimated_source_in_shutter_y'] + log.info(f'Found fixed slit {slitlet_id} with source_id = {source_id}.') + + # Get source info for this slitlet: + # note that slits with a real source assigned have source_id > 0, + # while slits with source_id < 0 contain "virtual" sources + try: + source_name, source_alias, stellarity, source_ra, source_dec = [ + (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec']) + for s in msa_source if s['source_id'] == source_id][0] + except IndexError: + # Missing source information: assign a virtual source name + log.warning("Could not retrieve source info from MSA file") + source_name = f"{prog_id}_VRT{slitlet_id}" + source_alias = "VRT{}".format(slitlet_id) + stellarity = 0.0 + source_ra = 0.0 + source_dec = 0.0 + + else: + log.warning(f'Fixed slit {slitlet_id} is not a primary source; ' + f'skipping it.') + continue + + elif any(is_fs): + # Unsupported fixed slit configuration + message = ("For slitlet_id = {}, metadata_id = {}, " + "dither_index = {}".format( + slitlet_id, msa_metadata_id, dither_position)) + log.warning(message) + message = ("MSA configuration file has an unsupported " + "fixed slit configuration.") + log.warning(message) + msa_file.close() + raise MSAFileError(message) + + # Now check for regular MSA slitlets + elif n_main_shutter == 0: + # There are no main shutters: all are background if len(open_shutters) == 1: jmin = jmax = j = open_shutters[0] else: - jmin = min([s['shutter_column'] for s in slitlets_sid]) - jmax = max([s['shutter_column'] for s in slitlets_sid]) + jmin = min([s['shutter_column'] for s in slitlet_rows]) + jmax = max([s['shutter_column'] for s in slitlet_rows]) j = jmin + (jmax - jmin) // 2 ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 - quadrant = slitlets_sid[0]['shutter_quadrant'] + quadrant = slitlet_rows[0]['shutter_quadrant'] ycen = j - xcen = slitlets_sid[0]['shutter_row'] # grab the first as they are all the same + xcen = slitlet_rows[0]['shutter_row'] # grab the first as they are all the same shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # Background slits all have source_id=0 in the msa_file, @@ -651,20 +747,21 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, (s['shutter_row'], s['shutter_column'], s['shutter_quadrant'], s['estimated_source_in_shutter_x'], s['estimated_source_in_shutter_y']) - for s in slitlets_sid if s['background'] == 'N'][0] + for s in slitlet_rows if s['background'] == 'N'][0] shutter_id = xcen + (ycen - 1) * 365 # shutter numbers in MSA file are 1-indexed # y-size - jmin = min([s['shutter_column'] for s in slitlets_sid]) - jmax = max([s['shutter_column'] for s in slitlets_sid]) + jmin = min([s['shutter_column'] for s in slitlet_rows]) + jmax = max([s['shutter_column'] for s in slitlet_rows]) j = ycen ymax = yhigh + margin + (jmax - j) * 1.15 ymin = -(-ylow + margin) + (jmin - j) * 1.15 # Get the source_id from the primary shutter entry - for i in range(len(slitlets_sid)): - if slitlets_sid[i]['primary_source'] == 'Y': - source_id = slitlets_sid[i]['source_id'] + source_id = None + for i in range(len(slitlet_rows)): + if slitlet_rows[i]['primary_source'] == 'Y': + source_id = slitlet_rows[i]['source_id'] # Get source info for this slitlet; # note that slits with a real source assigned have source_id > 0, @@ -683,14 +780,15 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, f"assigning virtual source_name={source_name}") if source_id < 0: - log.info(f'Slitlet {slitlet_id} contains virtual source, with source_id={source_id}') + log.info(f'Slitlet {slitlet_id} contains virtual source, ' + f'with source_id={source_id}') # More than 1 main shutter: Not allowed! else: message = ("For slitlet_id = {}, metadata_id = {}, " "and dither_index = {}".format(slitlet_id, msa_metadata_id, dither_position)) log.warning(message) - message = ("MSA configuration file has more than 1 shutter with primary source") + message = "MSA configuration file has more than 1 shutter with primary source" log.warning(message) msa_file.close() raise MSAFileError(message) @@ -711,9 +809,12 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position, # Create the shutter_state string all_shutters = _shutter_id_to_str(open_shutters, ycen) - slitlets.append(Slit(slitlet_id, shutter_id, dither_position, xcen, ycen, ymin, ymax, - quadrant, source_id, all_shutters, source_name, source_alias, - stellarity, source_xpos, source_ypos, source_ra, source_dec)) + slit_parameters = (slitlet_id, shutter_id, dither_position, xcen, ycen, ymin, ymax, + quadrant, source_id, all_shutters, source_name, source_alias, + stellarity, source_xpos, source_ypos, source_ra, source_dec) + log.debug(f'Appending slit: {slit_parameters}') + slitlets.append(Slit(*slit_parameters)) + msa_file.close() return slitlets diff --git a/jwst/assign_wcs/tests/data/msa_fs_configuration.fits b/jwst/assign_wcs/tests/data/msa_fs_configuration.fits new file mode 100644 index 0000000000..5a1f5b6e55 Binary files /dev/null and b/jwst/assign_wcs/tests/data/msa_fs_configuration.fits differ diff --git a/jwst/assign_wcs/tests/test_nirspec.py b/jwst/assign_wcs/tests/test_nirspec.py index 24a78bfab0..3c6d3591f5 100644 --- a/jwst/assign_wcs/tests/test_nirspec.py +++ b/jwst/assign_wcs/tests/test_nirspec.py @@ -2,27 +2,25 @@ Test functions for NIRSPEC WCS - all modes. """ import functools -from math import cos, sin import os.path import shutil +from math import cos, sin -import pytest +import astropy.units as u +import astropy.coordinates as coords import numpy as np -from numpy.testing import assert_allclose +import pytest from astropy.io import fits from astropy.modeling import models as astmodels from astropy import table from astropy import wcs as astwcs -import astropy.units as u -import astropy.coordinates as coords -from gwcs import wcs -from gwcs import wcstools +from gwcs import wcs, wcstools +from numpy.testing import assert_allclose from stdatamodels.jwst import datamodels from stdatamodels.jwst.transforms import models as trmodels -from jwst.assign_wcs import nirspec -from jwst.assign_wcs import assign_wcs_step +from jwst.assign_wcs import nirspec, assign_wcs_step from jwst.assign_wcs.tests import data from jwst.assign_wcs.util import MSAFileError, in_ifu_slice @@ -356,13 +354,69 @@ def test_msa_configuration_multiple_returns(): _compare_slits(slitlet_info[1], ref_slit2) +def test_msa_fs_configuration(): + """ + Test the get_open_msa_slits function with FS and MSA slits defined. + """ + prog_id = '1234' + msa_meta_id = 12 + msaconfl = get_file_path('msa_fs_configuration.fits') + dither_position = 1 + slitlet_info = nirspec.get_open_msa_slits( + prog_id, msaconfl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) + + # MSA slit: reads in as normal + ref_slit = trmodels.Slit(55, 9376, 1, 251, 26, -5.6, 1.0, 4, 1, '1111x', '95065_1', '2122', + 0.13, -0.31716078999999997, -0.18092266) + _compare_slits(slitlet_info[0], ref_slit) + + # FS primary: S200A1, shutter id 0, quadrant 5 + ref_slit = trmodels.Slit('S200A1', 0, 1, 0, 0, -0.5, 0.5, 5, 3, 'x', '95065_3', '3', + 1.0, -0.161, -0.229, 53.139904, -27.805002) + _compare_slits(slitlet_info[1], ref_slit) + + # The remaining fixed slits may be in the MSA file but not primary: + # they should not be defined. + fs_slits_defined = ['S200A1'] + n_fixed = 0 + for slit in slitlet_info: + if slit.quadrant == 5: + assert slit.name in fs_slits_defined + n_fixed += 1 + assert n_fixed == len(fs_slits_defined) + + +def test_msa_fs_configuration_unsupported(tmp_path): + """ + Test the get_open_msa_slits function with unsupported FS defined. + """ + # modify an existing MSA file to add a bad row + msaconfl = get_file_path('msa_fs_configuration.fits') + bad_confl = str(tmp_path / 'bad_msa_fs_configuration.fits') + shutil.copy(msaconfl, bad_confl) + + with fits.open(bad_confl) as msa_hdu_list: + shutter_table = table.Table(msa_hdu_list['SHUTTER_INFO'].data) + shutter_table.add_row(shutter_table[-1]) + msa_hdu_list['SHUTTER_INFO'] = fits.table_to_hdu(shutter_table) + msa_hdu_list[2].name = 'SHUTTER_INFO' + msa_hdu_list.writeto(bad_confl, overwrite=True) + + prog_id = '1234' + msa_meta_id = 12 + dither_position = 1 + with pytest.raises(MSAFileError, match='unsupported fixed slit'): + nirspec.get_open_msa_slits( + prog_id, bad_confl, msa_meta_id, dither_position, slit_y_range=[-.5, .5]) + + def test_msa_missing_source(tmp_path): """ Test the get_open_msa_slits function with missing source information. """ # modify an existing MSA file to remove source info - msaconfl = get_file_path('msa_configuration.fits') - bad_confl = str(tmp_path / 'bad_msa_configuration.fits') + msaconfl = get_file_path('msa_fs_configuration.fits') + bad_confl = str(tmp_path / 'bad_msa_fs_configuration.fits') shutil.copy(msaconfl, bad_confl) with fits.open(bad_confl) as msa_hdu_list: @@ -385,6 +439,13 @@ def test_msa_missing_source(tmp_path): -0.31716078999999997, -0.18092266, 0.0, 0.0) _compare_slits(slitlet_info[0], ref_slit) + # FS primary: S200A1, virtual source name assigned + ref_slit = trmodels.Slit('S200A1', 0, 1, 0, 0, -0.5, 0.5, 5, 3, 'x', + '1234_VRTS200A1', 'VRTS200A1', 0.0, + -0.161, -0.229, 0.0, 0.0) + _compare_slits(slitlet_info[1], ref_slit) + + open_shutters = [[24], [23, 24], [22, 23, 25, 27], [22, 23, 25, 27, 28]] main_shutter = [24, 23, 25, 28] result = ["x", "x1", "110x01", "110101x"] diff --git a/jwst/exp_to_source/exp_to_source.py b/jwst/exp_to_source/exp_to_source.py index baaaa1ce20..c0f385a339 100644 --- a/jwst/exp_to_source/exp_to_source.py +++ b/jwst/exp_to_source/exp_to_source.py @@ -47,6 +47,7 @@ def exp_to_source(inputs): log.debug(f'Copying source {key}') result_slit = result[str(key)] result_slit.exposures.append(slit) + # store values for later use (after merge_tree) # these values are incorrectly getting overwritten by # the top model. @@ -54,12 +55,14 @@ def exp_to_source(inputs): slit_bunit_err = slit.meta.bunit_err slit_model = slit.meta.model_type slit_wcsinfo = slit.meta.wcsinfo.instance - # exposure.meta.bunit_data and bunit_err does not exist - # before calling merge_tree save these values + slit_exptype = None + if hasattr(slit.meta, 'exposure'): + if hasattr(slit.meta.exposure, 'type'): + slit_exptype = slit.meta.exposure.type + # Before merge_tree the slits have a model_type of SlitModel. # After merge_tree it is overwritten with MultiSlitModel. # store the model type to undo overwriting of modeltype. - merge_tree(result_slit.exposures[-1].meta.instance, exposure.meta.instance) result_slit.exposures[-1].meta.bunit_data = slit_bunit @@ -67,9 +70,16 @@ def exp_to_source(inputs): result_slit.exposures[-1].meta.model_type = slit_model result_slit.exposures[-1].meta.wcsinfo = slit_wcsinfo + # make sure top-level exposure type matches slit exposure type + # (necessary for NIRSpec fixed slits defined as part of an MSA file) + if slit_exptype is not None: + result_slit.exposures[-1].meta.exposure.type = slit_exptype + result_slit.meta.exposure.type = slit_exptype + log.debug(f'Input exposure type: {exposure.meta.exposure.type}') + log.debug(f'Output exposure type: {result_slit.meta.exposure.type}') + if result_slit.meta.instrument.name is None: result_slit.update(exposure) - result_slit.meta.filename = None # Resulting merged data doesn't come from one file exposure.close() diff --git a/jwst/exp_to_source/tests/test_exp_to_source.py b/jwst/exp_to_source/tests/test_exp_to_source.py index 0216140027..72f84a5df9 100644 --- a/jwst/exp_to_source/tests/test_exp_to_source.py +++ b/jwst/exp_to_source/tests/test_exp_to_source.py @@ -75,3 +75,43 @@ def test_container_structure(): container.close() for model in inputs: model.close() + for model in outputs.values(): + model.close() + + +def test_slit_exptype(): + """Test for slit exposure type handling.""" + + # Setup input + inputs = [MultiSlitModel(f) for f in helpers.INPUT_FILES] + container = ModelContainer(inputs) + + # Add a slit exposure type to each input + for model in container: + for slit in model.slits: + if slit.source_id == 1: + slit.meta.exposure = {'type': 'NRS_MSASPEC'} + else: + slit.meta.exposure = {'type': 'NRS_FIXEDSLIT'} + + # Make the source-based containers + outputs = multislit_to_container(container) + + # Check that exposure type was passed from input to output + assert len(container) == 3 + assert len(outputs) == 5 + for i, model in enumerate(container): + for slit in model.slits: + exposure = outputs[str(slit.source_id)][i] + assert exposure.meta.exposure.type == slit.meta.exposure.type + if slit.source_id == 1: + assert exposure.meta.exposure.type == 'NRS_MSASPEC' + else: + assert exposure.meta.exposure.type == 'NRS_FIXEDSLIT' + + # Closeout + container.close() + for model in inputs: + model.close() + for model in outputs.values(): + model.close() diff --git a/jwst/extract_1d/extract.py b/jwst/extract_1d/extract.py index c9529fb0c6..32442c8f83 100644 --- a/jwst/extract_1d/extract.py +++ b/jwst/extract_1d/extract.py @@ -3736,13 +3736,6 @@ def create_extraction(extract_ref_dict, use_source_posn = False log.info(f"Setting use_source_posn to False for source type {source_type}") - # Turn off use_source_posn if working on non-primary NRS fixed slits - if is_multiple_slits: - if exp_type == 'NRS_FIXEDSLIT' and slitname != slit.meta.instrument.fixed_slit: - use_source_posn = False - log.info("Can only compute source location for primary NIRSpec slit,") - log.info("so setting use_source_posn to False") - if photom_has_been_run: pixel_solid_angle = meta_source.meta.photometry.pixelarea_steradians if pixel_solid_angle is None: diff --git a/jwst/extract_2d/nirspec.py b/jwst/extract_2d/nirspec.py index 571ad42563..c29ab427c2 100644 --- a/jwst/extract_2d/nirspec.py +++ b/jwst/extract_2d/nirspec.py @@ -165,7 +165,12 @@ def set_slit_attributes(output_model, slit, xlo, xhi, ylo, yhi): output_model.stellarity = float(slit.stellarity) output_model.source_xpos = float(slit.source_xpos) output_model.source_ypos = float(slit.source_ypos) - output_model.slitlet_id = int(slit.name) + try: + output_model.slitlet_id = int(slit.name) + except ValueError: + # Fixed slits in MOS data have string values for the name; + # use the shutter ID instead + output_model.slitlet_id = slit.shutter_id output_model.quadrant = int(slit.quadrant) output_model.xcen = int(slit.xcen) output_model.ycen = int(slit.ycen) diff --git a/jwst/extract_2d/tests/test_nirspec.py b/jwst/extract_2d/tests/test_nirspec.py new file mode 100644 index 0000000000..ab6c2747de --- /dev/null +++ b/jwst/extract_2d/tests/test_nirspec.py @@ -0,0 +1,214 @@ +import numpy as np +import pytest +from astropy.io import fits +from astropy.table import Table +from stdatamodels.jwst.datamodels import ImageModel, CubeModel, MultiSlitModel, SlitModel + +from jwst.assign_wcs import AssignWcsStep +from jwst.extract_2d.extract_2d_step import Extract2dStep + + +# WCS keywords, borrowed from NIRCam grism tests +WCS_KEYS = {'wcsaxes': 2, 'ra_ref': 53.1490299775, 'dec_ref': -27.8168745624, + 'v2_ref': 86.103458, 'v3_ref': -493.227512, 'roll_ref': 45.04234459270135, + 'crpix1': 1024.5, 'crpix2': 1024.5, + 'crval1': 53.1490299775, 'crval2': -27.8168745624, + 'cdelt1': 1.81661111111111e-05, 'cdelt2': 1.8303611111111e-05, + 'ctype1': 'RA---TAN', 'ctype2': 'DEC--TAN', + 'pc1_1': -0.707688557183348, 'pc1_2': 0.7065245261360363, + 'pc2_1': 0.7065245261360363, 'pc2_2': 1.75306861111111e-05, + 'cunit1': 'deg', 'cunit2': 'deg'} + + +def create_nirspec_hdul(detector='NRS1', grating='G395M', filter_name='F290LP', + exptype='NRS_MSASPEC', subarray='FULL', slit=None, nint=1, + wcskeys=None): + if wcskeys is None: + wcskeys = WCS_KEYS.copy() + + hdul = fits.HDUList() + phdu = fits.PrimaryHDU() + phdu.header['TELESCOP'] = 'JWST' + phdu.header['INSTRUME'] = 'NIRSPEC' + phdu.header['DETECTOR'] = detector + phdu.header['FILTER'] = filter_name + phdu.header['GRATING'] = grating + phdu.header['PROGRAM'] = '01234' + phdu.header['TIME-OBS'] = '8:59:37' + phdu.header['DATE-OBS'] = '2023-01-05' + phdu.header['EXP_TYPE'] = exptype + phdu.header['PATT_NUM'] = 1 + phdu.header['SUBARRAY'] = subarray + if subarray == 'SUBS200A1': + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 64 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 1041 + elif subarray == 'SUB2048': + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 32 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 946 + else: + phdu.header['SUBSIZE1'] = 2048 + phdu.header['SUBSIZE2'] = 2048 + phdu.header['SUBSTRT1'] = 1 + phdu.header['SUBSTRT2'] = 1 + + if exptype == 'NRS_MSASPEC': + phdu.header['MSAMETID'] = 1 + phdu.header['MSAMETFL'] = 'test_msa_01.fits' + + if slit is not None: + phdu.header['FXD_SLIT'] = slit + phdu.header['APERNAME'] = f'NRS_{slit}_SLIT' + + scihdu = fits.ImageHDU() + scihdu.header['EXTNAME'] = "SCI" + scihdu.header.update(wcskeys) + if nint > 1: + scihdu.data = np.ones((nint, phdu.header['SUBSIZE2'], phdu.header['SUBSIZE1'])) + else: + scihdu.data = np.ones((phdu.header['SUBSIZE2'], phdu.header['SUBSIZE1'])) + + hdul.append(phdu) + hdul.append(scihdu) + return hdul + + +def create_msa_hdul(): + # Two point sources, one in MSA, one fixed slit. + # Source locations for the fixed slit are placeholders, not realistic. + shutter_data = { + 'slitlet_id': [12, 12, 12, 100], + 'msa_metadata_id': [1, 1, 1, 1], + 'shutter_quadrant': [4, 4, 4, 0], + 'shutter_row': [251, 251, 251, 0], + 'shutter_column': [22, 23, 24, 0], + 'source_id': [1, 1, 1, 2], + 'background': ['Y', 'N', 'Y', 'N'], + 'shutter_state': ['OPEN', 'OPEN', 'OPEN', 'OPEN'], + 'estimated_source_in_shutter_x': [np.nan, 0.18283921, np.nan, 0.5], + 'estimated_source_in_shutter_y': [np.nan, 0.31907734, np.nan, 0.5], + 'dither_point_index': [1, 1, 1, 1], + 'primary_source': ['N', 'Y', 'N', 'Y'], + 'fixed_slit': ['NONE', 'NONE', 'NONE', 'S200A1']} + + source_data = { + 'program': [95065, 95065], + 'source_id': [1, 2], + 'source_name': ['95065_1', '95065_2'], + 'alias': ['2122', '2123'], + 'ra': [53.139904, 53.15], + 'dec': [-27.805002, -27.81], + 'preimage_id': ['95065001_000', '95065001_000'], + 'stellarity': [1.0, 1.0]} + + shutter_table = Table(shutter_data) + source_table = Table(source_data) + + hdul = fits.HDUList() + hdul.append(fits.PrimaryHDU()) + hdul.append(fits.ImageHDU()) + hdul.append(fits.table_to_hdu(shutter_table)) + hdul.append(fits.table_to_hdu(source_table)) + hdul[2].name = 'SHUTTER_INFO' + hdul[3].name = 'SOURCE_INFO' + + return hdul + + +@pytest.fixture +def nirspec_msa_rate(tmp_path): + hdul = create_nirspec_hdul() + hdul[0].header['MSAMETFL'] = str(tmp_path / 'test_msa_01.fits') + filename = str(tmp_path / 'test_nrs_msa_rate.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_fs_rate(tmp_path): + hdul = create_nirspec_hdul( + exptype='NRS_FIXEDSLIT', subarray='SUBS200A1', slit='S200A1') + filename = str(tmp_path / 'test_nrs_fs_rate.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_bots_rateints(tmp_path): + hdul = create_nirspec_hdul( + exptype='NRS_BRIGHTOBJ', subarray='SUB2048', slit='S1600A1', nint=3) + filename = str(tmp_path / 'test_nrs_bots_rateints.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +@pytest.fixture +def nirspec_msa_metfl(tmp_path): + hdul = create_msa_hdul() + filename = str(tmp_path / 'test_msa_01.fits') + hdul.writeto(filename, overwrite=True) + hdul.close() + return filename + + +def test_extract_2d_nirspec_msa_fs(nirspec_msa_rate, nirspec_msa_metfl): + model = ImageModel(nirspec_msa_rate) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + assert isinstance(result, MultiSlitModel) + + # there should be 2 slits extracted: one MSA, one FS + assert len(result.slits) == 2 + + # the MSA slit has an integer name, slitlet_id matches name + assert result.slits[0].name == '12' + assert result.slits[0].slitlet_id == 12 + assert result.slits[0].data.shape == (31, 1355) + + # the FS slit has a string name, slitlet_id matches shutter ID + assert result.slits[1].name == 'S200A1' + assert result.slits[1].slitlet_id == 0 + assert result.slits[1].data.shape == (45, 1254) + + model.close() + result.close() + + +def test_extract_2d_nirspec_fs(nirspec_fs_rate): + model = ImageModel(nirspec_fs_rate) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + assert isinstance(result, MultiSlitModel) + + # there should be 1 slit extracted: FS, S200A1 + assert len(result.slits) == 1 + + # the FS slit has a string name, slitlet_id matches shutter ID + assert result.slits[0].name == 'S200A1' + assert result.slits[0].slitlet_id == 0 + assert result.slits[0].data.shape == (45, 1254) + + model.close() + result.close() + + +def test_extract_2d_nirspec_bots(nirspec_bots_rateints): + model = CubeModel(nirspec_bots_rateints) + result = AssignWcsStep.call(model) + result = Extract2dStep.call(result) + + # output is a single slit + assert isinstance(result, SlitModel) + + # the BOTS slit has a string name, slitlet_id matches shutter ID + assert result.name == 'S1600A1' + assert result.data.shape == (3, 28, 1300) + + model.close() + result.close() diff --git a/jwst/flatfield/flat_field.py b/jwst/flatfield/flat_field.py index 01da31668d..0b653d8d29 100644 --- a/jwst/flatfield/flat_field.py +++ b/jwst/flatfield/flat_field.py @@ -354,7 +354,6 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa """ exposure_type = output_model.meta.exposure.type - primary_slit = output_model.meta.instrument.fixed_slit # Create a list to hold the list of slits. This will eventually be used # to extend the MultiSlitModel.slits attribute. We do it this way to @@ -375,12 +374,12 @@ def nirspec_fs_msa(output_model, f_flat_model, s_flat_model, d_flat_model, dispa if user_supplied_flat is not None: slit_flat = user_supplied_flat.slits[slit_idx] else: - if exposure_type == "NRS_FIXEDSLIT" and \ - slit.name == primary_slit and slit.source_type.upper() == "POINT": + if (exposure_type == "NRS_FIXEDSLIT" + and slit.source_type.upper() == "POINT"): - # For fixed-slit exposures, if this is the primary slit - # and it contains a point source, compute the flat-field - # corrections for both uniform (without wavecorr) and point + # For fixed-slit exposures, if this contains a point source, + # compute the flat-field corrections for both uniform + # (without wavecorr) and point # source (with wavecorr) modes, applying only the point # source version to the data. diff --git a/jwst/flatfield/tests/test_flatfield.py b/jwst/flatfield/tests/test_flatfield.py index 8b0176ae89..a332632bf2 100644 --- a/jwst/flatfield/tests/test_flatfield.py +++ b/jwst/flatfield/tests/test_flatfield.py @@ -1,8 +1,12 @@ import pytest import numpy as np +from astropy.io import fits +from astropy.table import Table from stdatamodels.jwst import datamodels +from jwst.assign_wcs import AssignWcsStep +from jwst.assign_wcs.tests.test_nirspec import create_nirspec_ifu_file from jwst.flatfield import FlatFieldStep from jwst.flatfield.flat_field_step import NRS_IMAGING_MODES, NRS_SPEC_MODES @@ -92,3 +96,285 @@ def test_nirspec_flatfield_step_interface(exptype): data.meta.subarray.ysize = shape[0] FlatFieldStep.call(data) + + +def create_nirspec_flats(shape, msa=False): + flats = [] + for flat_name in ['f', 's', 'd']: + if flat_name == 'f': + f_data = np.full(shape[0], 1.0) + f_err = np.full(shape[0], 0.1) + else: + f_data = np.full(shape[0], 1.0) + f_err = np.full(shape[0], np.nan) + + # make a fast variation table + flat_table = Table({ + 'slit_name': ['ANY'], + 'nelem': [shape[0]], + 'wavelength': [np.arange(1, shape[0] + 1, dtype=float)], + 'data': [f_data], + 'error': [f_err]}) + + fflat_hdul = fits.HDUList([fits.PrimaryHDU()]) + if flat_name == 's' and not msa: + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape[-2:], 0.0), name='ERR')) + + elif flat_name == 's' and msa: + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full((5, shape[1], shape[2]), 0.0), name='ERR')) + + fflat_hdul.append(fits.table_to_hdu( + Table({'wavelength': np.arange(1, shape[0] + 1)}))) + fflat_hdul[-1].name = 'WAVELENGTH' + + elif flat_name == 'd': + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 1.0), name='SCI')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0), name='DQ')) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0.0), name='ERR')) + fflat_hdul.append(fits.table_to_hdu( + Table({'wavelength': np.arange(1, shape[0] + 1)}))) + fflat_hdul[-1].name = 'WAVELENGTH' + + if flat_name == 'f' and msa: + for quadrant in range(4): + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 1.0), + name='SCI', ver=(quadrant + 1))) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0), + name='DQ', ver=(quadrant + 1))) + fflat_hdul.append(fits.ImageHDU(data=np.full(shape, 0.0), + name='ERR', ver=(quadrant + 1))) + hdu = fits.table_to_hdu( + Table({'wavelength': [np.arange(1, shape[0] + 1)]})) + hdu.header['EXTNAME'] = 'WAVELENGTH' + hdu.header['EXTVER'] = quadrant + 1 + fflat_hdul.append(hdu) + + hdu = fits.table_to_hdu(flat_table) + hdu.header['EXTNAME'] = 'FAST_VARIATION' + hdu.header['EXTVER'] = quadrant + 1 + fflat_hdul.append(hdu) + + else: + fflat_hdul.append(fits.table_to_hdu(flat_table)) + fflat_hdul[-1].name = 'FAST_VARIATION' + + if msa and flat_name == 'f': + flat = datamodels.NirspecQuadFlatModel(fflat_hdul) + else: + flat = datamodels.NirspecFlatModel(fflat_hdul) + fflat_hdul.close() + + flat.meta.instrument.name = 'NIRSPEC' + flat.meta.subarray.xstart = 1 + flat.meta.subarray.ystart = 1 + flat.meta.subarray.xsize = shape[1] + flat.meta.subarray.ysize = shape[0] + + flats.append(flat) + + return flats + + +def test_nirspec_bots_flat(): + """Test that the interface works for NIRSpec BOTS data""" + shape = (3, 20, 20) + w_shape = (10, 20, 20) + + data = datamodels.SlitModel(shape) + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_BRIGHTOBJ' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + data.xstart = 1 + data.ystart = 1 + data.xsize = shape[1] + data.ysize = shape[0] + data.data += 1 + data.wavelength = np.ones(shape[-2:]) + data.wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.data) + assert np.allclose(result.data[nn], data.data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result, ext) + assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.var_flat.shape == shape + assert np.allclose(result.var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() + + +@pytest.mark.parametrize('srctype', ['POINT', 'EXTENDED']) +def test_nirspec_fs_flat(srctype): + """Test that the interface works for NIRSpec FS data.""" + shape = (20, 20) + w_shape = (10, 20, 20) + + data = datamodels.MultiSlitModel() + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_FIXEDSLIT' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + + data.slits.append(datamodels.SlitModel(shape)) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].dq = np.full(shape, 0) + data.slits[0].err = np.full(shape, 0.0) + data.slits[0].var_poisson = np.full(shape, 0.0) + data.slits[0].var_rnoise = np.full(shape, 0.0) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].wavelength = np.ones(shape[-2:]) + data.slits[0].wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + data.slits[0].source_type = srctype + data.slits[0].name = 'S200A1' + data.slits[0].xstart = 1 + data.slits[0].ystart = 1 + data.slits[0].xsize = shape[1] + data.slits[0].ysize = shape[0] + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.slits[0].data) + assert np.allclose(result.slits[0].data[nn], data.slits[0].data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result.slits[0], ext) + assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.slits[0].dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.slits[0].var_flat.shape == shape + assert np.allclose(result.slits[0].var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() + + +def test_nirspec_msa_flat(): + """Test that the interface works for NIRSpec MSA data.""" + shape = (20, 20) + w_shape = (10, 20, 20) + + data = datamodels.MultiSlitModel() + data.meta.instrument.name = 'NIRSPEC' + data.meta.exposure.type = 'NRS_MSASPEC' + data.meta.subarray.xstart = 1 + data.meta.subarray.ystart = 1 + data.meta.subarray.xsize = shape[1] + data.meta.subarray.ysize = shape[0] + + data.slits.append(datamodels.SlitModel(shape)) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].dq = np.full(shape, 0) + data.slits[0].err = np.full(shape, 0.0) + data.slits[0].var_poisson = np.full(shape, 0.0) + data.slits[0].var_rnoise = np.full(shape, 0.0) + data.slits[0].data = np.full(shape, 1.0) + data.slits[0].wavelength = np.ones(shape[-2:]) + data.slits[0].wavelength[:] = np.linspace(1, 5, shape[-1], dtype=float) + data.slits[0].source_type = 'UNKNOWN' + data.slits[0].name = '11' + data.slits[0].quadrant = 1 + data.slits[0].xcen = 10 + data.slits[0].ycen = 10 + data.slits[0].xstart = 1 + data.slits[0].ystart = 1 + data.slits[0].xsize = shape[1] + data.slits[0].ysize = shape[0] + + flats = create_nirspec_flats(w_shape, msa=True) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.slits[0].data) + assert np.allclose(result.slits[0].data[nn], data.slits[0].data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result.slits[0], ext) + assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.slits[0].dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.slits[0].var_flat.shape == shape + assert np.allclose(result.slits[0].var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() + + +@pytest.mark.slow +def test_nirspec_ifu_flat(): + """Test that the interface works for NIRSpec IFU data. + + Larger data and more WCS operations required for testing make + this test take more than a minute, so marking this test 'slow'. + """ + shape = (2048, 2048) + w_shape = (10, 2048, 2048) + + # IFU mode requires WCS information, so make a more realistic model + hdul = create_nirspec_ifu_file(grating='PRISM', filter='CLEAR', + gwa_xtil=0.35986012, gwa_ytil=0.13448857, + gwa_tilt=37.1) + hdul['SCI'].data = np.ones(shape, dtype=float) + + data = datamodels.IFUImageModel(hdul) + data = AssignWcsStep.call(data) + + flats = create_nirspec_flats(w_shape) + result = FlatFieldStep.call(data, override_fflat=flats[0], override_sflat=flats[1], + override_dflat=flats[2], override_flat='N/A') + + # null flat, so data is the same, other than nan edges + nn = ~np.isnan(result.data) + assert np.allclose(result.data[nn], data.data[nn]) + + # check that NaNs match in every extension they should + for ext in ['data', 'err', 'var_rnoise', 'var_poisson', 'var_flat']: + test_data = getattr(result, ext) + assert np.all(np.isnan(test_data[~nn])) + assert np.all(result.dq[~nn] | datamodels.dqflags.pixel['DO_NOT_USE']) + + # error is propagated from non-empty fflat error + # (no other additive contribution, scale from data is 1.0) + assert result.var_flat.shape == shape + assert np.allclose(result.var_flat[nn], 0.1 ** 2) + assert result.meta.cal_step.flat_field == 'COMPLETE' + + result.close() + for flat in flats: + flat.close() diff --git a/jwst/lib/wcs_utils.py b/jwst/lib/wcs_utils.py index 3070756f13..bff72708b1 100644 --- a/jwst/lib/wcs_utils.py +++ b/jwst/lib/wcs_utils.py @@ -41,7 +41,6 @@ def get_wavelengths(model, exp_type="", order=None, use_wavecorr=None): got_wavelength = False wl_array = None - # Evaluate the WCS on the grid of pixel indexes, capturing only the # resulting wavelength values shape = model.data.shape diff --git a/jwst/master_background/expand_to_2d.py b/jwst/master_background/expand_to_2d.py index 34c5871447..de221eab9a 100644 --- a/jwst/master_background/expand_to_2d.py +++ b/jwst/master_background/expand_to_2d.py @@ -18,7 +18,7 @@ WFSS_EXPTYPES = ['NIS_WFSS', 'NRC_WFSS', 'NRC_GRISM', 'NRC_TSGRISM'] -def expand_to_2d(input, m_bkg_spec): +def expand_to_2d(input, m_bkg_spec, allow_mos=False): """Expand a 1-D background to 2-D. Parameters @@ -30,6 +30,14 @@ def expand_to_2d(input, m_bkg_spec): Either the name of a file containing a 1-D background spectrum, or a data model containing such a spectrum. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.JwstDataModel` @@ -56,15 +64,17 @@ def expand_to_2d(input, m_bkg_spec): # Handle associations, or input ModelContainers if isinstance(input, ModelContainer): - background = bkg_for_container(input, tab_wavelength, tab_background) + background = bkg_for_container(input, tab_wavelength, tab_background, + allow_mos=allow_mos) else: - background = create_bkg(input, tab_wavelength, tab_background) + background = create_bkg(input, tab_wavelength, tab_background, + allow_mos=allow_mos) return background -def bkg_for_container(input, tab_wavelength, tab_background): +def bkg_for_container(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background for a container object. Parameters @@ -78,6 +88,14 @@ def bkg_for_container(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.ModelContainer` @@ -87,13 +105,14 @@ def bkg_for_container(input, tab_wavelength, tab_background): background = ModelContainer() for input_model in input: - temp = create_bkg(input_model, tab_wavelength, tab_background) + temp = create_bkg(input_model, tab_wavelength, tab_background, + allow_mos=allow_mos) background.append(temp) return background -def create_bkg(input, tab_wavelength, tab_background): +def create_bkg(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background. Parameters @@ -107,6 +126,14 @@ def create_bkg(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.JwstDataModel` @@ -116,7 +143,8 @@ def create_bkg(input, tab_wavelength, tab_background): # Handle individual NIRSpec FS, NIRSpec MOS if isinstance(input, datamodels.MultiSlitModel): - background = bkg_for_multislit(input, tab_wavelength, tab_background) + background = bkg_for_multislit(input, tab_wavelength, tab_background, + allow_mos=allow_mos) # Handle MIRI LRS elif isinstance(input, datamodels.ImageModel): @@ -134,7 +162,7 @@ def create_bkg(input, tab_wavelength, tab_background): return background -def bkg_for_multislit(input, tab_wavelength, tab_background): +def bkg_for_multislit(input, tab_wavelength, tab_background, allow_mos=False): """Create a 2-D background for a MultiSlitModel. Parameters @@ -148,6 +176,14 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): tab_background : 1-D ndarray The surf_bright column read from the 1-D background table. + allow_mos : bool + If True, NIRSpec MOS data is supported. If False, + background is set to 0.0 for any slit marked as exposure + type NRS_MSASPEC. This parameter should be set to True only + for the master_background_mos step in the spec2 pipeline; + MOS data is not supported via the master_background step + in the spec3 pipeline. + Returns ------- background : `~jwst.datamodels.MultiSlitModel` @@ -162,6 +198,7 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): for (k, slit) in enumerate(input.slits): log.info(f'Expanding background for slit {slit.name}') + wl_array = get_wavelengths(slit, input.meta.exposure.type) if wl_array is None: raise RuntimeError(f"Can't determine wavelengths for {type(slit)}") @@ -184,12 +221,29 @@ def bkg_for_multislit(input, tab_wavelength, tab_background): background.slits[k].dq[mask_limit] = np.bitwise_or(background.slits[k].dq[mask_limit], dqflags.pixel['DO_NOT_USE']) + # Check exposure type for special handling + try: + exp_type = slit.meta.exposure.type + except AttributeError: + exp_type = None + if exp_type is None: + exp_type = input.meta.exposure.type + + # NIRSpec MOS should only have backgrounds assigned in spec2, + # via the master_background_mos step. + if exp_type == 'NRS_MSASPEC' and not allow_mos: + log.warning('Master background subtraction is not supported ' + 'for NIRSpec MOS spectra.') + log.warning('Setting the background to 0.0') + background.slits[k].data[:] = 0.0 + background.slits[k].dq[:] = 0 + continue + # NIRSpec fixed slits need corrections applied to the 2D background # if the slit contains a point source, in order to make the master bkg - # match the calibrated science data in the slit - if input.meta.exposure.type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': - primary = True if slit.name == input.meta.instrument.fixed_slit else False - background.slits[k] = correct_nrs_fs_bkg(background.slits[k], primary) + # match the calibrated science data in the slit. + if exp_type == 'NRS_FIXEDSLIT' and slit.source_type.upper() == 'POINT': + background.slits[k] = correct_nrs_fs_bkg(background.slits[k]) return background diff --git a/jwst/master_background/nirspec_utils.py b/jwst/master_background/nirspec_utils.py index 77bfa3285c..d29c9da26b 100644 --- a/jwst/master_background/nirspec_utils.py +++ b/jwst/master_background/nirspec_utils.py @@ -1,4 +1,5 @@ import logging +import warnings from stdatamodels.jwst import datamodels @@ -70,7 +71,7 @@ def map_to_science_slits(input_model, master_bkg): # Loop over all input slits, creating 2D master background to # match each 2D slitlet cutout - output_model = expand_to_2d(input_model, master_bkg) + output_model = expand_to_2d(input_model, master_bkg, allow_mos=True) return output_model @@ -166,7 +167,7 @@ def correct_nrs_ifu_bkg(input_model): return input_model -def correct_nrs_fs_bkg(input_model, primary_slit): +def correct_nrs_fs_bkg(input_model): """Apply point source vs. uniform source corrections to a NIRSpec Fixed-Slit 2D master background array. @@ -175,9 +176,6 @@ def correct_nrs_fs_bkg(input_model, primary_slit): input_model : `~jwst.datamodels.SlitModel` The input background data. - primary_slit : bool - Is this the primary slit in the exposure? - Returns ------- input_model : `~jwst.datamodels.SlitModel` @@ -201,46 +199,44 @@ def correct_nrs_fs_bkg(input_model, primary_slit): log.warning('Skipping background updates') return input_model - if primary_slit: - # If processing the primary slit, we also need flatfield and - # photom correction arrays - if 'flatfield_point' in input_model.instance: - ff_point = getattr(input_model, 'flatfield_point') - else: - log.warning('flatfield_point array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'flatfield_uniform' in input_model.instance: - ff_uniform = getattr(input_model, 'flatfield_uniform') - else: - log.warning('flatfield_uniform array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'photom_point' in input_model.instance: - ph_point = getattr(input_model, 'photom_point') - else: - log.warning('photom_point array not found in input') - log.warning('Skipping background updates') - return input_model - - if 'photom_uniform' in input_model.instance: - ph_uniform = getattr(input_model, 'photom_uniform') - else: - log.warning('photom_uniform array not found in input') - log.warning('Skipping background updates') - return input_model - - # Apply the corrections for the primary slit + # If processing the primary slit, we also need flatfield and + # photom correction arrays + if 'flatfield_point' in input_model.instance: + ff_point = getattr(input_model, 'flatfield_point') + else: + log.warning('flatfield_point array not found in input') + log.warning('Skipping background updates') + return input_model + + if 'flatfield_uniform' in input_model.instance: + ff_uniform = getattr(input_model, 'flatfield_uniform') + else: + log.warning('flatfield_uniform array not found in input') + log.warning('Skipping background updates') + return input_model + + if 'photom_point' in input_model.instance: + ph_point = getattr(input_model, 'photom_point') + else: + log.warning('photom_point array not found in input') + log.warning('Skipping background updates') + return input_model + + if 'photom_uniform' in input_model.instance: + ph_uniform = getattr(input_model, 'photom_uniform') + else: + log.warning('photom_uniform array not found in input') + log.warning('Skipping background updates') + return input_model + + # Apply the corrections for the primary slit + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "invalid value*", RuntimeWarning) + warnings.filterwarnings("ignore", "divide by zero*", RuntimeWarning) input_model.data *= (pl_uniform / pl_point) * \ (ff_uniform / ff_point) * \ (ph_point / ph_uniform) - else: - # Apply the corrections for secondary slits - input_model.data *= (pl_uniform / pl_point) - return input_model diff --git a/jwst/master_background/tests/test_nirspec_corrections.py b/jwst/master_background/tests/test_nirspec_corrections.py index 6e5a4f4276..5eaa41fcc8 100644 --- a/jwst/master_background/tests/test_nirspec_corrections.py +++ b/jwst/master_background/tests/test_nirspec_corrections.py @@ -52,7 +52,7 @@ def test_fs_correction(): photom_point=ph_ps, photom_uniform=ph_un) corrected = input.data * (ff_un / ff_ps) * (pl_un / pl_ps) * (ph_ps / ph_un) - result = correct_nrs_fs_bkg(input, primary_slit=True) + result = correct_nrs_fs_bkg(input) assert np.allclose(corrected, result.data, rtol=1.e-7) diff --git a/jwst/nsclean/nsclean.py b/jwst/nsclean/nsclean.py index 3117eb624a..bbd3b0d4bf 100644 --- a/jwst/nsclean/nsclean.py +++ b/jwst/nsclean/nsclean.py @@ -156,8 +156,20 @@ def create_mask(input_model, mask_spectral_regions, n_sigma): mask[nan_pix] = False # If IFU or MOS, mask the fixed-slit area of the image; uses hardwired indexes - if exptype in ['nrs_ifu', 'nrs_msaspec']: + if exptype == 'nrs_ifu': + log.info("Masking the fixed slit region for IFU data.") mask[922:1116, :] = False + elif exptype == 'nrs_msaspec': + # check for any slits defined in the fixed slit quadrant: + # if there is nothing there of interest, mask the whole FS region + slit2msa = input_model.meta.wcs.get_transform('slit_frame', 'msa_frame') + is_fs = [s.quadrant == 5 for s in slit2msa.slits] + if not any(is_fs): + log.info("Masking the fixed slit region for MOS data.") + mask[922:1116, :] = False + else: + log.info("Fixed slits found in MSA definition; " + "not masking the fixed slit region for MOS data.") # Use left/right reference pixel columns (first and last 4). Can only be # applied to data that uses all 2048 columns of the detector. diff --git a/jwst/photom/photom.py b/jwst/photom/photom.py index 7f7a6cc4da..c445d9601f 100644 --- a/jwst/photom/photom.py +++ b/jwst/photom/photom.py @@ -848,9 +848,8 @@ def photom_io(self, tabdata, order=None): slit = self.input.slits[self.slitnum] # The NIRSpec fixed-slit primary slit needs special handling if # it contains a point source - if self.exptype.upper() == 'NRS_FIXEDSLIT' and \ - slit.name == self.input.meta.instrument.fixed_slit and \ - slit.source_type.upper() == 'POINT': + if (self.exptype.upper() == 'NRS_FIXEDSLIT' + and slit.source_type.upper() == 'POINT'): # First, compute 2D array of photom correction values using # uncorrected wavelengths, which is appropriate for a uniform source diff --git a/jwst/pipeline/calwebb_spec2.py b/jwst/pipeline/calwebb_spec2.py index 57245439b4..9ce8c04d61 100644 --- a/jwst/pipeline/calwebb_spec2.py +++ b/jwst/pipeline/calwebb_spec2.py @@ -285,6 +285,8 @@ def process_exposure_product( # srctype and wavecorr before flat_field. if exp_type in GRISM_TYPES: calibrated = self._process_grism(calibrated) + elif exp_type == 'NRS_MSASPEC': + calibrated = self._process_nirspec_msa_slits(calibrated) elif exp_type in NRS_SLIT_TYPES: calibrated = self._process_nirspec_slits(calibrated) elif exp_type == 'NIS_SOSS': @@ -365,6 +367,13 @@ def process_exposure_product( x1d = self.photom(x1d) else: self.log.warning("Extract_1d did not return a DataModel - skipping photom.") + elif exp_type == 'NRS_MSASPEC': + # Special handling for MSA spectra, to handle mixed-in + # fixed slits separately + if not self.extract_1d.skip: + x1d = self._extract_nirspec_msa_slits(resampled) + else: + x1d = resampled.copy() else: x1d = resampled.copy() x1d = self.extract_1d(x1d) @@ -524,10 +533,15 @@ def _process_grism(self, data): return calibrated def _process_nirspec_slits(self, data): - """Process NIRSpec + """Process NIRSpec slits. + + This function handles FS, BOTS, and calibration modes. + MOS mode is handled separately, in order to do master + background subtraction and process fixed slits defined in + MSA files. - NIRSpec MOS and FS need srctype and wavecorr before flat_field. - Also have to deal with master background operations. + Note that NIRSpec MOS and FS need srctype and wavecorr before + flat_field. """ calibrated = self.extract_2d(data) calibrated = self.srctype(calibrated) @@ -540,6 +554,76 @@ def _process_nirspec_slits(self, data): return calibrated + def _process_nirspec_msa_slits(self, data): + """Process NIRSpec MSA slits. + + The NRS_MSASPEC exposure type may contain fixed slit definitions + in addition to standard MSA slitlets. These are handled + separately internally to this function, in order to pull the + correct reference files and perform the right algorithms for + each slit type. Processed slits are recombined into a single + model with EXP_TYPE=NRS_MSASPEC on return. + + Note that NIRSpec MOS and FS need srctype and wavecorr before + flat_field. Also have to deal with master background operations. + """ + calibrated = self.extract_2d(data) + calibrated = self.srctype(calibrated) + + # Split the datamodel into 2 pieces: one with MOS slits and + # the other with FS slits + calib_mos = datamodels.MultiSlitModel() + calib_fss = datamodels.MultiSlitModel() + for slit in calibrated.slits: + if slit.quadrant == 5: + slit.meta.exposure.type = "NRS_FIXEDSLIT" + calib_fss.slits.append(slit) + else: + calib_mos.slits.append(slit) + + # First process MOS slits through all remaining steps + calib_mos.update(calibrated) + if len(calib_mos.slits) > 0: + calib_mos = self.master_background_mos(calib_mos) + calib_mos = self.wavecorr(calib_mos) + calib_mos = self.flat_field(calib_mos) + calib_mos = self.pathloss(calib_mos) + calib_mos = self.barshadow(calib_mos) + calib_mos = self.photom(calib_mos) + + # Now repeat for FS slits + if len(calib_fss.slits) > 0: + calib_fss.update(calibrated) + calib_fss.meta.exposure.type = "NRS_FIXEDSLIT" + + # Run each step with an alternate suffix, + # to avoid overwriting previous products if save_results=True + fs_steps = ['wavecorr', 'flat_field', 'pathloss', 'photom'] + for step_name in fs_steps: + # Set suffix + step = getattr(self, step_name) + current_suffix = step.suffix + step.suffix = f'{current_suffix}_fs' + + # Run step + calib_fss = step(calib_fss) + + # Reset suffix + step.suffix = current_suffix + + # Append the FS results to the MOS results + for slit in calib_fss.slits: + calib_mos.slits.append(slit) + + if len(calib_mos.slits) == len(calib_fss.slits): + # update the MOS model with step completion status from the + # FS model, since there were no MOS slits to run + for step in fs_steps: + setattr(calib_mos.meta.cal_step, step, + getattr(calib_fss.meta.cal_step, step)) + + return calib_mos + def _process_niriss_soss(self, data): """Process SOSS @@ -567,3 +651,56 @@ def _process_common(self, data): calibrated = self.residual_fringe(calibrated) # only run on MIRI_MRS data return calibrated + + def _extract_nirspec_msa_slits(self, resampled): + """Extract NIRSpec MSA slits with separate handling for FS slits.""" + + # Check for fixed slits mixed in with MSA spectra: + # they need separate reference files + resamp_mos = datamodels.MultiSlitModel() + resamp_fss = datamodels.MultiSlitModel() + for slit in resampled.slits: + # Quadrant information is not preserved through resampling, + # but MSA slits have numbers for names, so use that to + # distinguish MSA from FS + try: + msa_name = int(slit.name) + except ValueError: + msa_name = None + if msa_name is None: + slit.meta.exposure.type = "NRS_FIXEDSLIT" + resamp_fss.slits.append(slit) + else: + slit.meta.exposure.type = "NRS_MSASPEC" + resamp_mos.slits.append(slit) + resamp_mos.update(resampled) + resamp_fss.update(resampled) + + # Extract the MOS slits + x1d = None + save_x1d = self.extract_1d.save_results + self.extract_1d.save_results = False + if len(resamp_mos.slits) > 0: + self.log.info(f'Extracting {len(resamp_mos.slits)} MSA slitlets') + x1d = self.extract_1d(resamp_mos) + + # Extract the FS slits + if len(resamp_fss.slits) > 0: + self.log.info(f'Extracting {len(resamp_fss.slits)} fixed slits') + resamp_fss.meta.exposure.type = "NRS_FIXEDSLIT" + x1d_fss = self.extract_1d(resamp_fss) + if x1d is None: + x1d = x1d_fss + x1d.meta.exposure.type = "NRS_MSASPEC" + else: + for spec in x1d_fss.spec: + x1d.spec.append(spec) + + # save the composite model + if save_x1d: + self.save_model(x1d, suffix='x1d') + + resamp_mos.close() + resamp_fss.close() + + return x1d diff --git a/jwst/pipeline/calwebb_spec3.py b/jwst/pipeline/calwebb_spec3.py index 667d1229b1..9357903a17 100644 --- a/jwst/pipeline/calwebb_spec3.py +++ b/jwst/pipeline/calwebb_spec3.py @@ -189,16 +189,15 @@ def process(self, input): # and potentially also the slit name (for NIRSpec fixed-slit only). if isinstance(source, tuple): source_id, result = source - # NIRSpec fixed-slit data - if result[0].meta.exposure.type == "NRS_FIXEDSLIT": + if exptype == "NRS_FIXEDSLIT": # Output file name is constructed using the source_id and the slit name slit_name = self._create_nrsfs_slit_name(result) srcid = f's{source_id:>09s}' self.output_file = format_product(output_file, source_id=srcid, slit_name=slit_name) # NIRSpec MOS/MSA data - elif result[0].meta.exposure.type == "NRS_MSASPEC": + elif exptype == "NRS_MSASPEC": # Construct the specially formatted source_id to use in the output file # name that separates source, background, and virtual slits srcid = self._create_nrsmos_source_id(result) @@ -368,4 +367,4 @@ def save_model(model, **kwargs): return result - return save_model \ No newline at end of file + return save_model diff --git a/jwst/regtest/test_nirspec_mos_fs_spec2.py b/jwst/regtest/test_nirspec_mos_fs_spec2.py new file mode 100644 index 0000000000..82dd419888 --- /dev/null +++ b/jwst/regtest/test_nirspec_mos_fs_spec2.py @@ -0,0 +1,55 @@ +import pytest + +from astropy.io.fits.diff import FITSDiff + +from jwst.stpipe import Step + + +@pytest.fixture(scope="module") +def run_pipeline(rtdata_module): + """Run the calwebb_spec2 pipeline on a single NIRSpec MOS/FS exposure.""" + + rtdata = rtdata_module + + # Get the MSA metadata file referenced in the input exposure + rtdata.get_data("nirspec/mos/jw02674004001_01_msa.fits") + + # Get the input ASN file and exposures + rtdata.get_data("nirspec/mos/jw02674004001_03101_00001_nrs1_rate.fits") + + # Run the calwebb_spec2 pipeline; save results from intermediate steps + args = ["calwebb_spec2", rtdata.input, + "--steps.assign_wcs.save_results=true", + "--steps.msa_flagging.save_results=true", + "--steps.master_background_mos.save_results=true", + "--steps.extract_2d.save_results=true", + "--steps.srctype.save_results=true", + "--steps.wavecorr.save_results=true", + "--steps.flat_field.save_results=true", + "--steps.pathloss.save_results=true", + "--steps.barshadow.save_results=true"] + Step.from_cmdline(args) + + return rtdata + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", [ + "assign_wcs", "msa_flagging", "extract_2d", "srctype", + "master_background_mos", "wavecorr", "flat_field", "pathloss", "barshadow", + "wavecorr_fs", "flat_field_fs", "pathloss_fs", + "cal", "s2d", "x1d"]) +def test_nirspec_mos_fs_spec2(run_pipeline, fitsdiff_default_kwargs, suffix): + """Regression test for calwebb_spec2 on a NIRSpec MOS/FS exposure.""" + + # Run the pipeline and retrieve outputs + rtdata = run_pipeline + output = f"jw02674004001_03101_00001_nrs1_{suffix}.fits" + rtdata.output = output + + # Get the truth files + rtdata.get_truth("truth/test_nirspec_mos_fs_spec2/" + output) + + # Compare the results + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() diff --git a/jwst/regtest/test_nirspec_mos_fs_spec3.py b/jwst/regtest/test_nirspec_mos_fs_spec3.py new file mode 100644 index 0000000000..33e82e5a2c --- /dev/null +++ b/jwst/regtest/test_nirspec_mos_fs_spec3.py @@ -0,0 +1,39 @@ +import pytest +from astropy.io.fits.diff import FITSDiff + +from jwst.stpipe import Step + + +@pytest.fixture(scope="module") +def run_pipeline(rtdata_module): + """Run calwebb_spec3 on NIRSpec MOS data.""" + rtdata = rtdata_module + rtdata.get_asn("nirspec/mos/jw02674-o004_20240305t054741_spec3_00001_asn.json") + + # Run the calwebb_spec3 pipeline on the association + args = ["calwebb_spec3", rtdata.input] + Step.from_cmdline(args) + + return rtdata + + +@pytest.mark.bigdata +@pytest.mark.parametrize("suffix", ["cal", "crf", "s2d", "x1d"]) +@pytest.mark.parametrize("source_id", ["b000000003", "b000000004", "b000000048", + "b000000052", "s000001354", "s000012105", + "s000034946"]) +def test_nirspec_mos_fs_spec3(run_pipeline, suffix, source_id, fitsdiff_default_kwargs): + """Check results of calwebb_spec3""" + rtdata = run_pipeline + + output = f"jw02674-o004_{source_id}_nirspec_f290lp-g395m_{suffix}.fits" + rtdata.output = output + rtdata.get_truth(f"truth/test_nirspec_mos_fs_spec3/{output}") + + # Adjust tolerance for machine precision with float32 drizzle code + if suffix == "s2d": + fitsdiff_default_kwargs["rtol"] = 1e-4 + fitsdiff_default_kwargs["atol"] = 1e-5 + + diff = FITSDiff(rtdata.output, rtdata.truth, **fitsdiff_default_kwargs) + assert diff.identical, diff.report() diff --git a/jwst/srctype/srctype.py b/jwst/srctype/srctype.py index 46f192e533..8448f1109f 100644 --- a/jwst/srctype/srctype.py +++ b/jwst/srctype/srctype.py @@ -131,7 +131,7 @@ def set_source_type(input_model, source_type=None): # NIRSpec fixed-slit is a special case: Apply the source type # determined above to only the primary slit (the one in which # the target is located). Set all other slits to the default - # value, which for NRS_FIXEDSLIT is 'POINT'. + # value, which for NRS_FIXEDSLIT is 'EXTENDED'. default_type = 'EXTENDED' primary_slit = input_model.meta.instrument.fixed_slit log.debug(f' primary_slit = {primary_slit}') diff --git a/jwst/wavecorr/tests/test_wavecorr.py b/jwst/wavecorr/tests/test_wavecorr.py index d2e3d08ce6..85e74a0d0a 100644 --- a/jwst/wavecorr/tests/test_wavecorr.py +++ b/jwst/wavecorr/tests/test_wavecorr.py @@ -191,8 +191,9 @@ def test_mos_slit_status(): im_src.slits[0].source_type = 'EXTENDED' im_wave = WavecorrStep.call(im_src) - # check that the step is recorded as completed - assert im_wave.meta.cal_step.wavecorr == 'COMPLETE' + # check that the step is recorded as skipped, + # since no slits were corrected + assert im_wave.meta.cal_step.wavecorr == 'SKIPPED' # check that the step is listed as skipped for extended mos sources assert im_wave.slits[0].meta.cal_step.wavecorr == 'SKIPPED' @@ -248,7 +249,7 @@ def test_wavecorr_fs(): assert_allclose(result.slits[0].source_xpos, 0.127111, atol=1e-6) slit = result.slits[0] - source_xpos = wavecorr.get_source_xpos(slit, slit.meta.wcs, lam=2) + source_xpos = wavecorr.get_source_xpos(slit) assert_allclose(result.slits[0].source_xpos, source_xpos, atol=1e-6) mean_correction = np.abs(src_result.slits[0].wavelength - result.slits[0].wavelength) @@ -261,7 +262,7 @@ def test_wavecorr_fs(): corrected_wavelength = wavecorr.compute_wavelength(slit.meta.wcs, x, y) assert_allclose(slit.wavelength, corrected_wavelength) - # test the roundtripping on the wavelength correction transform + # test the round-tripping on the wavelength correction transform ref_name = result.meta.ref_file.wavecorr.name freference = datamodels.WaveCorrModel(reference_uri_to_cache_path(ref_name, im.crds_observatory)) diff --git a/jwst/wavecorr/wavecorr.py b/jwst/wavecorr/wavecorr.py index 46a92513b2..d514355397 100644 --- a/jwst/wavecorr/wavecorr.py +++ b/jwst/wavecorr/wavecorr.py @@ -64,44 +64,48 @@ def do_correction(input_model, wavecorr_file): output_model = input_model.copy() - # Get the primary slit for a FS exposure - if exp_type == 'NRS_FIXEDSLIT': - primary_slit = input_model.meta.instrument.fixed_slit - if primary_slit is None or primary_slit == 'NONE': - log.warning('Primary slit name not found in input') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - return input_model - # For BRIGHTOBJ, operate on the single SlitModel if isinstance(input_model, datamodels.SlitModel): if _is_point_source(input_model, exp_type): apply_zero_point_correction(output_model, wavecorr_file) else: - # For FS only work on the primary slit + # For FS only work on point source slits with + # position information + corrected = False if exp_type == 'NRS_FIXEDSLIT': + primary_slit = input_model.meta.instrument.fixed_slit for slit in output_model.slits: - if slit.name == primary_slit: - if not hasattr(slit.meta, "dither"): - log.warning('meta.dither is not populated for the primary slit') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - break - if slit.meta.dither.x_offset is None or slit.meta.dither.y_offset is None: - log.warning('dither.x(y)_offset values are None for primary slit') - log.warning('Skipping wavecorr correction') - input_model.meta.cal_step.wavecorr = 'SKIPPED' - break - if _is_point_source(slit, exp_type): - completed = apply_zero_point_correction(slit, wavecorr_file) - if completed: - output_model.meta.cal_step.wavecorr = 'COMPLETE' - else: # pragma: no cover - log.warning(f'Corrections are not invertible for slit {slit.name}') + if _is_point_source(slit, exp_type): + # If fixed slit was not defined via MSA file, + # it must have dither information to find the + # source position, and it must be the primary slit + if not _is_msa_fixed_slit(slit): + if slit.name != primary_slit: + log.warning(f'Skipping wavecorr correction for ' + f'non-primary slit {slit.name}') + continue + if not hasattr(slit.meta, "dither"): + log.warning('meta.dither is not populated for the primary slit') log.warning('Skipping wavecorr correction') - output_model.meta.cal_step.wavecorr = 'SKIPPED' + continue + if slit.meta.dither.x_offset is None or slit.meta.dither.y_offset is None: + log.warning('dither.x(y)_offset values are None for primary slit') + log.warning('Skipping wavecorr correction') + input_model.meta.cal_step.wavecorr = 'SKIPPED' + continue + completed = apply_zero_point_correction(slit, wavecorr_file) + if completed: + corrected = True + slit.meta.cal_step.wavecorr = 'COMPLETE' + else: # pragma: no cover + log.warning(f'Corrections are not invertible for slit {slit.name}') + log.warning('Skipping wavecorr correction') + slit.meta.cal_step.wavecorr = 'SKIPPED' - break + if corrected: + output_model.meta.cal_step.wavecorr = 'COMPLETE' + else: + output_model.meta.cal_step.wavecorr = 'SKIPPED' # For MOS work on all slits containing a point source else: @@ -110,14 +114,18 @@ def do_correction(input_model, wavecorr_file): completed = apply_zero_point_correction(slit, wavecorr_file) if completed: slit.meta.cal_step.wavecorr = 'COMPLETE' + corrected = True else: # pragma: no cover log.warning(f'Corrections are not invertible for slit {slit.name}') log.warning('Skipping wavecorr correction') slit.meta.cal_step.wavecorr = 'SKIPPED' else: slit.meta.cal_step.wavecorr = 'SKIPPED' - - output_model.meta.cal_step.wavecorr = 'COMPLETE' + + if corrected: + output_model.meta.cal_step.wavecorr = 'COMPLETE' + else: + output_model.meta.cal_step.wavecorr = 'SKIPPED' return output_model @@ -142,9 +150,15 @@ def apply_zero_point_correction(slit, reffile): # Get the source position in the slit and set the aperture name if slit.meta.exposure.type in ['NRS_FIXEDSLIT', 'NRS_BRIGHTOBJ']: - # pass lam = 2 microns - # needed for wavecorr with fixed slits - source_xpos = get_source_xpos(slit, slit_wcs, lam=2) + # Check for fixed slits defined via MSA files in + # MOS/FS combination processing: they should not have their + # source position overridden by dither keywords + if _is_msa_fixed_slit(slit): + # get the planned source position + source_xpos = slit.source_xpos + else: + # get the source position from the dither offsets + source_xpos = get_source_xpos(slit) aperture_name = slit.name else: source_xpos = slit.source_xpos @@ -154,11 +168,9 @@ def apply_zero_point_correction(slit, reffile): lam = slit.wavelength.copy() * 1e-6 dispersion = compute_dispersion(slit.meta.wcs) - wave2wavecorr = calculate_wavelength_correction_transform(lam, - dispersion, - reffile, - source_xpos, - aperture_name) + wave2wavecorr = calculate_wavelength_correction_transform( + lam, dispersion, reffile, source_xpos, aperture_name) + # wave2wavecorr should not be None for real data if wave2wavecorr is None: # pragma: no cover completed = False @@ -309,6 +321,25 @@ def compute_wavelength(wcs, xpix=None, ypix=None): return lam +def _is_msa_fixed_slit(slit): + """ + Determine if a fixed slit source was defined via a MSA file. + + Parameters + ---------- + slit : `~stdatamodels.jwst.transforms.models.Slit` + A slit object. + """ + # Fixed slits defined via MSA files in MOS/FS combination + # processing will have a non-empty source name + if (not hasattr(slit, 'source_name') + or slit.source_name is None + or slit.source_name == ""): + return False + else: + return True + + def _is_point_source(slit, exp_type): """ Determine if a source is a point source. @@ -343,7 +374,7 @@ def _is_point_source(slit, exp_type): return result -def get_source_xpos(slit, slit_wcs, lam): +def get_source_xpos(slit): """ Compute the source position within the slit for a NIRSpec fixed slit. @@ -351,10 +382,6 @@ def get_source_xpos(slit, slit_wcs, lam): ---------- slit : `~jwst.datamodels.SlitModel` The slit object. - slit_wcs : `~gwcs.wcs.WCS` - The WCS object for this slit. - lam : float - Wavelength in microns. Returns -------