Merge branch 'master' into nirspec_mos_fs

CHANGES.rst

@@ -22,7 +22,7 @@ assign_wcs
 
 - Updated the routines that load NIRSpec MOS slit and source data from the MSA meta
   data file to properly handle background and virtual slits, and assign appropriate
-  meta data to them for use downstream. [#8442]
+  meta data to them for use downstream. [#8442, #8533]
 
 - Added handling for fixed slit sources defined in a MSA metadata file, for combined
   NIRSpec MOS and fixed slit observations. Slits are now appended to the data
@@ -40,8 +40,16 @@ associations
 - Updated Level3 rules for new handling of NIRSpec MOS source_id formatting when
   constructing output file names. [#8442]
 
+- Added default values for new non-header keywords (``MOSTILNO`` and ``DITHPTIN``)
+  to the default values in the ``asn_make_pool`` script. [#8508]
+
 - Create WFSS Pure-Parallel associations [#8528]
 
+combine_1d
+----------
+
+- Fix weights for combining errors from 1D spectra. [#8520]
+
 dark_current
 ------------
 
@@ -89,6 +97,11 @@ extract_1d
 - Add propagation of uncertainty when annular backgrounds are subtracted
   from source spectra during IFU spectral extraction. [#8515]
 
+- Add propagation of background uncertainty when background is subtracted from 
+  source spectra during non-IFU spectral extraction. [#8532]
+
+- Fix error in application of aperture correction to variance arrays. [#8530]
+
 - Removed a check for the primary slit for NIRSpec fixed slit mode:
   all slits containing point sources are now handled consistently,
   whether they are marked primary or not. [#8467]
@@ -135,6 +148,13 @@ master_background
   Master background correction for MOS mode should be performed
   via ``master_background_mos``, called in ``calwebb_spec2``. [#8467]
 
+master_background_mos
+---------------------
+
+- Updated check for NIRSpec MOS background slits to use new naming convention:
+  ``slit.source_name`` now contains the string "BKG" instead of
+  "background". [#8533]
+
 nsclean
 -------
 
@@ -266,6 +286,11 @@ tweakreg
 - Change code default to use IRAF StarFinder instead of
   DAO StarFinder [#8487]
 
+- Added a check for ``(abs_)separation`` and ``(abs_)tolerance`` parameters
+  that ``separation`` > ``sqrt(2) * tolerance`` that will now log an error
+  message and skip ``tweakreg`` step when this condition is not satisfied and
+  source confusion is possible during catalog matching. [#8476]
+
 wavecorr
 --------
 
@@ -578,6 +603,9 @@ resample
 
 - Removed any reference to the "tophat" kernel for resample step. [#8364]
 
+- Removing unnecessary warning. Errors are propagated identically for
+  the 'exptime' and 'ivm' weight options. [#8258]
+
 - Increased specificity of several warning filters. [#8320]
 
 - Changed deprecated ``stpipe.extern.configobj`` to ``astropy.extern.configobj``. [#8320]

docs/jwst/tweakreg/README.rst

@@ -307,7 +307,8 @@ The ``tweakreg`` step has the following optional arguments:
 * ``use2dhist``: A boolean indicating whether to use 2D histogram to find
   initial offset. (Default=True)
 
-* ``separation``: Minimum object separation in arcsec. (Default=1.0)
+* ``separation``: Minimum object separation in arcsec. It **must be** at least
+  ``sqrt(2)`` times larger than ``tolerance``. (Default=1.0)
 
 * ``tolerance``: Matching tolerance for ``xyxymatch`` in arcsec. (Default=0.7)
 
@@ -374,7 +375,8 @@ Parameters used for absolute astrometry to a reference catalog.
   Otherwise the initial guess for the offsets will be set to zero
   (Default=True)
 
-* ``abs_separation``: Minimum object separation in arcsec. (Default=1.0)
+* ``abs_separation``: Minimum object separation in arcsec. It **must be** at
+  least ``sqrt(2)`` times larger than ``abs_tolerance``. (Default=1.0)
 
 * ``abs_tolerance``: Matching tolerance for ``xyxymatch`` in arcsec.
   (Default=0.7)

jwst/assign_wcs/nirspec.py

@@ -772,13 +772,13 @@ def get_open_msa_slits(prog_id, msa_file, msa_metadata_id, dither_position,
                     (s['source_name'], s['alias'], s['stellarity'], s['ra'], s['dec'])
                     for s in msa_source if s['source_id'] == source_id][0]
             except IndexError:
-                # Missing source information: assign a virtual source name
-                log.warning("Could not retrieve source info from MSA file")
                 source_name = f"{prog_id}_VRT{slitlet_id}"
                 source_alias = "VRT{}".format(slitlet_id)
                 stellarity = 0.0
                 source_ra = 0.0
                 source_dec = 0.0
+                log.warning(f"Could not retrieve source info from MSA file; "
+                            f"assigning virtual source_name={source_name}")
 
             if source_id < 0:
                 log.info(f'Slitlet {slitlet_id} contains virtual source, '

jwst/assign_wcs/tests/test_nirspec.py

@@ -5,6 +5,7 @@
 import shutil
 from math import cos, sin
 import os.path
+import shutil
 
 import pytest
 import numpy as np

jwst/associations/mkpool.py

@@ -25,12 +25,16 @@
     'is_imprt': 'f',
     'pntgtype': 'science',
     'targetid': '1',
+    'mostilno': '1',
+    'dithptin': '1'
 }
 
 
 def mkpool(data,
-           asn_candidate=NON_HEADER_COLS['asn_candidate'], dms_note=NON_HEADER_COLS['dms_note'],
-           is_imprt=NON_HEADER_COLS['is_imprt'], pntgtype=NON_HEADER_COLS['pntgtype'],
+           asn_candidate=NON_HEADER_COLS['asn_candidate'],
+           dms_note=NON_HEADER_COLS['dms_note'],
+           is_imprt=NON_HEADER_COLS['is_imprt'],
+           pntgtype=NON_HEADER_COLS['pntgtype'],
            **kwargs):
     """Create an association pool from a list of FITS files.
 
@@ -112,8 +116,14 @@ def mkpool(data,
     defaults = {param: 'null' for param in params}
     pool = AssociationPool(names=params, dtype=[object] * len(params))
 
-    # Set default values for user-settable non-header parameters
-    non_header_params = {'dms_note': dms_note, 'is_imprt': is_imprt, 'pntgtype': pntgtype}
+    # Set non-header values from hard-coded defaults
+    non_header_params = NON_HEADER_COLS.copy()
+
+    # Update default values for user-settable non-header parameters
+    non_header_params.update(
+        {'dms_note': dms_note,
+         'is_imprt': is_imprt,
+         'pntgtype': pntgtype})
 
     # Setup for target id calculation
     targetid = 0  # Start off with no target id.

jwst/associations/tests/test_mkpool.py

@@ -9,7 +9,7 @@
 
 from jwst.associations.tests import helpers
 from jwst.associations import AssociationPool
-from jwst.associations.mkpool import from_cmdline, mkpool
+from jwst.associations.mkpool import from_cmdline, mkpool, NON_HEADER_COLS
 
 # Optional column settings
 OPT_COLS = [('asn_candidate', [('a3001', 'coron')]),
@@ -19,6 +19,7 @@
 
 # Required column names
 REQUIRED_PARAMS = set(('program', 'filename'))
+NON_HEADER_PARAMS = set(NON_HEADER_COLS.keys())
 
 
 def test_hdu(exposures):
@@ -29,6 +30,7 @@ def test_hdu(exposures):
     pool = mkpool(hdus)
     assert isinstance(pool, AssociationPool)
     assert REQUIRED_PARAMS.issubset(pool.colnames)
+    assert NON_HEADER_PARAMS.issubset(pool.colnames)
     assert len(pool) == len(exposures)
 
 
@@ -40,6 +42,7 @@ def test_hdulist(exposures):
     pool = mkpool(hduls)
     assert isinstance(pool, AssociationPool)
     assert REQUIRED_PARAMS.issubset(pool.colnames)
+    assert NON_HEADER_PARAMS.issubset(pool.colnames)
     assert len(pool) == len(exposures)
     [h.close() for h in hduls]
 
@@ -48,6 +51,7 @@ def test_mkpool(exposures):
     pool = mkpool(exposures)
     assert isinstance(pool, AssociationPool)
     assert REQUIRED_PARAMS.issubset(pool.colnames)
+    assert NON_HEADER_PARAMS.issubset(pool.colnames)
     assert len(pool) == len(exposures)
     filenames = [
         filename
@@ -68,6 +72,16 @@ def test_opt_cols_cmdline(mkpool_cmdline, opt_cols):
     _test_opt_cols(mkpool_cmdline, opt_cols)
 
 
+def test_nonheader_cols(mkpool_with_args):
+    """Ensure that non-header defaults are properly set."""
+    _test_nonheader_cols(mkpool_with_args)
+
+
+def test_nonheader_cols_cmdline(mkpool_cmdline):
+    """Ensure that non-header defaults are properly set with the command line."""
+    _test_nonheader_cols(mkpool_cmdline)
+
+
 # ####################
 # Fixtures & Utilities
 # ####################
@@ -111,3 +125,14 @@ def _test_opt_cols(mkpool_with_args, opt_cols):
     if column == 'asn_candidate':
         expected = '[' + str(('o001', 'observation')) + ', ' + str(expected)[1:]
     assert mkpool_with_args[0][column] == expected
+
+
+def _test_nonheader_cols(mkpool_with_args):
+    """Ensure that non-header defaults are properly set"""
+    opt_dict = dict(OPT_COLS)
+    for column, expected in NON_HEADER_COLS.items():
+        if column in opt_dict:
+            expected = opt_dict[column]
+        if column == 'asn_candidate':
+            expected = '[' + str(('o001', 'observation')) + ', ' + str(expected)[1:]
+        assert mkpool_with_args[0][column] == expected

jwst/combine_1d/tests/test_dq.py → jwst/combine_1d/tests/test_dq_err.py

@@ -31,15 +31,35 @@ def test_dq():
     assert np.isclose(result_dq.spec[0].spec_table['FLUX'][BAD_PIX], spec2.spec_table['FLUX'][BAD_PIX])
 
 
-def create_spec_model(npoints=10, flux=1e-9, wave_range=(11, 13)):
+def test_err():
+    """Test error propagation."""
+    spec1 = create_spec_model(flux=1.0, error=0.1)
+    spec2 = create_spec_model(flux=1.0, error=0.2)
+    ms = datamodels.MultiSpecModel()
+    ms.meta.exposure.exposure_time = 1
+    ms.spec.append(spec1)
+    ms.spec.append(spec2)
+
+    result = Combine1dStep.call(ms)
+
+    expected_error = np.sqrt(0.1**2 + 0.2**2) / 2
+    assert np.allclose(result.spec[0].spec_table['FLUX'], 1.0)
+    assert np.allclose(result.spec[0].spec_table['ERROR'], expected_error)
+    assert np.allclose(result.spec[0].spec_table['SURF_BRIGHT'], 1.0)
+    assert np.allclose(result.spec[0].spec_table['SB_ERROR'], expected_error)
+
+
+def create_spec_model(npoints=10, flux=1e-9, error=1e-10, wave_range=(11, 13)):
     """Create a SpecModel"""
 
-    flux = np.full(npoints, flux)
     wavelength = np.arange(*wave_range, step=(wave_range[1] - wave_range[0]) / npoints)
-    error = np.zeros(npoints)
-    surf_bright = np.zeros(npoints)
-    sb_error = np.zeros(npoints)
-    var_dummy = error.copy()
+    flux = np.full(npoints, flux)
+    error = np.full(npoints, error)
+
+    surf_bright = np.full(npoints, flux)
+    sb_error = np.full(npoints, error)
+
+    var = np.zeros(npoints)
     dq = np.zeros(npoints)
     background = np.zeros(npoints)
     berror = np.zeros(npoints)
@@ -50,9 +70,9 @@ def create_spec_model(npoints=10, flux=1e-9, wave_range=(11, 13)):
     otab = np.array(
         list(
             zip(
-                wavelength, flux, error, var_dummy, var_dummy, var_dummy,
-                surf_bright, sb_error, var_dummy, var_dummy, var_dummy,
-                dq, background, berror, var_dummy, var_dummy, var_dummy,
+                wavelength, flux, error, var, var, var,
+                surf_bright, sb_error, var, var, var,
+                dq, background, berror, var, var, var,
                 npixels
             ),
         ), dtype=spec_dtype

jwst/extract_1d/apply_apcorr.py

@@ -153,15 +153,17 @@ def apply(self, spec_table: fits.FITS_rec):
             Table of aperture corrections values from apcorr reference file.
 
         """
-        cols_to_correct = ('flux', 'flux_error', 'flux_var_poisson', 'flux_var_rnoise',
-                           'flux_var_flat', 'surf_bright', 'sb_error', 'sb_var_poisson',
-                           'sb_var_rnoise', 'sb_var_flat')
+        flux_cols_to_correct = ('flux', 'flux_error', 'surf_bright', 'sb_error')
+        var_cols_to_correct = ('flux_var_poisson', 'flux_var_rnoise', 'flux_var_flat', 
+                            'sb_var_poisson', 'sb_var_rnoise', 'sb_var_flat')
 
         for row in spec_table:
             correction = self.apcorr_func(row['npixels'], row['wavelength'])
 
-            for col in cols_to_correct:
+            for col in flux_cols_to_correct:
                 row[col] *= correction.item()
+            for col in var_cols_to_correct:
+                row[col] *= correction.item() * correction.item()
 
 
 class ApCorrPhase(ApCorrBase):
@@ -234,19 +236,21 @@ def apply(self, spec_table: fits.FITS_rec):
             Table of aperture corrections values from apcorr reference file.
 
         """
-        cols_to_correct = ('flux', 'flux_error', 'flux_var_poisson', 'flux_var_rnoise',
-                           'flux_var_flat', 'surf_bright', 'sb_error', 'sb_var_poisson',
-                           'sb_var_rnoise', 'sb_var_flat')
+        flux_cols_to_correct = ('flux', 'flux_error', 'surf_bright', 'sb_error')
+        var_cols_to_correct = ('flux_var_poisson', 'flux_var_rnoise', 'flux_var_flat', 
+                            'sb_var_poisson', 'sb_var_rnoise', 'sb_var_flat')
 
         for row in spec_table:
             try:
                 correction = self.apcorr_func(row['wavelength'], row['npixels'], self.phase)
             except ValueError:
                 correction = None  # Some input wavelengths might not be supported (especially at the ends of the range)
 
-            for col in cols_to_correct:
-                if correction:
+            if correction:
+                for col in flux_cols_to_correct:
                     row[col] *= correction.item()
+                for col in var_cols_to_correct:
+                    row[col] *= correction.item() * correction.item()
 
 
 class ApCorrRadial(ApCorrBase):
@@ -293,14 +297,16 @@ def apply(self, spec_table: fits.FITS_rec):
             Table of aperture corrections values from apcorr reference file.
 
         """
-        cols_to_correct = ('flux', 'flux_error', 'flux_var_poisson', 'flux_var_rnoise',
-                           'flux_var_flat', 'surf_bright', 'sb_error', 'sb_var_poisson',
-                           'sb_var_rnoise', 'sb_var_flat')
+        flux_cols_to_correct = ('flux', 'flux_error', 'surf_bright', 'sb_error')
+        var_cols_to_correct = ('flux_var_poisson', 'flux_var_rnoise', 'flux_var_flat', 
+                            'sb_var_poisson', 'sb_var_rnoise', 'sb_var_flat')
 
         for i, row in enumerate(spec_table):
             correction = self.apcorr_correction[i]
-            for col in cols_to_correct:
+            for col in flux_cols_to_correct:
                 row[col] *= correction
+            for col in var_cols_to_correct:
+                row[col] *= correction * correction
 
     def match_wavelengths(self, wavelength_ifu):
         # given the ifu wavelength value - redefine the apcor func and radius to this wavelength

jwst/extract_1d/extract1d.py

@@ -457,6 +457,11 @@ def _extract_src_flux(image, var_poisson, var_rnoise, var_flat, x, j, lam, srcli
     # subtract background per pixel:
     val -= bkg
 
+    # add background variance to variance in source extraction region
+    f_var_poisson += b_var_poisson
+    f_var_rnoise += b_var_rnoise
+    f_var_flat += b_var_flat
+
     # scale per pixel values by pixel area included in extraction
     val *= area
     bkg *= area

jwst/master_background/master_background_mos_step.py

@@ -191,7 +191,7 @@ def _classify_slits(self, data):
         # count how many are background vs source.
         num_bkg = num_src = 0
         for slit in data.slits:
-            if "background" in slit.source_name:
+            if nirspec_utils.is_background_msa_slit(slit):
                 num_bkg += 1
             else:
                 num_src += 1