From 3e5761a63b72ab3fec7b121eafd5914d39058600 Mon Sep 17 00:00:00 2001 From: Howard Bushouse Date: Tue, 11 Jun 2024 15:26:03 -0400 Subject: [PATCH] JP-2682: Discontinue use of drizpars reference file (#8546) --- CHANGES.rst | 4 + .../references_general/drizpars_reffile.inc | 51 ----- .../references_general/drizpars_selection.inc | 16 -- .../references_general/references_general.rst | 4 - docs/jwst/resample/main.rst | 8 +- docs/jwst/resample/reference_files.rst | 4 +- jwst/resample/resample_spec_step.py | 29 +-- jwst/resample/resample_step.py | 198 ++++-------------- 8 files changed, 50 insertions(+), 264 deletions(-) delete mode 100644 docs/jwst/references_general/drizpars_reffile.inc delete mode 100644 docs/jwst/references_general/drizpars_selection.inc diff --git a/CHANGES.rst b/CHANGES.rst index d4e6dbd9ab..830a597e8a 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -229,6 +229,8 @@ resample - Change `fillval` parameter default from INDEF to NaN [#8488] +- Removed the use of the `drizpars` reference file [#8546] + resample_spec ------------- @@ -240,6 +242,8 @@ resample_spec when the slit was closely aligned with the RA direction sky. [#8511] +- Removed the use of the `drizpars` reference file [#8546] + residual_fringe --------------- diff --git a/docs/jwst/references_general/drizpars_reffile.inc b/docs/jwst/references_general/drizpars_reffile.inc deleted file mode 100644 index faeb08009f..0000000000 --- a/docs/jwst/references_general/drizpars_reffile.inc +++ /dev/null @@ -1,51 +0,0 @@ -.. _drizpars_reffile: - -DRIZPARS Reference File ------------------------ - -:REFTYPE: DRIZPARS -:Data model: `~jwst.datamodels.DrizParsModel` - -The DRIZPARS reference file contains various drizzle parameter values that -control the characteristics of a drizzled image and how it is built. - -.. include:: ../references_general/drizpars_selection.inc - -.. include:: ../includes/standard_keywords.inc - -Type Specific Keywords for DRIZPARS -+++++++++++++++++++++++++++++++++++++ -No additional specific keywords are required in DRIZPARS reference -files, because CRDS selection is based only on the instrument name -(see :ref:`drizpars_selectors`). - -Reference File Format -+++++++++++++++++++++ -DRIZPARS reference files are FITS format, with 1 BINTABLE extension. -The FITS primary HDU does not contain a data array. -The format and content of the file is as follows: - -======== ======== ===== ======================= ========= -EXTNAME XTENSION NAXIS Dimensions Data type -======== ======== ===== ======================= ========= -DRIZPARS BINTABLE 2 TFIELDS = 7 N/A -======== ======== ===== ======================= ========= - -The DRIZPARS extension contains various step parameter values to be -used when processing certain types of image collections. The first -two columns (numimages and filter) are used as row selectors within -the table. Image collections that match those selectors then use the -parameter values specified in the remainder of that table row. -The table contains the following 7 columns: - -=========== ======= =========================================== -TTYPE TFORM Description -=========== ======= =========================================== -numimages integer The number of images to be combined -filter integer The filter used to obtain the images -pixfrac float The pixel "shrinkage" fraction -kernel string The kernel function used to distribute flux -fillval float Value assigned to pixels with no input flux -wht_type string The input image weighting type -stepsize integer Output WCS grid interpolation step size -=========== ======= =========================================== diff --git a/docs/jwst/references_general/drizpars_selection.inc b/docs/jwst/references_general/drizpars_selection.inc deleted file mode 100644 index d45fd71e64..0000000000 --- a/docs/jwst/references_general/drizpars_selection.inc +++ /dev/null @@ -1,16 +0,0 @@ -.. _drizpars_selectors: - -Reference Selection Keywords for DRIZPARS -+++++++++++++++++++++++++++++++++++++++++ -CRDS selects appropriate DRIZPARS references based on the following keywords. -DRIZPARS is not applicable for instruments not in the table. -All keywords used for file selection are *required*. - -========== ======== -Instrument Keywords -========== ======== -MIRI INSTRUME -NIRCam INSTRUME -NIRISS INSTRUME -========== ======== - diff --git a/docs/jwst/references_general/references_general.rst b/docs/jwst/references_general/references_general.rst index 417b6fd894..550eb4dad8 100644 --- a/docs/jwst/references_general/references_general.rst +++ b/docs/jwst/references_general/references_general.rst @@ -150,8 +150,6 @@ documentation on each reference file. +-----------------------------------------------+--------------------------------------------------+ | :ref:`refpix ` | :ref:`REFPIX ` | +-----------------------------------------------+--------------------------------------------------+ -| :ref:`resample ` | :ref:`DRIZPARS ` | -+-----------------------------------------------+--------------------------------------------------+ | :ref:`reset ` | :ref:`RESET ` | +-----------------------------------------------+--------------------------------------------------+ | :ref:`residual_fringe ` | :ref:`FRINGEFREQ ` | @@ -204,8 +202,6 @@ documentation on each reference file. +--------------------------------------------------+-----------------------------------------------+ | :ref:`DISTORTION ` | :ref:`assign_wcs ` | +--------------------------------------------------+-----------------------------------------------+ -| :ref:`DRIZPARS ` | :ref:`resample ` | -+--------------------------------------------------+-----------------------------------------------+ | :ref:`EMICORR ` | :ref:`emicorr ` | +--------------------------------------------------+-----------------------------------------------+ | :ref:`EXTRACT1D ` | :ref:`extract_1d ` | diff --git a/docs/jwst/resample/main.rst b/docs/jwst/resample/main.rst index 08e0069f2d..9291519018 100644 --- a/docs/jwst/resample/main.rst +++ b/docs/jwst/resample/main.rst @@ -15,11 +15,9 @@ The ``resample`` step can take as input either: #. a single 2D input image #. an association table (in json format) -The defined parameters for the drizzle operation itself get -provided by the DRIZPARS reference file (from CRDS). The exact values -used depends on the number of input images being combined and the filter -being used. Other information may be added as selection criteria later, -but for now, only basic information is used. +The parameters for the drizzle operation are provided via ``resample`` +step parameters, which can be overridden by a step parameter reference +file from CRDS. The output product gets defined using the WCS information of all inputs, even if it is just a single input image. The output WCS defines a diff --git a/docs/jwst/resample/reference_files.rst b/docs/jwst/resample/reference_files.rst index d292c96f09..75ad1e0057 100644 --- a/docs/jwst/resample/reference_files.rst +++ b/docs/jwst/resample/reference_files.rst @@ -1,5 +1,3 @@ Reference File ============== -The ``resample`` step uses the DRIZPARS reference file. - -.. include:: ../references_general/drizpars_reffile.inc +The ``resample`` step does not use any reference files. diff --git a/jwst/resample/resample_spec_step.py b/jwst/resample/resample_spec_step.py index fa705f5372..e215a7bb61 100755 --- a/jwst/resample/resample_spec_step.py +++ b/jwst/resample/resample_spec_step.py @@ -62,35 +62,12 @@ def process(self, input): output = input_new.meta.filename self.blendheaders = False - # Get the drizpars reference file - for reftype in self.reference_file_types: - ref_filename = self.get_reference_file(input_models[0], reftype) - - if ref_filename != 'N/A': - self.log.info('Drizpars reference file: {}'.format(ref_filename)) - kwargs = self.get_drizpars(ref_filename, input_models) - else: - # Deal with NIRSpec, which currently has no default drizpars reffile - self.log.info("No DRIZPARS reffile") - kwargs = self._set_spec_defaults() - kwargs['blendheaders'] = self.blendheaders - - kwargs['output_shape'] = self._check_list_pars( - self.output_shape, - 'output_shape', - min_vals=[1, 1] - ) - kwargs['output_wcs'] = self._load_custom_wcs( - self.output_wcs, - kwargs['output_shape'] - ) - - kwargs['allowed_memory'] = self.allowed_memory + # Setup drizzle-related parameters + kwargs = self.get_drizpars() kwargs['output'] = output - - # Call resampling self.drizpars = kwargs + # Call resampling if isinstance(input_models[0], MultiSlitModel): result = self._process_multislit(input_models) diff --git a/jwst/resample/resample_step.py b/jwst/resample/resample_step.py index f19e3cfcb9..1f99d12a41 100755 --- a/jwst/resample/resample_step.py +++ b/jwst/resample/resample_step.py @@ -2,10 +2,7 @@ import re from copy import deepcopy -import numpy as np import asdf -from astropy.extern.configobj.validate import Validator -from astropy.extern.configobj.configobj import ConfigObj from stdatamodels.jwst import datamodels @@ -43,24 +40,24 @@ class ResampleStep(Step): class_alias = "resample" spec = """ - pixfrac = float(default=1.0) # change back to None when drizpar reference files are updated - kernel = string(default='square') # change back to None when drizpar reference files are updated - fillval = string(default='NAN' ) - weight_type = option('ivm', 'exptime', None, default='ivm') # change back to None when drizpar ref update + pixfrac = float(min=0.0, max=1.0, default=1.0) # Pixel shrinkage factor + kernel = option('square','gaussian','point','turbo','lanczos2','lanczos3',default='square') # Flux distribution kernel + fillval = string(default='NAN') # Output value for pixels with no weight or flux + weight_type = option('ivm', 'exptime', None, default='ivm') # Input image weighting type output_shape = int_list(min=2, max=2, default=None) # [x, y] order crpix = float_list(min=2, max=2, default=None) crval = float_list(min=2, max=2, default=None) - rotation = float(default=None) - pixel_scale_ratio = float(default=1.0) # Ratio of input to output pixel scale - pixel_scale = float(default=None) # Absolute pixel scale in arcsec - output_wcs = string(default='') # Custom output WCS. - single = boolean(default=False) - blendheaders = boolean(default=True) - allowed_memory = float(default=None) # Fraction of memory to use for the combined image. - in_memory = boolean(default=True) + rotation = float(default=None) # Output image Y-axis PA relative to North + pixel_scale_ratio = float(default=1.0) # Ratio of input to output pixel scale + pixel_scale = float(default=None) # Absolute pixel scale in arcsec + output_wcs = string(default='') # Custom output WCS + single = boolean(default=False) # Resample each input to its own output grid + blendheaders = boolean(default=True) # Blend metadata from inputs into output + allowed_memory = float(default=None) # Fraction of memory to use for the combined image + in_memory = boolean(default=True) # Keep images in memory """ - reference_file_types = ['drizpars'] + reference_file_types = [] def process(self, input): @@ -87,39 +84,8 @@ def process(self, input): # resample can only handle 2D images, not 3D cubes, etc raise RuntimeError("Input {} is not a 2D image.".format(input_models[0])) - # Get drizzle parameters reference file, if there is one - self.wht_type = self.weight_type - if 'drizpars' in self.reference_file_types: - ref_filename = self.get_reference_file(input_models[0], 'drizpars') - else: # no drizpars reference file found - ref_filename = 'N/A' - - if ref_filename == 'N/A': - self.log.info('No drizpars reference file found.') - kwargs = self._set_spec_defaults() - else: - self.log.info('Using drizpars reference file: {}'.format(ref_filename)) - kwargs = self.get_drizpars(ref_filename, input_models) - - kwargs['allowed_memory'] = self.allowed_memory - - # Custom output WCS parameters. - # Modify get_drizpars if any of these get into reference files: - kwargs['output_shape'] = self._check_list_pars( - self.output_shape, - 'output_shape', - min_vals=[1, 1] - ) - kwargs['output_wcs'] = self._load_custom_wcs( - self.output_wcs, - kwargs['output_shape'] - ) - kwargs['crpix'] = self._check_list_pars(self.crpix, 'crpix') - kwargs['crval'] = self._check_list_pars(self.crval, 'crval') - kwargs['rotation'] = self.rotation - kwargs['pscale'] = self.pixel_scale - kwargs['pscale_ratio'] = self.pixel_scale_ratio - kwargs['in_memory'] = self.in_memory + # Setup drizzle-related parameters + kwargs = self.get_drizpars() # Call the resampling routine resamp = resample.ResampleData(input_models, output=output, **kwargs) @@ -195,128 +161,42 @@ def _load_custom_wcs(asdf_wcs_file, output_shape): return wcs - def get_drizpars(self, ref_filename, input_models): + def get_drizpars(self): """ - Extract drizzle parameters from reference file. - - This method extracts parameters from the drizpars reference file and - uses those to set defaults on the following ResampleStep configuration - parameters: - - pixfrac = float(default=None) - kernel = string(default=None) - fillval = string(default=None) - wht_type = option('ivm', 'exptime', None, default=None) - - Once the defaults are set from the reference file, if the user has - used a resample.cfg file or run ResampleStep using command line args, - then these will overwrite the defaults pulled from the reference file. + Load all drizzle-related parameter values into kwargs list. """ - with datamodels.DrizParsModel(ref_filename) as drpt: - drizpars_table = drpt.data - - num_groups = len(input_models.group_names) - filtname = input_models[0].meta.instrument.filter - row = None - filter_match = False - # look for row that applies to this set of input data models - for n, filt, num in zip( - range(0, len(drizpars_table)), - drizpars_table['filter'], - drizpars_table['numimages'] - ): - # only remember this row if no exact match has already been made for - # the filter. This allows the wild-card row to be anywhere in the - # table; since it may be placed at beginning or end of table. - - if str(filt) == "ANY" and not filter_match and num_groups >= num: - row = n - # always go for an exact match if present, though... - if filtname == filt and num_groups >= num: - row = n - filter_match = True - - # With presence of wild-card rows, code should never trigger this logic - if row is None: - self.log.error("No row found in %s matching input data.", ref_filename) - raise ValueError - - # Define the keys to pull from drizpars reffile table. - # All values should be None unless the user set them on the command - # line or in the call to the step - - drizpars = dict( + # Define the keys pulled from step parameters + kwargs = dict( pixfrac=self.pixfrac, kernel=self.kernel, fillval=self.fillval, - wht_type=self.weight_type - # pscale_ratio=self.pixel_scale_ratio, # I think this can be removed JEM (??) - ) - - # For parameters that are set in drizpars table but not set by the - # user, use these. Otherwise, use values set by user. - reffile_drizpars = {k: v for k, v in drizpars.items() if v is None} - user_drizpars = {k: v for k, v in drizpars.items() if v is not None} - - # read in values from that row for each parameter - for k in reffile_drizpars: - if k in drizpars_table.names: - reffile_drizpars[k] = drizpars_table[k][row] - - # Convert the strings in the FITS binary table from np.bytes_ to str - for k, v in reffile_drizpars.items(): - if isinstance(v, np.bytes_): - reffile_drizpars[k] = v.decode('UTF-8') - - all_drizpars = {**reffile_drizpars, **user_drizpars} - - kwargs = dict( + wht_type=self.weight_type, good_bits=GOOD_BITS, single=self.single, - blendheaders=self.blendheaders + blendheaders=self.blendheaders, + allowed_memory = self.allowed_memory, + in_memory = self.in_memory ) - kwargs.update(all_drizpars) - - for k, v in kwargs.items(): - self.log.debug(' {}={}'.format(k, v)) - - return kwargs - - def _set_spec_defaults(self): - """NIRSpec currently has no default drizpars reference file, so default - drizzle parameters are not set properly. This method sets them. - - Remove this class method when a drizpars reffile is delivered. - """ - configspec = self.load_spec_file() - config = ConfigObj(configspec=configspec) - if config.validate(Validator()): - kwargs = config.dict() - - if self.pixfrac is None: - self.pixfrac = 1.0 - if self.kernel is None: - self.kernel = 'square' - if self.fillval is None: - self.fillval = 'NAN' - # Force definition of good bits - kwargs['good_bits'] = GOOD_BITS - kwargs['pixfrac'] = self.pixfrac - kwargs['kernel'] = str(self.kernel) - kwargs['fillval'] = str(self.fillval) - # self.weight_type has a default value of None - # The other instruments read this parameter from a reference file - if self.wht_type is None: - self.wht_type = 'ivm' - - kwargs['wht_type'] = str(self.wht_type) + # Custom output WCS parameters. + kwargs['output_shape'] = self._check_list_pars( + self.output_shape, + 'output_shape', + min_vals=[1, 1] + ) + kwargs['output_wcs'] = self._load_custom_wcs( + self.output_wcs, + kwargs['output_shape'] + ) + kwargs['crpix'] = self._check_list_pars(self.crpix, 'crpix') + kwargs['crval'] = self._check_list_pars(self.crval, 'crval') + kwargs['rotation'] = self.rotation + kwargs['pscale'] = self.pixel_scale kwargs['pscale_ratio'] = self.pixel_scale_ratio - kwargs.pop('pixel_scale_ratio') + # Report values to processing log for k, v in kwargs.items(): - if k in ['pixfrac', 'kernel', 'fillval', 'wht_type', 'pscale_ratio']: - log.info(' using: %s=%s', k, repr(v)) + self.log.debug(' {}={}'.format(k, v)) return kwargs