diff --git a/caom2utils/caom2utils/fits2caom2.py b/caom2utils/caom2utils/fits2caom2.py index 6fc7f975..fd436d10 100755 --- a/caom2utils/caom2utils/fits2caom2.py +++ b/caom2utils/caom2utils/fits2caom2.py @@ -1468,6 +1468,8 @@ def get_configed_axes_count(self): configed_axes += 1 if self._obs_axis_configed: configed_axes += 1 + if self._custom_axis_configed: + configed_axes += 1 return configed_axes @property @@ -1843,8 +1845,7 @@ def __init__(self, src, obs_blueprint=None, uri=None): else: # assume file self.file = src - self._headers = data_util.get_local_file_headers( - self.file) + self._headers = data_util.get_local_headers_from_fits(self.file) if obs_blueprint: self._blueprint = obs_blueprint else: @@ -3638,7 +3639,6 @@ def get_vos_headers(uri, subject=None): Creates the FITS headers object from a vospace file. :param uri: vos URI :param subject: user credentials. Anonymous if subject is None - :param cadc_client: StorageClientWrapper instance :return: List of headers corresponding to each extension. Each header is of astropy.wcs.Header type - essentially a dictionary of FITS keywords. """ @@ -3851,7 +3851,7 @@ def _augment(obs, product_id, uri, blueprint, subject, dumpconfig=False, if uri.startswith('vos'): headers = get_vos_headers(uri, subject) elif uri.startswith('file'): - headers = data_util.get_local_file_headers(uri) + headers = data_util.get_local_headers_from_fits(uri) else: headers = client.get_head(uri) logging.debug(f'Using a FitsParser for remote file {uri}') @@ -4402,7 +4402,11 @@ def augment(blueprints, no_validate=False, dump_config=False, plugin=None, dump_config, validate_wcs, plugin, file_name, **kwargs) writer = ObservationWriter() - writer.write(obs, out_obs_xml) + if out_obs_xml: + writer.write(obs, out_obs_xml) + else: + sys.stdout.flush() + writer.write(obs, sys.stdout) logging.info('Done augment.') diff --git a/caom2utils/caom2utils/tests/test_collections.py b/caom2utils/caom2utils/tests/test_collections.py index b3345c8b..6c88d06f 100644 --- a/caom2utils/caom2utils/tests/test_collections.py +++ b/caom2utils/caom2utils/tests/test_collections.py @@ -141,7 +141,9 @@ def test_differences(directory): patch('cadcutils.net.ws.WsCapabilities.get_access_url', autospec=True) as cap_mock,\ patch('caom2utils.fits2caom2.get_vos_headers') as gvh_mock, \ - patch('caom2utils.fits2caom2._get_vos_meta') as gvm_mock: + patch('caom2utils.fits2caom2._get_vos_meta') as gvm_mock, \ + patch('caom2utils.data_util.get_local_headers_from_fits') as \ + header_mock: def info_mock(uri): if uri.startswith('vos'): archive = uri.split('/')[-2] @@ -168,12 +170,30 @@ def _vos_client_meta(subject, uri): size=682560, file_type='application/fits') + def _header(fqn): + # during operation, want to use astropy on FITS files + # but during testing want to use headers and built-in Python file + # operations + from urllib.parse import urlparse + from astropy.io import fits + file_uri = urlparse(fqn) + try: + fits_header = open(file_uri.path).read() + headers = data_util.make_headers_from_string(fits_header) + except UnicodeDecodeError: + hdulist = fits.open(fqn, memmap=True, lazy_load_hdus=True) + hdulist.verify('fix') + hdulist.close() + headers = [h.header for h in hdulist] + return headers + swc_si_mock.return_value.cadcinfo.side_effect = info_mock swc_si_mock.cadcget.return_value = [] data_util.get_local_file_info.side_effect = info_mock gvh_mock.side_effect = _get_vos_headers gvm_mock.side_effect = _vos_client_meta cap_mock.return_value = 'https://localhost' + header_mock.side_effect = _header temp = tempfile.NamedTemporaryFile() sys.argv = ('{} -o {} --no_validate --observation {} {} {} {} ' diff --git a/caom2utils/caom2utils/tests/test_data_util.py b/caom2utils/caom2utils/tests/test_data_util.py index 611a2c3f..eab08332 100644 --- a/caom2utils/caom2utils/tests/test_data_util.py +++ b/caom2utils/caom2utils/tests/test_data_util.py @@ -72,6 +72,7 @@ from cadcdata import FileInfo from cadcutils import exceptions from caom2utils import data_util +from os.path import join import pytest from unittest.mock import ANY, Mock, patch @@ -313,6 +314,12 @@ def test_clean_headers(): assert len(test_result) == 2, 'expect two headers' +def test_unicode_decode_error(): + test_fqn = join(test_fits2caom2.TESTDATA_DIR, 'time_axes.fits') + result = data_util.get_local_file_headers(test_fqn) + assert result is not None, 'expect retry using a different method' + + def _check_get_result(test_fqn): assert test_fqn.exists(), 'expected file creation' diff --git a/caom2utils/caom2utils/tests/test_fits2caom2.py b/caom2utils/caom2utils/tests/test_fits2caom2.py index d5118ef3..7f6f1cf9 100755 --- a/caom2utils/caom2utils/tests/test_fits2caom2.py +++ b/caom2utils/caom2utils/tests/test_fits2caom2.py @@ -65,7 +65,7 @@ # # *********************************************************************** # - +import io from astropy.io import fits from astropy.wcs import WCS as awcs @@ -73,7 +73,7 @@ from cadcdata import FileInfo from caom2utils import FitsParser, WcsParser, main_app, update_blueprint from caom2utils import ObsBlueprint, GenericParser, gen_proc -from caom2utils import get_gen_proc_arg_parser +from caom2utils import get_gen_proc_arg_parser, augment from caom2utils.legacy import load_config from caom2utils.fits2caom2 import _visit, _load_plugin, _update_artifact_meta @@ -82,6 +82,7 @@ from caom2 import get_differences, obs_reader_writer, ObservationReader, Chunk from caom2 import SpectralWCS, TemporalWCS, PolarizationWCS, SpatialWCS from caom2 import Axis, CoordAxis1D, CoordAxis2D, ChecksumURI, DataProductType +from caom2 import CalibrationLevel import logging import caom2utils @@ -156,8 +157,8 @@ def test_augment_energy(): assert result is None, repr(energy) -def test_augment_custom_failure(): - bp = ObsBlueprint(custom_axis=1) +def test_augment_failure(): + bp = ObsBlueprint() test_fitsparser = FitsParser(sample_file_4axes, bp) artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes), ProductType.SCIENCE, ReleaseType.DATA) @@ -1473,6 +1474,40 @@ def test_gen_proc_failure(augment_mock, stdout_mock, cap_mock, client_mock): assert result is None +@patch('sys.stdout', new_callable=io.StringIO) +@patch('caom2utils.fits2caom2.Client') +def test_parser_construction(vos_mock, stdout_mock): + vos_mock.get_node.side_effect = _get_node + test_uri = 'vos:goliaths/abc.fits.gz' + test_blueprint = ObsBlueprint() + test_blueprint.set('Observation.instrument.keywords', 'instrument keyword') + test_blueprint.set('Plane.dataProductType', DataProductType.IMAGE) + test_blueprint.set('Plane.calibrationLevel', CalibrationLevel.RAW_STANDARD) + test_blueprint.configure_custom_axis(1) + test_blueprints = {test_uri: test_blueprint} + kwargs = {} + augment(test_blueprints, no_validate=False, dump_config=False, + plugin=None, obs_obs_xml=None, in_obs_xml=None, collection='TEST', + observation='ABC', product_id='test_product_id', uri=test_uri, + netrc=False, file_name=None, verbose=False, debug=False, + quiet=False, **kwargs) + assert 'vos:goliaths/abc.fits.gz' in \ + stdout_mock.getvalue(), 'Artifact URI missing from Observation' + test_out_fqn = os.path.join(TESTDATA_DIR, 'augment.xml') + assert not os.path.exists(test_out_fqn) + try: + augment(test_blueprints, no_validate=False, dump_config=False, + plugin=None, out_obs_xml=test_out_fqn, in_obs_xml=None, + collection='TEST', observation='ABC', + product_id='test_product_id', uri=test_uri, netrc=False, + file_name=None, verbose=False, debug=False, quiet=False, + **kwargs) + assert os.path.exists(test_out_fqn) + finally: + if os.path.exists(test_out_fqn): + os.unlink(test_out_fqn) + + def _get_local_headers(file_name): return _get_headers(file_name, None) diff --git a/caom2utils/setup.cfg b/caom2utils/setup.cfg index 3f3d60f3..c806c859 100644 --- a/caom2utils/setup.cfg +++ b/caom2utils/setup.cfg @@ -34,7 +34,7 @@ edit_on_github = False github_project = opencadc/caom2tools install_requires = cadcdata==2.0rc2 caom2>=2.4 astropy>=2.0 spherical-geometry>=1.2.11;python_version>="3.6" vos>=3.1.1 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -version = 1.6.3 +version = 1.6.4