CFHT size (#151)

* CADC-9555 - avoid where possible reading the entire FITS files into memory.

caom2utils/caom2utils/fits2caom2.py

@@ -1468,6 +1468,8 @@ def get_configed_axes_count(self):
             configed_axes += 1
         if self._obs_axis_configed:
             configed_axes += 1
+        if self._custom_axis_configed:
+            configed_axes += 1
         return configed_axes
 
     @property
@@ -1843,8 +1845,7 @@ def __init__(self, src, obs_blueprint=None, uri=None):
         else:
             # assume file
             self.file = src
-            self._headers = data_util.get_local_file_headers(
-                self.file)
+            self._headers = data_util.get_local_headers_from_fits(self.file)
         if obs_blueprint:
             self._blueprint = obs_blueprint
         else:
@@ -3638,7 +3639,6 @@ def get_vos_headers(uri, subject=None):
     Creates the FITS headers object from a vospace file.
     :param uri: vos URI
     :param subject: user credentials. Anonymous if subject is None
-    :param cadc_client: StorageClientWrapper instance
     :return: List of headers corresponding to each extension. Each header is
     of astropy.wcs.Header type - essentially a dictionary of FITS keywords.
     """
@@ -3851,7 +3851,7 @@ def _augment(obs, product_id, uri, blueprint, subject, dumpconfig=False,
             if uri.startswith('vos'):
                 headers = get_vos_headers(uri, subject)
             elif uri.startswith('file'):
-                headers = data_util.get_local_file_headers(uri)
+                headers = data_util.get_local_headers_from_fits(uri)
             else:
                 headers = client.get_head(uri)
             logging.debug(f'Using a FitsParser for remote file {uri}')
@@ -4402,7 +4402,11 @@ def augment(blueprints, no_validate=False, dump_config=False, plugin=None,
                        dump_config, validate_wcs, plugin, file_name, **kwargs)
 
     writer = ObservationWriter()
-    writer.write(obs, out_obs_xml)
+    if out_obs_xml:
+        writer.write(obs, out_obs_xml)
+    else:
+        sys.stdout.flush()
+        writer.write(obs, sys.stdout)
     logging.info('Done augment.')
 
 

caom2utils/caom2utils/tests/test_collections.py

@@ -141,7 +141,9 @@ def test_differences(directory):
             patch('cadcutils.net.ws.WsCapabilities.get_access_url',
                   autospec=True) as cap_mock,\
             patch('caom2utils.fits2caom2.get_vos_headers') as gvh_mock, \
-            patch('caom2utils.fits2caom2._get_vos_meta') as gvm_mock:
+            patch('caom2utils.fits2caom2._get_vos_meta') as gvm_mock, \
+            patch('caom2utils.data_util.get_local_headers_from_fits') as \
+                header_mock:
         def info_mock(uri):
             if uri.startswith('vos'):
                 archive = uri.split('/')[-2]
@@ -168,12 +170,30 @@ def _vos_client_meta(subject, uri):
                             size=682560,
                             file_type='application/fits')
 
+        def _header(fqn):
+            # during operation, want to use astropy on FITS files
+            # but during testing want to use headers and built-in Python file
+            # operations
+            from urllib.parse import urlparse
+            from astropy.io import fits
+            file_uri = urlparse(fqn)
+            try:
+                fits_header = open(file_uri.path).read()
+                headers = data_util.make_headers_from_string(fits_header)
+            except UnicodeDecodeError:
+                hdulist = fits.open(fqn, memmap=True, lazy_load_hdus=True)
+                hdulist.verify('fix')
+                hdulist.close()
+                headers = [h.header for h in hdulist]
+            return headers
+
         swc_si_mock.return_value.cadcinfo.side_effect = info_mock
         swc_si_mock.cadcget.return_value = []
         data_util.get_local_file_info.side_effect = info_mock
         gvh_mock.side_effect = _get_vos_headers
         gvm_mock.side_effect = _vos_client_meta
         cap_mock.return_value = 'https://localhost'
+        header_mock.side_effect = _header
 
         temp = tempfile.NamedTemporaryFile()
         sys.argv = ('{} -o {} --no_validate --observation {} {} {} {} '

caom2utils/caom2utils/tests/test_data_util.py

@@ -72,6 +72,7 @@
 from cadcdata import FileInfo
 from cadcutils import exceptions
 from caom2utils import data_util
+from os.path import join
 
 import pytest
 from unittest.mock import ANY, Mock, patch
@@ -313,6 +314,12 @@ def test_clean_headers():
     assert len(test_result) == 2, 'expect two headers'
 
 
+def test_unicode_decode_error():
+    test_fqn = join(test_fits2caom2.TESTDATA_DIR, 'time_axes.fits')
+    result = data_util.get_local_file_headers(test_fqn)
+    assert result is not None, 'expect retry using a different method'
+
+
 def _check_get_result(test_fqn):
     assert test_fqn.exists(), 'expected file creation'
 

caom2utils/caom2utils/tests/test_fits2caom2.py

@@ -65,15 +65,15 @@
 #
 # ***********************************************************************
 #
-
+import io
 
 from astropy.io import fits
 from astropy.wcs import WCS as awcs
 from cadcutils import net
 from cadcdata import FileInfo
 from caom2utils import FitsParser, WcsParser, main_app, update_blueprint
 from caom2utils import ObsBlueprint, GenericParser, gen_proc
-from caom2utils import get_gen_proc_arg_parser
+from caom2utils import get_gen_proc_arg_parser, augment
 from caom2utils.legacy import load_config
 from caom2utils.fits2caom2 import _visit, _load_plugin, _update_artifact_meta
 
@@ -82,6 +82,7 @@
 from caom2 import get_differences, obs_reader_writer, ObservationReader, Chunk
 from caom2 import SpectralWCS, TemporalWCS, PolarizationWCS, SpatialWCS
 from caom2 import Axis, CoordAxis1D, CoordAxis2D, ChecksumURI, DataProductType
+from caom2 import CalibrationLevel
 import logging
 
 import caom2utils
@@ -156,8 +157,8 @@ def test_augment_energy():
     assert result is None, repr(energy)
 
 
-def test_augment_custom_failure():
-    bp = ObsBlueprint(custom_axis=1)
+def test_augment_failure():
+    bp = ObsBlueprint()
     test_fitsparser = FitsParser(sample_file_4axes, bp)
     artifact = Artifact('ad:{}/{}'.format('TEST', sample_file_4axes),
                         ProductType.SCIENCE, ReleaseType.DATA)
@@ -1473,6 +1474,40 @@ def test_gen_proc_failure(augment_mock, stdout_mock, cap_mock, client_mock):
         assert result is None
 
 
+@patch('sys.stdout', new_callable=io.StringIO)
+@patch('caom2utils.fits2caom2.Client')
+def test_parser_construction(vos_mock, stdout_mock):
+    vos_mock.get_node.side_effect = _get_node
+    test_uri = 'vos:goliaths/abc.fits.gz'
+    test_blueprint = ObsBlueprint()
+    test_blueprint.set('Observation.instrument.keywords', 'instrument keyword')
+    test_blueprint.set('Plane.dataProductType', DataProductType.IMAGE)
+    test_blueprint.set('Plane.calibrationLevel', CalibrationLevel.RAW_STANDARD)
+    test_blueprint.configure_custom_axis(1)
+    test_blueprints = {test_uri: test_blueprint}
+    kwargs = {}
+    augment(test_blueprints, no_validate=False, dump_config=False,
+            plugin=None, obs_obs_xml=None, in_obs_xml=None, collection='TEST',
+            observation='ABC', product_id='test_product_id', uri=test_uri,
+            netrc=False, file_name=None, verbose=False, debug=False,
+            quiet=False, **kwargs)
+    assert '<caom2:uri>vos:goliaths/abc.fits.gz</caom2:uri>' in \
+           stdout_mock.getvalue(), 'Artifact URI missing from Observation'
+    test_out_fqn = os.path.join(TESTDATA_DIR, 'augment.xml')
+    assert not os.path.exists(test_out_fqn)
+    try:
+        augment(test_blueprints, no_validate=False, dump_config=False,
+                plugin=None, out_obs_xml=test_out_fqn, in_obs_xml=None,
+                collection='TEST', observation='ABC',
+                product_id='test_product_id', uri=test_uri, netrc=False,
+                file_name=None, verbose=False, debug=False, quiet=False,
+                **kwargs)
+        assert os.path.exists(test_out_fqn)
+    finally:
+        if os.path.exists(test_out_fqn):
+            os.unlink(test_out_fqn)
+
+
 def _get_local_headers(file_name):
     return _get_headers(file_name, None)
 

caom2utils/setup.cfg

@@ -34,7 +34,7 @@ edit_on_github = False
 github_project = opencadc/caom2tools
 install_requires = cadcdata==2.0rc2 caom2>=2.4 astropy>=2.0 spherical-geometry>=1.2.11;python_version>="3.6" vos>=3.1.1
 # version should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
-version = 1.6.3
+version = 1.6.4