diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d4caab1..808cdf63a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.8.2] 2024-01-08 + +### Added + - GDAL 3.8.2 + ## [3.8.1] 2023-12-12 ### Added diff --git a/deps/libgdal.sh b/deps/libgdal.sh index 9b6e9f91d..b268ab2d7 100755 --- a/deps/libgdal.sh +++ b/deps/libgdal.sh @@ -5,7 +5,7 @@ set -eu DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$DIR/libgdal" -GDAL_VERSION=3.8.1 +GDAL_VERSION=3.8.2 GDAL_VERSION_SUFFIX= dir_gdal=./gdal dir_formats_gyp=./gyp-formats diff --git a/deps/libgdal/gdal/CITATION.cff b/deps/libgdal/gdal/CITATION.cff index cd99a2ee6..c9badc1d4 100644 --- a/deps/libgdal/gdal/CITATION.cff +++ b/deps/libgdal/gdal/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: Please cite this software using these metadata or in the CITATION file. type: software title: GDAL -version: 3.8.1 -date-released: 2023-11-23 +version: 3.8.2 +date-released: 2023-12-16 doi: 10.5281/zenodo.5884351 abstract: GDAL is a translator library for raster and vector geospatial data formats that is released under an MIT style Open Source License by the Open diff --git a/deps/libgdal/gdal/VERSION b/deps/libgdal/gdal/VERSION index f28071967..a08ffae0c 100644 --- a/deps/libgdal/gdal/VERSION +++ b/deps/libgdal/gdal/VERSION @@ -1 +1 @@ -3.8.1 +3.8.2 diff --git a/deps/libgdal/gdal/alg/llrasterize.cpp b/deps/libgdal/gdal/alg/llrasterize.cpp index 6444a0b09..7788e2c6d 100644 --- a/deps/libgdal/gdal/alg/llrasterize.cpp +++ b/deps/libgdal/gdal/alg/llrasterize.cpp @@ -477,7 +477,8 @@ void GDALdllImageLineAllTouched(int nRasterXSize, int nRasterYSize, const int iX = static_cast(floor(dfXEnd)); int iY = static_cast(floor(dfY)); - int iYEnd = static_cast(floor(dfYEnd)); + int iYEnd = + static_cast(floor(dfYEnd - EPSILON_INTERSECT_ONLY)); if (iX < 0 || iX >= nRasterXSize) continue; @@ -556,7 +557,8 @@ void GDALdllImageLineAllTouched(int nRasterXSize, int nRasterYSize, int iX = static_cast(floor(dfX)); const int iY = static_cast(floor(dfY)); - int iXEnd = static_cast(floor(dfXEnd)); + int iXEnd = + static_cast(floor(dfXEnd - EPSILON_INTERSECT_ONLY)); if (iY < 0 || iY >= nRasterYSize) continue; diff --git a/deps/libgdal/gdal/apps/gdal_footprint_lib.cpp b/deps/libgdal/gdal/apps/gdal_footprint_lib.cpp index 68f86f1d0..cc8c88db0 100644 --- a/deps/libgdal/gdal/apps/gdal_footprint_lib.cpp +++ b/deps/libgdal/gdal/apps/gdal_footprint_lib.cpp @@ -452,6 +452,12 @@ GetOutputLayerAndUpdateDstDS(const char *pszDest, GDALDatasetH &hDstDS, if (!psOptions->osDestLayerName.empty()) { poLayer = poDstDS->GetLayerByName(psOptions->osDestLayerName.c_str()); + if (!poLayer) + { + CPLError(CE_Failure, CPLE_AppDefined, "Cannot find layer %s", + psOptions->osDestLayerName.c_str()); + return nullptr; + } } else if (poDstDS->GetLayerCount() == 1 && poDstDS->GetDriver() && EQUAL(poDstDS->GetDriver()->GetDescription(), "ESRI Shapefile")) diff --git a/deps/libgdal/gdal/apps/gdal_translate_lib.cpp b/deps/libgdal/gdal/apps/gdal_translate_lib.cpp index c3bf02d17..882e72af0 100644 --- a/deps/libgdal/gdal/apps/gdal_translate_lib.cpp +++ b/deps/libgdal/gdal/apps/gdal_translate_lib.cpp @@ -1110,100 +1110,16 @@ GDALDatasetH GDALTranslate(const char *pszDest, GDALDatasetH hSrcDataset, /* -------------------------------------------------------------------- */ if (!psOptions->aosCreateOptions.FetchBool("APPEND_SUBDATASET", false)) { - // Someone issuing Create("foo.tif") on a - // memory driver doesn't expect files with those names to be deleted - // on a file system... - // This is somewhat messy. Ideally there should be a way for the - // driver to overload the default behavior - if (!EQUAL(psOptions->osFormat.c_str(), "MEM") && - !EQUAL(psOptions->osFormat.c_str(), "Memory")) + if (!EQUAL(psOptions->osFormat.c_str(), "VRT")) { - /* -------------------------------------------------------------------- - */ - /* Establish list of files of output dataset if it already - * exists. */ - /* -------------------------------------------------------------------- - */ - std::set oSetExistingDestFiles; - { - CPLPushErrorHandler(CPLQuietErrorHandler); - const char *const apszAllowedDrivers[] = { - psOptions->osFormat.c_str(), nullptr}; - auto poExistingOutputDS = - std::unique_ptr(GDALDataset::Open( - pszDest, GDAL_OF_RASTER, apszAllowedDrivers)); - if (poExistingOutputDS) - { - char **papszFileList = poExistingOutputDS->GetFileList(); - for (char **papszIter = papszFileList; - papszIter && *papszIter; ++papszIter) - { - oSetExistingDestFiles.insert( - CPLString(*papszIter).replaceAll('\\', '/')); - } - CSLDestroy(papszFileList); - } - CPLPopErrorHandler(); - } - - /* -------------------------------------------------------------------- - */ - /* Check if the source dataset shares some files with the dest - * one.*/ - /* -------------------------------------------------------------------- - */ - std::set oSetExistingDestFilesFoundInSource; - if (!oSetExistingDestFiles.empty()) - { - CPLPushErrorHandler(CPLQuietErrorHandler); - // We need to reopen in a temporary dataset for the particular - // case of overwritten a .tif.ovr file from a .tif - // If we probe the file list of the .tif, it will then open the - // .tif.ovr ! - const char *const apszAllowedDrivers[] = { - poSrcDS->GetDriver() - ? poSrcDS->GetDriver()->GetDescription() - : nullptr, - nullptr}; - auto poSrcDSTmp = std::unique_ptr( - GDALDataset::Open(poSrcDS->GetDescription(), GDAL_OF_RASTER, - apszAllowedDrivers)); - if (poSrcDSTmp) - { - char **papszFileList = poSrcDSTmp->GetFileList(); - for (char **papszIter = papszFileList; - papszIter && *papszIter; ++papszIter) - { - CPLString osFilename(*papszIter); - osFilename.replaceAll('\\', '/'); - if (oSetExistingDestFiles.find(osFilename) != - oSetExistingDestFiles.end()) - { - oSetExistingDestFilesFoundInSource.insert( - osFilename); - } - } - CSLDestroy(papszFileList); - } - CPLPopErrorHandler(); - } + // Prevent GDALDriver::CreateCopy() from doing that again. + psOptions->aosCreateOptions.SetNameValue( + "@QUIET_DELETE_ON_CREATE_COPY", "NO"); + } - // If the source file(s) and the dest one share some files in - // common, only remove the files that are *not* in common - if (!oSetExistingDestFilesFoundInSource.empty()) - { - for (const std::string &osFilename : oSetExistingDestFiles) - { - if (oSetExistingDestFilesFoundInSource.find(osFilename) == - oSetExistingDestFilesFoundInSource.end()) - { - VSIUnlink(osFilename.c_str()); - } - } - } + GDALDriver::FromHandle(hDriver)->QuietDeleteForCreateCopy(pszDest, + poSrcDS); - GDALDriver::FromHandle(hDriver)->QuietDelete(pszDest); - } // Make sure to load early overviews, so that on the GTiff driver // external .ovr is looked for before it might be created as the // output dataset ! diff --git a/deps/libgdal/gdal/apps/gdal_utils_priv.h b/deps/libgdal/gdal/apps/gdal_utils_priv.h index a91a44b15..2a705cdbc 100644 --- a/deps/libgdal/gdal/apps/gdal_utils_priv.h +++ b/deps/libgdal/gdal/apps/gdal_utils_priv.h @@ -88,26 +88,6 @@ struct GDALWarpAppOptionsForBinary char **papszAllowInputDrivers; }; -/* Access modes */ -typedef enum -{ - ACCESS_CREATION, - ACCESS_UPDATE, /* open existing output datasource in update mode rather than - trying to create a new one */ - ACCESS_APPEND, /* append to existing layer instead of creating new */ - ACCESS_OVERWRITE /* delete the output layer and recreate it empty */ -} GDALVectorTranslateAccessMode; - -struct GDALVectorTranslateOptionsForBinary -{ - char *pszDataSource; - char *pszDestDataSource; - int bQuiet; - char **papszOpenOptions; - char *pszFormat; - GDALVectorTranslateAccessMode eAccessMode; -}; - struct GDALDEMProcessingOptionsForBinary { char *pszProcessing; @@ -135,6 +115,30 @@ struct GDALBuildVRTOptionsForBinary CPL_C_END +/* Access modes */ +typedef enum +{ + ACCESS_CREATION, + ACCESS_UPDATE, /* open existing output datasource in update mode rather than + trying to create a new one */ + ACCESS_APPEND, /* append to existing layer instead of creating new */ + ACCESS_OVERWRITE /* delete the output layer and recreate it empty */ +} GDALVectorTranslateAccessMode; + +struct GDALVectorTranslateOptionsForBinary +{ + std::string osDataSource{}; + bool bDestSpecified = false; + std::string osDestDataSource{}; + bool bQuiet = false; + CPLStringList aosOpenOptions{}; + std::string osFormat; + GDALVectorTranslateAccessMode eAccessMode = ACCESS_CREATION; + + /* Allowed input drivers. */ + CPLStringList aosAllowInputDrivers{}; +}; + struct GDALMultiDimInfoOptionsForBinary { /* Filename to open. */ diff --git a/deps/libgdal/gdal/apps/ogr2ogr_bin.cpp b/deps/libgdal/gdal/apps/ogr2ogr_bin.cpp index b0d708697..c497b7f2f 100644 --- a/deps/libgdal/gdal/apps/ogr2ogr_bin.cpp +++ b/deps/libgdal/gdal/apps/ogr2ogr_bin.cpp @@ -81,7 +81,8 @@ static void Usage(bool bIsError, const char *pszAdditionalMsg = nullptr, " [-a_srs ] [-t_srs ] [-s_srs " "] " "[-ct ]\n" - " [-f ] [-overwrite] " + " [-if ] [-f ] " + "[-overwrite] " "[-dsco =]...\n" " [-lco =]... [-nln ] \n" " [-nlt " @@ -233,8 +234,7 @@ static void Usage(bool bIsError, const char *pszAdditionalMsg = nullptr, static GDALVectorTranslateOptionsForBinary * GDALVectorTranslateOptionsForBinaryNew() { - return static_cast( - CPLCalloc(1, sizeof(GDALVectorTranslateOptionsForBinary))); + return new GDALVectorTranslateOptionsForBinary; } /************************************************************************/ @@ -244,14 +244,7 @@ GDALVectorTranslateOptionsForBinaryNew() static void GDALVectorTranslateOptionsForBinaryFree( GDALVectorTranslateOptionsForBinary *psOptionsForBinary) { - if (psOptionsForBinary) - { - CPLFree(psOptionsForBinary->pszDataSource); - CPLFree(psOptionsForBinary->pszDestDataSource); - CSLDestroy(psOptionsForBinary->papszOpenOptions); - CPLFree(psOptionsForBinary->pszFormat); - CPLFree(psOptionsForBinary); - } + delete psOptionsForBinary; } /************************************************************************/ @@ -327,10 +320,10 @@ MAIN_START(nArgc, papszArgv) goto exit; } - if (psOptionsForBinary->pszDataSource == nullptr || - psOptionsForBinary->pszDestDataSource == nullptr) + if (psOptionsForBinary->osDataSource.empty() || + !psOptionsForBinary->bDestSpecified) { - if (psOptionsForBinary->pszDestDataSource == nullptr) + if (!psOptionsForBinary->bDestSpecified) Usage(true, "no target datasource provided"); else Usage(true, "no source datasource provided"); @@ -339,8 +332,8 @@ MAIN_START(nArgc, papszArgv) goto exit; } - if (strcmp(psOptionsForBinary->pszDestDataSource, "/vsistdout/") == 0) - psOptionsForBinary->bQuiet = TRUE; + if (psOptionsForBinary->osDestDataSource == "/vsistdout/") + psOptionsForBinary->bQuiet = true; /* -------------------------------------------------------------------- */ /* Open data source. */ @@ -350,12 +343,13 @@ MAIN_START(nArgc, papszArgv) // output Known to cause problems with at least FGdb, SQlite and GPKG // drivers. See #4270 if (psOptionsForBinary->eAccessMode != ACCESS_CREATION && - strcmp(psOptionsForBinary->pszDestDataSource, - psOptionsForBinary->pszDataSource) == 0) + psOptionsForBinary->osDestDataSource == + psOptionsForBinary->osDataSource) { - hODS = GDALOpenEx(psOptionsForBinary->pszDataSource, - GDAL_OF_UPDATE | GDAL_OF_VECTOR, nullptr, - psOptionsForBinary->papszOpenOptions, nullptr); + hODS = GDALOpenEx(psOptionsForBinary->osDataSource.c_str(), + GDAL_OF_UPDATE | GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), + psOptionsForBinary->aosOpenOptions.List(), nullptr); GDALDriverH hDriver = hODS != nullptr ? GDALGetDatasetDriver(hODS) : nullptr; @@ -366,9 +360,10 @@ MAIN_START(nArgc, papszArgv) EQUAL(GDALGetDescription(hDriver), "SQLite") || EQUAL(GDALGetDescription(hDriver), "GPKG"))) { - hDS = GDALOpenEx(psOptionsForBinary->pszDataSource, GDAL_OF_VECTOR, - nullptr, psOptionsForBinary->papszOpenOptions, - nullptr); + hDS = GDALOpenEx( + psOptionsForBinary->osDataSource.c_str(), GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), + psOptionsForBinary->aosOpenOptions.List(), nullptr); } else { @@ -379,8 +374,9 @@ MAIN_START(nArgc, papszArgv) else { hDS = - GDALOpenEx(psOptionsForBinary->pszDataSource, GDAL_OF_VECTOR, - nullptr, psOptionsForBinary->papszOpenOptions, nullptr); + GDALOpenEx(psOptionsForBinary->osDataSource.c_str(), GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), + psOptionsForBinary->aosOpenOptions.List(), nullptr); } /* -------------------------------------------------------------------- */ @@ -393,7 +389,7 @@ MAIN_START(nArgc, papszArgv) fprintf(stderr, "FAILURE:\n" "Unable to open datasource `%s' with the following drivers.\n", - psOptionsForBinary->pszDataSource); + psOptionsForBinary->osDataSource.c_str()); for (int iDriver = 0; iDriver < poDM->GetDriverCount(); iDriver++) { @@ -411,16 +407,16 @@ MAIN_START(nArgc, papszArgv) goto exit; } - if (hODS != nullptr && psOptionsForBinary->pszFormat != nullptr) + if (hODS != nullptr && !psOptionsForBinary->osFormat.empty()) { GDALDriverManager *poDM = GetGDALDriverManager(); GDALDriver *poDriver = - poDM->GetDriverByName(psOptionsForBinary->pszFormat); + poDM->GetDriverByName(psOptionsForBinary->osFormat.c_str()); if (poDriver == nullptr) { fprintf(stderr, "Unable to find driver `%s'.\n", - psOptionsForBinary->pszFormat); + psOptionsForBinary->osFormat.c_str()); fprintf(stderr, "The following drivers are available:\n"); for (int iDriver = 0; iDriver < poDM->GetDriverCount(); iDriver++) @@ -452,8 +448,9 @@ MAIN_START(nArgc, papszArgv) { // TODO(schwehr): Remove scope after removing gotos int bUsageError = FALSE; - hDstDS = GDALVectorTranslate(psOptionsForBinary->pszDestDataSource, - hODS, 1, &hDS, psOptions, &bUsageError); + hDstDS = + GDALVectorTranslate(psOptionsForBinary->osDestDataSource.c_str(), + hODS, 1, &hDS, psOptions, &bUsageError); if (bUsageError) Usage(true); else diff --git a/deps/libgdal/gdal/apps/ogr2ogr_lib.cpp b/deps/libgdal/gdal/apps/ogr2ogr_lib.cpp index 30f2115b1..35b3acaa7 100644 --- a/deps/libgdal/gdal/apps/ogr2ogr_lib.cpp +++ b/deps/libgdal/gdal/apps/ogr2ogr_lib.cpp @@ -6487,7 +6487,22 @@ GDALVectorTranslateOptions *GDALVectorTranslateOptionsNew( { psOptions->bQuiet = true; if (psOptionsForBinary) - psOptionsForBinary->bQuiet = TRUE; + psOptionsForBinary->bQuiet = true; + } + else if (EQUAL(papszArgv[i], "-if")) + { + CHECK_HAS_ENOUGH_ADDITIONAL_ARGS(1); + i++; + if (psOptionsForBinary) + { + if (GDALGetDriverByName(papszArgv[i]) == nullptr) + { + CPLError(CE_Warning, CPLE_AppDefined, + "%s is not a recognized driver", papszArgv[i]); + } + psOptionsForBinary->aosAllowInputDrivers.AddString( + papszArgv[i]); + } } else if (EQUAL(papszArgv[i], "-f") || EQUAL(papszArgv[i], "-of")) { @@ -6511,8 +6526,7 @@ GDALVectorTranslateOptions *GDALVectorTranslateOptionsNew( ++i; if (psOptionsForBinary) { - psOptionsForBinary->papszOpenOptions = CSLAddString( - psOptionsForBinary->papszOpenOptions, papszArgv[i]); + psOptionsForBinary->aosOpenOptions.AddString(papszArgv[i]); } } else if (EQUAL(papszArgv[i], "-doo")) @@ -7262,17 +7276,16 @@ GDALVectorTranslateOptions *GDALVectorTranslateOptionsNew( papszArgv[i]); return nullptr; } - else if (psOptionsForBinary && - psOptionsForBinary->pszDestDataSource == nullptr) + else if (psOptionsForBinary && !psOptionsForBinary->bDestSpecified) { iArgStart = -1; - psOptionsForBinary->pszDestDataSource = CPLStrdup(papszArgv[i]); + psOptionsForBinary->bDestSpecified = true; + psOptionsForBinary->osDestDataSource = papszArgv[i]; } - else if (psOptionsForBinary && - psOptionsForBinary->pszDataSource == nullptr) + else if (psOptionsForBinary && psOptionsForBinary->osDataSource.empty()) { iArgStart = -1; - psOptionsForBinary->pszDataSource = CPLStrdup(papszArgv[i]); + psOptionsForBinary->osDataSource = papszArgv[i]; } else { @@ -7292,26 +7305,23 @@ GDALVectorTranslateOptions *GDALVectorTranslateOptionsNew( if (psOptionsForBinary) { psOptionsForBinary->eAccessMode = psOptions->eAccessMode; - if (!psOptions->osFormat.empty()) - psOptionsForBinary->pszFormat = - CPLStrdup(psOptions->osFormat.c_str()); + psOptionsForBinary->osFormat = psOptions->osFormat; - if (!(CPLTestBool(CSLFetchNameValueDef( - psOptionsForBinary->papszOpenOptions, "NATIVE_DATA", - CSLFetchNameValueDef(psOptionsForBinary->papszOpenOptions, - "@NATIVE_DATA", "TRUE"))))) + if (!(CPLTestBool(psOptionsForBinary->aosOpenOptions.FetchNameValueDef( + "NATIVE_DATA", + psOptionsForBinary->aosOpenOptions.FetchNameValueDef( + "@NATIVE_DATA", "TRUE"))))) { psOptions->bNativeData = false; } if (psOptions->bNativeData && - CSLFetchNameValue(psOptionsForBinary->papszOpenOptions, - "NATIVE_DATA") == nullptr && - CSLFetchNameValue(psOptionsForBinary->papszOpenOptions, - "@NATIVE_DATA") == nullptr) + psOptionsForBinary->aosOpenOptions.FetchNameValue("NATIVE_DATA") == + nullptr && + psOptionsForBinary->aosOpenOptions.FetchNameValue("@NATIVE_DATA") == + nullptr) { - psOptionsForBinary->papszOpenOptions = CSLAddString( - psOptionsForBinary->papszOpenOptions, "@NATIVE_DATA=YES"); + psOptionsForBinary->aosOpenOptions.AddString("@NATIVE_DATA=YES"); } } diff --git a/deps/libgdal/gdal/apps/ogrinfo_bin.cpp b/deps/libgdal/gdal/apps/ogrinfo_bin.cpp index c637db890..392c0a843 100644 --- a/deps/libgdal/gdal/apps/ogrinfo_bin.cpp +++ b/deps/libgdal/gdal/apps/ogrinfo_bin.cpp @@ -44,7 +44,7 @@ static void Usage(bool bIsError, const char *pszErrorMsg = nullptr) { fprintf(bIsError ? stderr : stdout, "Usage: ogrinfo [--help] [--help-general]\n" - " [-json] [-ro] [-q] [-where " + " [-if ] [-json] [-ro] [-q] [-where " "|@f]\n" " [-spat ] [-geomfield " "] " @@ -137,8 +137,9 @@ MAIN_START(argc, argv) else if (psOptionsForBinary->osSQLStatement.empty()) { nFlags |= GDAL_OF_READONLY; - if (GDALIdentifyDriverEx(psOptionsForBinary->osFilename.c_str(), - GDAL_OF_VECTOR, nullptr, nullptr)) + if (GDALIdentifyDriverEx( + psOptionsForBinary->osFilename.c_str(), GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), nullptr)) { bMayRetryUpdateMode = true; } @@ -151,7 +152,8 @@ MAIN_START(argc, argv) else nFlags |= GDAL_OF_UPDATE | GDAL_OF_VERBOSE_ERROR; GDALDataset *poDS = GDALDataset::Open( - psOptionsForBinary->osFilename.c_str(), nFlags, nullptr, + psOptionsForBinary->osFilename.c_str(), nFlags, + psOptionsForBinary->aosAllowInputDrivers.List(), psOptionsForBinary->aosOpenOptions.List(), nullptr); if (poDS == nullptr && !psOptionsForBinary->bReadOnly && @@ -164,14 +166,16 @@ MAIN_START(argc, argv) // read-only mode fails, so retry in update mode poDS = GDALDataset::Open( psOptionsForBinary->osFilename.c_str(), - GDAL_OF_UPDATE | GDAL_OF_VECTOR, nullptr, + GDAL_OF_UPDATE | GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), psOptionsForBinary->aosOpenOptions.List(), nullptr); } else if (!psOptionsForBinary->osSQLStatement.empty()) { poDS = GDALDataset::Open( psOptionsForBinary->osFilename.c_str(), - GDAL_OF_READONLY | GDAL_OF_VECTOR, nullptr, + GDAL_OF_READONLY | GDAL_OF_VECTOR, + psOptionsForBinary->aosAllowInputDrivers.List(), psOptionsForBinary->aosOpenOptions.List(), nullptr); if (poDS != nullptr && psOptionsForBinary->bVerbose) { diff --git a/deps/libgdal/gdal/frmts/hdf4/hdf4dataset.cpp b/deps/libgdal/gdal/frmts/hdf4/hdf4dataset.cpp index 5b87e8a86..0ec2d26bc 100644 --- a/deps/libgdal/gdal/frmts/hdf4/hdf4dataset.cpp +++ b/deps/libgdal/gdal/frmts/hdf4/hdf4dataset.cpp @@ -1360,27 +1360,39 @@ struct HDF4DriverSubdatasetInfo : public GDALSubdatasetInfo m_driverPrefixComponent.append(aosParts[1]); int subdatasetIndex{3}; - const bool hasDriveLetter{ - (strlen(aosParts[2]) == 2 && std::isalpha(aosParts[2][1])) || - (strlen(aosParts[2]) == 1 && std::isalpha(aosParts[2][0]))}; if (iPartsCount >= 4) { + const bool hasDriveLetter{ + (strlen(aosParts[3]) > 1 && + (aosParts[3][0] == '\\' || aosParts[3][0] == '/')) && + ((strlen(aosParts[2]) == 2 && + std::isalpha(aosParts[2][1])) || + (strlen(aosParts[2]) == 1 && + std::isalpha(aosParts[2][0])))}; m_pathComponent = aosParts[2]; - if (hasDriveLetter) + + const bool hasProtocol{m_pathComponent.find("/vsicurl/") != + std::string::npos}; + + if (hasDriveLetter || hasProtocol) { m_pathComponent.append(":"); m_pathComponent.append(aosParts[3]); subdatasetIndex++; } } - m_subdatasetComponent = aosParts[subdatasetIndex]; - // Append any remaining part - for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + if (iPartsCount > subdatasetIndex) { - m_subdatasetComponent.append(":"); - m_subdatasetComponent.append(aosParts[i]); + m_subdatasetComponent = aosParts[subdatasetIndex]; + + // Append any remaining part + for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + { + m_subdatasetComponent.append(":"); + m_subdatasetComponent.append(aosParts[i]); + } } } } diff --git a/deps/libgdal/gdal/frmts/hdf5/gh5_convenience.cpp b/deps/libgdal/gdal/frmts/hdf5/gh5_convenience.cpp index df5695da1..304032e08 100644 --- a/deps/libgdal/gdal/frmts/hdf5/gh5_convenience.cpp +++ b/deps/libgdal/gdal/frmts/hdf5/gh5_convenience.cpp @@ -171,10 +171,44 @@ bool GH5_FetchAttribute(hid_t loc_id, const char *pszAttrName, double &dfResult, H5Aread(hAttr, hAttrNativeType, buf); // Translate to double. - if (H5Tequal(H5T_NATIVE_SHORT, hAttrNativeType)) + if (H5Tequal(H5T_NATIVE_CHAR, hAttrNativeType)) + dfResult = *((char *)buf); + else if (H5Tequal(H5T_NATIVE_SCHAR, hAttrNativeType)) + dfResult = *((signed char *)buf); + else if (H5Tequal(H5T_NATIVE_UCHAR, hAttrNativeType)) + dfResult = *((unsigned char *)buf); + else if (H5Tequal(H5T_NATIVE_SHORT, hAttrNativeType)) dfResult = *((short *)buf); + else if (H5Tequal(H5T_NATIVE_USHORT, hAttrNativeType)) + dfResult = *((unsigned short *)buf); else if (H5Tequal(H5T_NATIVE_INT, hAttrNativeType)) dfResult = *((int *)buf); + else if (H5Tequal(H5T_NATIVE_UINT, hAttrNativeType)) + dfResult = *((unsigned int *)buf); + else if (H5Tequal(H5T_NATIVE_INT64, hAttrNativeType)) + { + const auto nVal = *static_cast(buf); + dfResult = static_cast(nVal); + if (nVal != static_cast(dfResult)) + { + CPLDebug("HDF5", + "Loss of accuracy when reading attribute %s. " + "Value " CPL_FRMT_GIB " will be read as %.18g", + pszAttrName, static_cast(nVal), dfResult); + } + } + else if (H5Tequal(H5T_NATIVE_UINT64, hAttrNativeType)) + { + const auto nVal = *static_cast(buf); + dfResult = static_cast(nVal); + if (nVal != static_cast(dfResult)) + { + CPLDebug("HDF5", + "Loss of accuracy when reading attribute %s. " + "Value " CPL_FRMT_GUIB " will be read as %.18g", + pszAttrName, static_cast(nVal), dfResult); + } + } else if (H5Tequal(H5T_NATIVE_FLOAT, hAttrNativeType)) dfResult = *((float *)buf); else if (H5Tequal(H5T_NATIVE_DOUBLE, hAttrNativeType)) diff --git a/deps/libgdal/gdal/frmts/hdf5/hdf5dataset.cpp b/deps/libgdal/gdal/frmts/hdf5/hdf5dataset.cpp index ab09e1485..e79035342 100644 --- a/deps/libgdal/gdal/frmts/hdf5/hdf5dataset.cpp +++ b/deps/libgdal/gdal/frmts/hdf5/hdf5dataset.cpp @@ -121,27 +121,43 @@ struct HDF5DriverSubdatasetInfo : public GDALSubdatasetInfo m_driverPrefixComponent = aosParts[0]; + std::string part1{aosParts[1]}; + if (!part1.empty() && part1[0] == '"') + { + part1 = part1.substr(1); + } + int subdatasetIndex{2}; const bool hasDriveLetter{ - (strlen(aosParts[1]) == 2 && std::isalpha(aosParts[1][1])) || - (strlen(aosParts[1]) == 1 && std::isalpha(aosParts[1][0]))}; + part1.length() == 1 && std::isalpha(part1.at(0)) && + (strlen(aosParts[2]) > 1 && + (aosParts[2][0] == '\\' || + (aosParts[2][0] == '/' && aosParts[2][1] != '/')))}; + + const bool hasProtocol{part1 == "/vsicurl/http" || + part1 == "/vsicurl/https" || + part1 == "/vsicurl_streaming/http" || + part1 == "/vsicurl_streaming/https"}; m_pathComponent = aosParts[1]; - if (hasDriveLetter) + if (hasDriveLetter || hasProtocol) { m_pathComponent.append(":"); m_pathComponent.append(aosParts[2]); subdatasetIndex++; } - m_subdatasetComponent = aosParts[subdatasetIndex]; - - // Append any remaining part - for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + if (iPartsCount > subdatasetIndex) { - m_subdatasetComponent.append(":"); - m_subdatasetComponent.append(aosParts[i]); + m_subdatasetComponent = aosParts[subdatasetIndex]; + + // Append any remaining part + for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + { + m_subdatasetComponent.append(":"); + m_subdatasetComponent.append(aosParts[i]); + } } } } diff --git a/deps/libgdal/gdal/frmts/hdf5/hdf5multidim.cpp b/deps/libgdal/gdal/frmts/hdf5/hdf5multidim.cpp index 1f759724f..b361224c3 100644 --- a/deps/libgdal/gdal/frmts/hdf5/hdf5multidim.cpp +++ b/deps/libgdal/gdal/frmts/hdf5/hdf5multidim.cpp @@ -1480,11 +1480,15 @@ herr_t HDF5Array::GetAttributesCallback(hid_t hArray, const char *pszObjName, &pszVal, GDALExtendedDataType::CreateString()); CPLError(CE_Warning, CPLE_AppDefined, - "%s attribute value (%s) is not in " + "Array %s: %s attribute value (%s) is " + "not in " "the range of the " - "array data type", - pszObjName, - pszVal ? pszVal : "(null)"); + "array data type (%s)", + self->GetName().c_str(), pszObjName, + pszVal ? pszVal : "(null)", + GDALGetDataTypeName( + self->GetDataType() + .GetNumericDataType())); CPLFree(pszVal); } } diff --git a/deps/libgdal/gdal/frmts/jpegxl/jpegxl.cpp b/deps/libgdal/gdal/frmts/jpegxl/jpegxl.cpp index de2fb71c8..6494d87ea 100644 --- a/deps/libgdal/gdal/frmts/jpegxl/jpegxl.cpp +++ b/deps/libgdal/gdal/frmts/jpegxl/jpegxl.cpp @@ -2743,13 +2743,14 @@ GDALDataset *JPEGXLDataset::CreateCopy(const char *pszFilename, JxlColorEncodingSetToSRGB(&color_encoding, basic_info.num_color_channels == 1 /*is_gray*/); - if (JXL_ENC_SUCCESS != - JxlEncoderSetColorEncoding(encoder.get(), &color_encoding)) - { - CPLError(CE_Failure, CPLE_AppDefined, - "JxlEncoderSetColorEncoding() failed"); - return nullptr; - } + // libjxl until commit + // https://github.com/libjxl/libjxl/commits/c70c9d0bdc03f77d6bd8d9c3c56d4dac1b9b1652 + // needs JxlEncoderSetColorEncoding() + // But post it (308b5f1eed81becac506569080e4490cc486660c, + // "Use chunked frame adapter instead of image bundle in + // EncodeFrame. (#2983)"), this errors out. + CPL_IGNORE_RET_VAL( + JxlEncoderSetColorEncoding(encoder.get(), &color_encoding)); const auto nDataSize = GDALGetDataTypeSizeBytes(eDT); if (nDataSize <= 0 || diff --git a/deps/libgdal/gdal/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp b/deps/libgdal/gdal/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp index 2f688e3eb..e735883ef 100644 --- a/deps/libgdal/gdal/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp +++ b/deps/libgdal/gdal/frmts/kmlsuperoverlay/kmlsuperoverlaydataset.cpp @@ -167,7 +167,7 @@ static void GenerateTiles(const std::string &filename, CPL_UNUSED int zoom, CPLSetThreadLocalConfigOption("GDAL_OPEN_AFTER_COPY", "NO"); /* to prevent CreateCopy() from calling QuietDelete() */ char **papszOptions = - CSLAddNameValue(nullptr, "QUIET_DELETE_ON_CREATE_COPY", "NO"); + CSLAddNameValue(nullptr, "@QUIET_DELETE_ON_CREATE_COPY", "NO"); GDALDataset *outDs = poOutputTileDriver->CreateCopy( filename.c_str(), poTmpDataset, FALSE, papszOptions, nullptr, nullptr); CSLDestroy(papszOptions); diff --git a/deps/libgdal/gdal/frmts/netcdf/netcdfdataset.cpp b/deps/libgdal/gdal/frmts/netcdf/netcdfdataset.cpp index c2ec415ee..a3b5b66d2 100644 --- a/deps/libgdal/gdal/frmts/netcdf/netcdfdataset.cpp +++ b/deps/libgdal/gdal/frmts/netcdf/netcdfdataset.cpp @@ -11236,25 +11236,56 @@ struct NCDFDriverSubdatasetInfo : public GDALSubdatasetInfo m_driverPrefixComponent = aosParts[0]; int subdatasetIndex{2}; + + std::string part1{aosParts[1]}; + if (!part1.empty() && part1[0] == '"') + { + part1 = part1.substr(1); + } + const bool hasDriveLetter{ - (strlen(aosParts[1]) == 2 && std::isalpha(aosParts[1][1])) || - (strlen(aosParts[1]) == 1 && std::isalpha(aosParts[1][0]))}; + (strlen(aosParts[2]) > 1 && + (aosParts[2][0] == '\\' || aosParts[2][0] == '/')) && + part1.length() == 1 && std::isalpha(part1.at(0))}; + + const bool hasProtocol{part1 == "/vsicurl/http" || + part1 == "/vsicurl/https" || + part1 == "/vsicurl_streaming/http" || + part1 == "/vsicurl_streaming/https" || + part1 == "http" || part1 == "https"}; m_pathComponent = aosParts[1]; - if (hasDriveLetter) + if (hasDriveLetter || hasProtocol) { m_pathComponent.append(":"); m_pathComponent.append(aosParts[2]); subdatasetIndex++; } - m_subdatasetComponent = aosParts[subdatasetIndex]; + // Check for bogus paths + if (subdatasetIndex < iPartsCount) + { + m_subdatasetComponent = aosParts[subdatasetIndex]; + + // Append any remaining part + for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + { + m_subdatasetComponent.append(":"); + m_subdatasetComponent.append(aosParts[i]); + } + } - // Append any remaining part - for (int i = subdatasetIndex + 1; i < iPartsCount; ++i) + // Remove quotes from subdataset component + if (!m_subdatasetComponent.empty() && + m_subdatasetComponent[0] == '"') + { + m_subdatasetComponent = m_subdatasetComponent.substr(1); + } + if (m_subdatasetComponent.rfind('"') == + m_subdatasetComponent.length() - 1) { - m_subdatasetComponent.append(":"); - m_subdatasetComponent.append(aosParts[i]); + m_subdatasetComponent = m_subdatasetComponent.substr( + 0, m_subdatasetComponent.length() - 1); } } } @@ -11266,8 +11297,8 @@ static GDALSubdatasetInfo *NCDFDriverGetSubdatasetInfo(const char *pszFileName) { std::unique_ptr info = cpl::make_unique(pszFileName); - if (!info->GetSubdatasetComponent().empty() && - !info->GetPathComponent().empty()) + // Subdataset component can be empty, path cannot. + if (!info->GetPathComponent().empty()) { return info.release(); } diff --git a/deps/libgdal/gdal/frmts/ngsgeoid/ngsgeoiddataset.cpp b/deps/libgdal/gdal/frmts/ngsgeoid/ngsgeoiddataset.cpp index 9de48be56..b15b61acb 100644 --- a/deps/libgdal/gdal/frmts/ngsgeoid/ngsgeoiddataset.cpp +++ b/deps/libgdal/gdal/frmts/ngsgeoid/ngsgeoiddataset.cpp @@ -279,8 +279,9 @@ int NGSGEOIDDataset::GetHeaderInfo(const GByte *pBuffer, return FALSE; /* Grids go over +180 in longitude */ - if (dfSLAT < -90.0 || dfSLAT + nNLAT * dfDLAT > 90.0 || dfWLON < -180.0 || - dfWLON + nNLON * dfDLON > 360.0) + // Test written that way to be robust to NaN values + if (!(dfSLAT >= -90.0 && dfSLAT + nNLAT * dfDLAT <= 90.0 && + dfWLON >= -180.0 && dfWLON + nNLON * dfDLON <= 360.0)) return FALSE; padfGeoTransform[0] = dfWLON - dfDLON / 2; diff --git a/deps/libgdal/gdal/frmts/ogcapi/gdalogcapidataset.cpp b/deps/libgdal/gdal/frmts/ogcapi/gdalogcapidataset.cpp index 868c45e37..3276aa12e 100644 --- a/deps/libgdal/gdal/frmts/ogcapi/gdalogcapidataset.cpp +++ b/deps/libgdal/gdal/frmts/ogcapi/gdalogcapidataset.cpp @@ -760,6 +760,7 @@ bool OGCAPIDataset::InitFromCollection(GDALOpenInfo *poOpenInfo, bool bItemsJson = false; CPLString osSelfURL; + bool bSelfJson = false; for (const auto &oLink : oLinks) { @@ -830,9 +831,17 @@ bool OGCAPIDataset::InitFromCollection(GDALOpenInfo *poOpenInfo, osItemsURL = BuildURL(oLink["href"].ToString()); } } - else if (osRel == "self" && osType == "application/json") + else if (!bSelfJson && osRel == "self") { - osSelfURL = BuildURL(oLink["href"].ToString()); + if (osType == "application/json") + { + bSelfJson = true; + osSelfURL = BuildURL(oLink["href"].ToString()); + } + else if (osType.empty()) + { + osSelfURL = BuildURL(oLink["href"].ToString()); + } } } diff --git a/deps/libgdal/gdal/frmts/stacta/stactadataset.cpp b/deps/libgdal/gdal/frmts/stacta/stactadataset.cpp index 36e3dc7a4..3ce16b0a9 100644 --- a/deps/libgdal/gdal/frmts/stacta/stactadataset.cpp +++ b/deps/libgdal/gdal/frmts/stacta/stactadataset.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -194,6 +195,25 @@ STACTARawRasterBand::STACTARawRasterBand(STACTARawDataset *poDSIn, int nBandIn, m_dfNoData = poProtoBand->GetNoDataValue(&m_bHasNoDataValue); } +/************************************************************************/ +/* STACTARawRasterBand() */ +/************************************************************************/ + +STACTARawRasterBand::STACTARawRasterBand(STACTARawDataset *poDSIn, int nBandIn, + GDALDataType eDT, bool bSetNoData, + double dfNoData) +{ + poDS = poDSIn; + nBand = nBandIn; + eDataType = eDT; + nBlockXSize = 256; + nBlockYSize = 256; + nRasterXSize = poDSIn->GetRasterXSize(); + nRasterYSize = poDSIn->GetRasterYSize(); + m_bHasNoDataValue = bSetNoData; + m_dfNoData = dfNoData; +} + /************************************************************************/ /* GetNoDataValue() */ /************************************************************************/ @@ -390,15 +410,6 @@ CPLErr STACTARawDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, nDTSize < 128 * 1024); - // Avoid probing side car files - std::unique_ptr poSetter; - const CPLString osExt(CPLGetExtension(m_osURLTemplate)); - if (osExt.size() > 0 && osExt.size() <= 3) - { - poSetter.reset(new CPLConfigOptionSetter( - "CPL_VSIL_CURL_ALLOWED_EXTENSIONS", osExt, false)); - } - // Split the request on each metatile that it intersects for (int iY = nMinBlockY; iY <= nMaxBlockY; iY++) { @@ -433,6 +444,12 @@ CPLErr STACTARawDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, m_poMasterDS->m_oCacheTileDS.getPtr(osURL); if (ppoTileDS == nullptr) { + + // Avoid probing side car files + CPLConfigOptionSetter oSetter( + "GDAL_DISABLE_READDIR_ON_OPEN", "EMPTY_DIR", + /* bSetOnlyIfUndefined = */ true); + CPLStringList aosAllowedDrivers; aosAllowedDrivers.AddString("GTiff"); aosAllowedDrivers.AddString("PNG"); @@ -647,7 +664,10 @@ int STACTADataset::Identify(GDALOpenInfo *poOpenInfo) const char *pszHeader = reinterpret_cast(poOpenInfo->pabyHeader); if (strstr(pszHeader, "\"stac_extensions\"") != nullptr && - strstr(pszHeader, "\"tiled-assets\"") != nullptr) + (strstr(pszHeader, "\"tiled-assets\"") != nullptr || + strstr(pszHeader, + "https://stac-extensions.github.io/tiled-assets/") != + nullptr)) { return true; } @@ -872,47 +892,190 @@ bool STACTADataset::Open(GDALOpenInfo *poOpenInfo) poOpenInfo->papszOpenOptions, "SKIP_MISSING_METATILE", CPLGetConfigOption("GDAL_STACTA_SKIP_MISSING_METATILE", "NO"))); - std::unique_ptr poProtoDS; - for (int i = 0; i < static_cast(tmsList.size()); i++) + // STAC 1.1 uses bands instead of eo:bands and raster:bands + const auto oBands = oAssetTemplate.GetArray("bands"); + + // Check if there are both eo:bands and raster:bands extension + // If so, we don't need to fetch a prototype metatile to derive the + // information we need (number of bands, data type and nodata value) + const auto oEoBands = + oBands.IsValid() ? oBands : oAssetTemplate.GetArray("eo:bands"); + const auto oRasterBands = + oBands.IsValid() ? oBands : oAssetTemplate.GetArray("raster:bands"); + + std::vector aeDT; + std::vector adfNoData; + std::vector abSetNoData; + int nExpectedBandCount = 0; + if (oRasterBands.IsValid()) { - // Open a metatile to get mostly its band data type - int nProtoTileCol = 0; - int nProtoTileRow = 0; - auto oIterLimit = oMapLimits.find(tmsList[i].mId); - if (oIterLimit != oMapLimits.end()) + if (oEoBands.IsValid() && oEoBands.Size() != oRasterBands.Size()) { - nProtoTileCol = oIterLimit->second.min_tile_col; - nProtoTileRow = oIterLimit->second.min_tile_row; + CPLError(CE_Warning, CPLE_AppDefined, + "Number of bands in eo:bands and raster:bands is not " + "identical. Ignoring the later"); + } + else + { + nExpectedBandCount = oRasterBands.Size(); + const struct + { + const char *pszStacDataType; + GDALDataType eGDALDataType; + } aDataTypeMapping[] = { + {"int8", GDT_Int8}, + {"int16", GDT_Int16}, + {"int32", GDT_Int32}, + {"int64", GDT_Int64}, + {"uint8", GDT_Byte}, + {"uint16", GDT_UInt16}, + {"uint32", GDT_UInt32}, + {"uint64", GDT_UInt64}, + // float16: 16-bit float; unhandled + {"float32", GDT_Float32}, + {"float64", GDT_Float64}, + {"cint16", GDT_CInt16}, + {"cint32", GDT_CInt32}, + {"cfloat32", GDT_CFloat32}, + {"cfloat64", GDT_CFloat64}, + }; + for (int i = 0; i < nExpectedBandCount; ++i) + { + if (oRasterBands[i].GetType() != CPLJSONObject::Type::Object) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Wrong raster:bands[%d]", i); + return false; + } + const std::string osDataType = + oRasterBands[i].GetString("data_type"); + GDALDataType eDT = GDT_Unknown; + for (const auto &oTuple : aDataTypeMapping) + { + if (osDataType == oTuple.pszStacDataType) + { + eDT = oTuple.eGDALDataType; + break; + } + } + if (eDT == GDT_Unknown) + { + CPLError(CE_Failure, CPLE_AppDefined, + "Wrong raster:bands[%d].data_type = %s", i, + osDataType.c_str()); + return false; + } + aeDT.push_back(eDT); + + const auto oNoData = oRasterBands[i].GetObj("nodata"); + if (oNoData.GetType() == CPLJSONObject::Type::String) + { + const std::string osNoData = oNoData.ToString(); + if (osNoData == "inf") + { + abSetNoData.push_back(true); + adfNoData.push_back( + std::numeric_limits::infinity()); + } + else if (osNoData == "-inf") + { + abSetNoData.push_back(true); + adfNoData.push_back( + -std::numeric_limits::infinity()); + } + else if (osNoData == "nan") + { + abSetNoData.push_back(true); + adfNoData.push_back( + std::numeric_limits::quiet_NaN()); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Invalid raster:bands[%d].nodata = %s", i, + osNoData.c_str()); + abSetNoData.push_back(false); + adfNoData.push_back( + std::numeric_limits::quiet_NaN()); + } + } + else if (oNoData.GetType() == CPLJSONObject::Type::Integer || + oNoData.GetType() == CPLJSONObject::Type::Long || + oNoData.GetType() == CPLJSONObject::Type::Double) + { + abSetNoData.push_back(true); + adfNoData.push_back(oNoData.ToDouble()); + } + else if (!oNoData.IsValid()) + { + abSetNoData.push_back(false); + adfNoData.push_back( + std::numeric_limits::quiet_NaN()); + } + else + { + CPLError(CE_Warning, CPLE_AppDefined, + "Invalid raster:bands[%d].nodata", i); + abSetNoData.push_back(false); + adfNoData.push_back( + std::numeric_limits::quiet_NaN()); + } + } + + CPLAssert(aeDT.size() == abSetNoData.size()); + CPLAssert(adfNoData.size() == abSetNoData.size()); } - const CPLString osURL = - CPLString(osURLTemplate) - .replaceAll("{TileMatrix}", tmsList[i].mId) - .replaceAll("{TileRow}", CPLSPrintf("%d", nProtoTileRow)) - .replaceAll("{TileCol}", CPLSPrintf("%d", nProtoTileCol)); - CPLString osProtoDSName = - (STARTS_WITH(osURL, "http://") || STARTS_WITH(osURL, "https://")) - ? CPLString("/vsicurl/" + osURL) - : osURL; - if (m_bSkipMissingMetaTile) - CPLPushErrorHandler(CPLQuietErrorHandler); - poProtoDS.reset(GDALDataset::Open(osProtoDSName.c_str())); - if (m_bSkipMissingMetaTile) - CPLPopErrorHandler(); - if (poProtoDS != nullptr) + } + + std::unique_ptr poProtoDS; + if (aeDT.empty()) + { + for (int i = 0; i < static_cast(tmsList.size()); i++) { - break; + // Open a metatile to get mostly its band data type + int nProtoTileCol = 0; + int nProtoTileRow = 0; + auto oIterLimit = oMapLimits.find(tmsList[i].mId); + if (oIterLimit != oMapLimits.end()) + { + nProtoTileCol = oIterLimit->second.min_tile_col; + nProtoTileRow = oIterLimit->second.min_tile_row; + } + const CPLString osURL = + CPLString(osURLTemplate) + .replaceAll("{TileMatrix}", tmsList[i].mId) + .replaceAll("{TileRow}", CPLSPrintf("%d", nProtoTileRow)) + .replaceAll("{TileCol}", CPLSPrintf("%d", nProtoTileCol)); + CPLString osProtoDSName = (STARTS_WITH(osURL, "http://") || + STARTS_WITH(osURL, "https://")) + ? CPLString("/vsicurl/" + osURL) + : osURL; + CPLConfigOptionSetter oSetter("GDAL_DISABLE_READDIR_ON_OPEN", + "EMPTY_DIR", + /* bSetOnlyIfUndefined = */ true); + if (m_bSkipMissingMetaTile) + CPLPushErrorHandler(CPLQuietErrorHandler); + poProtoDS.reset(GDALDataset::Open(osProtoDSName.c_str())); + if (m_bSkipMissingMetaTile) + CPLPopErrorHandler(); + if (poProtoDS != nullptr) + { + break; + } + if (!m_bSkipMissingMetaTile) + { + CPLError(CE_Failure, CPLE_OpenFailed, "Cannot open %s", + osURL.c_str()); + return false; + } } - if (!m_bSkipMissingMetaTile) + if (poProtoDS == nullptr) { - CPLError(CE_Failure, CPLE_OpenFailed, "Cannot open %s", - osURL.c_str()); + CPLError(CE_Failure, CPLE_AppDefined, + "Cannot find prototype dataset"); return false; } - } - if (poProtoDS == nullptr) - { - CPLError(CE_Failure, CPLE_AppDefined, "Cannot find prototype dataset"); - return false; + nExpectedBandCount = poProtoDS->GetRasterCount(); } // Iterate over tile matrices to create corresponding STACTARawDataset @@ -936,8 +1099,8 @@ bool STACTADataset::Open(GDALOpenInfo *poOpenInfo) continue; } auto poRawDS = cpl::make_unique(); - if (!poRawDS->InitRaster(poProtoDS.get(), poTMS.get(), tmsList[i].mId, - oTM, oMapLimits)) + if (!poRawDS->InitRaster(poProtoDS.get(), aeDT, abSetNoData, adfNoData, + poTMS.get(), tmsList[i].mId, oTM, oMapLimits)) { return false; } @@ -1011,24 +1174,71 @@ bool STACTADataset::Open(GDALOpenInfo *poOpenInfo) } // Create main bands - const auto oEoBands = oAssetTemplate.GetArray("eo:bands"); for (int i = 0; i < m_poDS->GetRasterCount(); i++) { auto poSrcBand = m_poDS->GetRasterBand(i + 1); auto poBand = new STACTARasterBand(this, i + 1, poSrcBand); - if (oEoBands.IsValid() && - oEoBands.Size() == poProtoDS->GetRasterCount()) + if (oEoBands.IsValid() && oEoBands.Size() == nExpectedBandCount) { // Set band metadata if (oEoBands[i].GetType() == CPLJSONObject::Type::Object) { for (const auto &oItem : oEoBands[i].GetChildren()) { - poBand->GDALRasterBand::SetMetadataItem( - oItem.GetName().c_str(), oItem.ToString().c_str()); + if (oBands.IsValid()) + { + // STAC 1.1 + if (STARTS_WITH(oItem.GetName().c_str(), "eo:")) + { + poBand->GDALRasterBand::SetMetadataItem( + oItem.GetName().c_str() + strlen("eo:"), + oItem.ToString().c_str()); + } + else if (oItem.GetName() != "data_type" && + oItem.GetName() != "nodata" && + oItem.GetName() != "unit" && + oItem.GetName() != "raster:scale" && + oItem.GetName() != "raster:offset" && + oItem.GetName() != "raster:bits_per_sample") + { + poBand->GDALRasterBand::SetMetadataItem( + oItem.GetName().c_str(), + oItem.ToString().c_str()); + } + } + else + { + // STAC 1.0 + poBand->GDALRasterBand::SetMetadataItem( + oItem.GetName().c_str(), oItem.ToString().c_str()); + } } } } + if (oRasterBands.IsValid() && + oRasterBands.Size() == nExpectedBandCount && + oRasterBands[i].GetType() == CPLJSONObject::Type::Object) + { + poBand->m_osUnit = oRasterBands[i].GetString("unit"); + const double dfScale = oRasterBands[i].GetDouble( + oBands.IsValid() ? "raster:scale" : "scale"); + if (dfScale != 0) + poBand->m_dfScale = dfScale; + poBand->m_dfOffset = oRasterBands[i].GetDouble( + oBands.IsValid() ? "raster:offset" : "offset"); + const int nBitsPerSample = oRasterBands[i].GetInteger( + oBands.IsValid() ? "raster:bits_per_sample" + : "bits_per_sample"); + if (((nBitsPerSample >= 1 && nBitsPerSample <= 7) && + poBand->GetRasterDataType() == GDT_Byte) || + ((nBitsPerSample >= 9 && nBitsPerSample <= 15) && + poBand->GetRasterDataType() == GDT_UInt16)) + { + poBand->GDALRasterBand::SetMetadataItem( + "NBITS", CPLSPrintf("%d", nBitsPerSample), + "IMAGE_STRUCTURE"); + } + } SetBand(i + 1, poBand); } @@ -1044,11 +1254,20 @@ bool STACTADataset::Open(GDALOpenInfo *poOpenInfo) } } - const char *pszInterleave = - poProtoDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE"); - GDALDataset::SetMetadataItem("INTERLEAVE", - pszInterleave ? pszInterleave : "PIXEL", - "IMAGE_STRUCTURE"); + if (poProtoDS) + { + const char *pszInterleave = + poProtoDS->GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE"); + GDALDataset::SetMetadataItem("INTERLEAVE", + pszInterleave ? pszInterleave : "PIXEL", + "IMAGE_STRUCTURE"); + } + else + { + // A bit bold to assume that, but that should be a reasonable + // setting + GDALDataset::SetMetadataItem("INTERLEAVE", "PIXEL", "IMAGE_STRUCTURE"); + } m_bDownloadWholeMetaTile = CPLTestBool(CSLFetchNameValueDef( poOpenInfo->papszOpenOptions, "WHOLE_METATILE", "NO")); @@ -1082,6 +1301,9 @@ CPLErr STACTADataset::FlushCache(bool bAtClosing) /************************************************************************/ bool STACTARawDataset::InitRaster(GDALDataset *poProtoDS, + const std::vector &aeDT, + const std::vector &abSetNoData, + const std::vector &adfNoData, const gdal::TileMatrixSet *poTMS, const std::string &osTMId, const gdal::TileMatrixSet::TileMatrix &oTM, @@ -1102,11 +1324,23 @@ bool STACTARawDataset::InitRaster(GDALDataset *poProtoDS, nRasterXSize = nMatrixWidth * m_nMetaTileWidth; nRasterYSize = nMatrixHeight * m_nMetaTileHeight; - for (int i = 0; i < poProtoDS->GetRasterCount(); i++) + if (poProtoDS) { - auto poProtoBand = poProtoDS->GetRasterBand(i + 1); - auto poBand = new STACTARawRasterBand(this, i + 1, poProtoBand); - SetBand(i + 1, poBand); + for (int i = 0; i < poProtoDS->GetRasterCount(); i++) + { + auto poProtoBand = poProtoDS->GetRasterBand(i + 1); + auto poBand = new STACTARawRasterBand(this, i + 1, poProtoBand); + SetBand(i + 1, poBand); + } + } + else + { + for (int i = 0; i < static_cast(aeDT.size()); i++) + { + auto poBand = new STACTARawRasterBand(this, i + 1, aeDT[i], + abSetNoData[i], adfNoData[i]); + SetBand(i + 1, poBand); + } } CPLString osCRS = poTMS->crs().c_str(); diff --git a/deps/libgdal/gdal/frmts/stacta/stactadataset.h b/deps/libgdal/gdal/frmts/stacta/stactadataset.h index bd0bf3223..af0afbb59 100644 --- a/deps/libgdal/gdal/frmts/stacta/stactadataset.h +++ b/deps/libgdal/gdal/frmts/stacta/stactadataset.h @@ -107,9 +107,13 @@ class STACTADataset final : public GDALPamDataset class STACTARasterBand final : public GDALRasterBand { + friend class STACTADataset; GDALColorInterp m_eColorInterp = GCI_Undefined; int m_bHasNoDataValue = false; double m_dfNoData = 0; + double m_dfScale = 1.0; + double m_dfOffset = 0.0; + std::string m_osUnit{}; public: STACTARasterBand(STACTADataset *poDSIn, int nBandIn, @@ -125,6 +129,22 @@ class STACTARasterBand final : public GDALRasterBand int GetOverviewCount() override; GDALRasterBand *GetOverview(int nIdx) override; double GetNoDataValue(int *pbHasNoData = nullptr) override; + const char *GetUnitType() override + { + return m_osUnit.c_str(); + } + double GetScale(int *pbHasValue = nullptr) override + { + if (pbHasValue) + *pbHasValue = m_dfScale != 1.0; + return m_dfScale; + } + double GetOffset(int *pbHasValue = nullptr) override + { + if (pbHasValue) + *pbHasValue = m_dfOffset != 0.0; + return m_dfOffset; + } }; /************************************************************************/ @@ -149,8 +169,11 @@ class STACTARawDataset final : public GDALDataset OGRSpatialReference m_oSRS{}; public: - bool InitRaster(GDALDataset *poProtoDS, const gdal::TileMatrixSet *poTMS, - const std::string &osTMId, + bool InitRaster(GDALDataset *poProtoDS, + const std::vector &aeDT, + const std::vector &abSetNoData, + const std::vector &adfNoData, + const gdal::TileMatrixSet *poTMS, const std::string &osTMId, const gdal::TileMatrixSet::TileMatrix &oTM, const std::map &oMapLimits); @@ -182,6 +205,10 @@ class STACTARawRasterBand final : public GDALRasterBand public: STACTARawRasterBand(STACTARawDataset *poDSIn, int nBandIn, GDALRasterBand *poProtoBand); + + STACTARawRasterBand(STACTARawDataset *poDSIn, int nBandIn, GDALDataType eDT, + bool bSetNoData, double dfNoData); + CPLErr IReadBlock(int nBlockXOff, int nBlockYOff, void *pImage) override; CPLErr IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, void *pData, int nBufXSize, int nBufYSize, diff --git a/deps/libgdal/gdal/frmts/vrt/vrtsources.cpp b/deps/libgdal/gdal/frmts/vrt/vrtsources.cpp index 8ad88172a..7b0832b1d 100644 --- a/deps/libgdal/gdal/frmts/vrt/vrtsources.cpp +++ b/deps/libgdal/gdal/frmts/vrt/vrtsources.cpp @@ -2533,6 +2533,21 @@ static inline bool hasZeroByte(uint32_t v) /* RasterIOProcessNoData() */ /************************************************************************/ +namespace +{ +template struct VRTWorkBufferReleaser +{ + std::vector &m_oVector; + explicit VRTWorkBufferReleaser(std::vector &oVector) : m_oVector(oVector) + { + } + ~VRTWorkBufferReleaser() + { + std::vector().swap(m_oVector); + } +}; +} // namespace + // This method is an optimization of the generic RasterIOInternal() // that deals with a VRTComplexSource with only a NODATA value in it, and // no other processing flags. @@ -2549,6 +2564,12 @@ CPLErr VRTComplexSource::RasterIOProcessNoData( CPLAssert(m_nProcessingFlags == PROCESSING_FLAG_NODATA); CPLAssert(GDALIsValueInRange(m_dfNoDataValue)); + // WARNING: we need to make sure to clear that buffer in all exit + // paths otherwise when using many sources, we can easily exhaust RAM. + // That buffer should be at the dataset level, not the source, but we + // can't fix that in 3.8 branch without breaking ABI... + VRTWorkBufferReleaser oWrkBufferReleaser(m_abyWrkBuffer); + /* -------------------------------------------------------------------- */ /* Read into a temporary buffer. */ /* -------------------------------------------------------------------- */ @@ -2733,6 +2754,14 @@ CPLErr VRTComplexSource::RasterIOInternal( const int nWordSize = GDALGetDataTypeSizeBytes(eWrkDataType); assert(nWordSize != 0); + // WARNING: we need to make sure to clear those buffers in all exit + // paths otherwise when using many sources, we can easily exhaust RAM. + // Those buffers should be at the dataset level, not the source, but we + // can't fix that in 3.8 branch without breaking ABI... + VRTWorkBufferReleaser oWrkBufferReleaser(m_abyWrkBuffer); + VRTWorkBufferReleaser oWrkBufferMaskReleaser( + m_abyWrkBufferMask); + // If no explicit is set, but UseMaskBand is set, and the band // has a nodata value, then use it as if it was set as int bNoDataSet = (m_nProcessingFlags & PROCESSING_FLAG_NODATA) != 0; diff --git a/deps/libgdal/gdal/gcore/gdal_priv.h b/deps/libgdal/gdal/gcore/gdal_priv.h index 7ee8966fb..3b326d2bc 100644 --- a/deps/libgdal/gdal/gcore/gdal_priv.h +++ b/deps/libgdal/gdal/gcore/gdal_priv.h @@ -1844,6 +1844,10 @@ class CPL_DLL GDALDriver : public GDALMajorObject int bStrict, CSLConstList papszOptions, GDALProgressFunc pfnProgress, void *pProgressData); + + CPLErr QuietDeleteForCreateCopy(const char *pszFilename, + GDALDataset *poSrcDS); + //! @endcond static CPLErr QuietDelete(const char *pszName, CSLConstList papszAllowedDrivers = nullptr); diff --git a/deps/libgdal/gdal/gcore/gdal_version.h b/deps/libgdal/gdal/gcore/gdal_version.h index 55d30b073..f13fa9d44 100644 --- a/deps/libgdal/gdal/gcore/gdal_version.h +++ b/deps/libgdal/gdal/gcore/gdal_version.h @@ -7,7 +7,7 @@ #ifndef GDAL_VERSION_MAJOR # define GDAL_VERSION_MAJOR 3 # define GDAL_VERSION_MINOR 8 -# define GDAL_VERSION_REV 1 +# define GDAL_VERSION_REV 2 # define GDAL_VERSION_BUILD 0 #endif @@ -24,9 +24,9 @@ #if !defined(DO_NOT_DEFINE_GDAL_DATE_NAME) #ifndef GDAL_RELEASE_DATE -# define GDAL_RELEASE_DATE 20231128 +# define GDAL_RELEASE_DATE 20231612 #endif #ifndef GDAL_RELEASE_NAME -# define GDAL_RELEASE_NAME "3.8.1" +# define GDAL_RELEASE_NAME "3.8.2" #endif #endif diff --git a/deps/libgdal/gdal/gcore/gdal_version_full/gdal_version.h b/deps/libgdal/gdal/gcore/gdal_version_full/gdal_version.h index 55d30b073..f13fa9d44 100644 --- a/deps/libgdal/gdal/gcore/gdal_version_full/gdal_version.h +++ b/deps/libgdal/gdal/gcore/gdal_version_full/gdal_version.h @@ -7,7 +7,7 @@ #ifndef GDAL_VERSION_MAJOR # define GDAL_VERSION_MAJOR 3 # define GDAL_VERSION_MINOR 8 -# define GDAL_VERSION_REV 1 +# define GDAL_VERSION_REV 2 # define GDAL_VERSION_BUILD 0 #endif @@ -24,9 +24,9 @@ #if !defined(DO_NOT_DEFINE_GDAL_DATE_NAME) #ifndef GDAL_RELEASE_DATE -# define GDAL_RELEASE_DATE 20231128 +# define GDAL_RELEASE_DATE 20231612 #endif #ifndef GDAL_RELEASE_NAME -# define GDAL_RELEASE_NAME "3.8.1" +# define GDAL_RELEASE_NAME "3.8.2" #endif #endif diff --git a/deps/libgdal/gdal/gcore/gdaldriver.cpp b/deps/libgdal/gdal/gcore/gdaldriver.cpp index 96e85a74d..704bb31bd 100644 --- a/deps/libgdal/gdal/gcore/gdaldriver.cpp +++ b/deps/libgdal/gdal/gcore/gdaldriver.cpp @@ -980,6 +980,112 @@ void GDALDriver::DefaultCopyMetadata(GDALDataset *poSrcDS, GDALDataset *poDstDS, } CSLDestroy(papszSrcMDD); } + +/************************************************************************/ +/* QuietDeleteForCreateCopy() */ +/************************************************************************/ + +CPLErr GDALDriver::QuietDeleteForCreateCopy(const char *pszFilename, + GDALDataset *poSrcDS) +{ + // Someone issuing CreateCopy("foo.tif") on a + // memory driver doesn't expect files with those names to be deleted + // on a file system... + // This is somewhat messy. Ideally there should be a way for the + // driver to overload the default behavior + if (!EQUAL(GetDescription(), "MEM") && !EQUAL(GetDescription(), "Memory") && + // Also exclude database formats for which there's no file list + // and whose opening might be slow (GeoRaster in particular) + !EQUAL(GetDescription(), "GeoRaster") && + !EQUAL(GetDescription(), "PostGISRaster")) + { + /* -------------------------------------------------------------------- + */ + /* Establish list of files of output dataset if it already + * exists. */ + /* -------------------------------------------------------------------- + */ + std::set oSetExistingDestFiles; + { + CPLPushErrorHandler(CPLQuietErrorHandler); + const char *const apszAllowedDrivers[] = {GetDescription(), + nullptr}; + auto poExistingOutputDS = + std::unique_ptr(GDALDataset::Open( + pszFilename, GDAL_OF_RASTER, apszAllowedDrivers)); + if (poExistingOutputDS) + { + char **papszFileList = poExistingOutputDS->GetFileList(); + for (char **papszIter = papszFileList; papszIter && *papszIter; + ++papszIter) + { + oSetExistingDestFiles.insert( + CPLString(*papszIter).replaceAll('\\', '/')); + } + CSLDestroy(papszFileList); + } + CPLPopErrorHandler(); + } + + /* -------------------------------------------------------------------- + */ + /* Check if the source dataset shares some files with the dest + * one.*/ + /* -------------------------------------------------------------------- + */ + std::set oSetExistingDestFilesFoundInSource; + if (!oSetExistingDestFiles.empty()) + { + CPLPushErrorHandler(CPLQuietErrorHandler); + // We need to reopen in a temporary dataset for the particular + // case of overwritten a .tif.ovr file from a .tif + // If we probe the file list of the .tif, it will then open the + // .tif.ovr ! + const char *const apszAllowedDrivers[] = { + poSrcDS->GetDriver() ? poSrcDS->GetDriver()->GetDescription() + : nullptr, + nullptr}; + auto poSrcDSTmp = std::unique_ptr(GDALDataset::Open( + poSrcDS->GetDescription(), GDAL_OF_RASTER, apszAllowedDrivers)); + if (poSrcDSTmp) + { + char **papszFileList = poSrcDSTmp->GetFileList(); + for (char **papszIter = papszFileList; papszIter && *papszIter; + ++papszIter) + { + CPLString osFilename(*papszIter); + osFilename.replaceAll('\\', '/'); + if (oSetExistingDestFiles.find(osFilename) != + oSetExistingDestFiles.end()) + { + oSetExistingDestFilesFoundInSource.insert(osFilename); + } + } + CSLDestroy(papszFileList); + } + CPLPopErrorHandler(); + } + + // If the source file(s) and the dest one share some files in + // common, only remove the files that are *not* in common + if (!oSetExistingDestFilesFoundInSource.empty()) + { + for (const std::string &osFilename : oSetExistingDestFiles) + { + if (oSetExistingDestFilesFoundInSource.find(osFilename) == + oSetExistingDestFilesFoundInSource.end()) + { + VSIUnlink(osFilename.c_str()); + } + } + } + + QuietDelete(pszFilename); + } + + return CE_None; +} + //! @endcond /************************************************************************/ @@ -1165,109 +1271,17 @@ GDALDataset *GDALDriver::CreateCopy(const char *pszFilename, /* -------------------------------------------------------------------- */ const bool bAppendSubdataset = CPLFetchBool(papszOptions, "APPEND_SUBDATASET", false); - // Note: QUIET_DELETE_ON_CREATE_COPY is set to NO by the KMLSuperOverlay - // driver when writing a .kmz file + // Note: @QUIET_DELETE_ON_CREATE_COPY is set to NO by the KMLSuperOverlay + // driver when writing a .kmz file. Also by GDALTranslate() if it has + // already done a similar job. if (!bAppendSubdataset && - CPLFetchBool(papszOptions, "QUIET_DELETE_ON_CREATE_COPY", true)) + CPLFetchBool(papszOptions, "@QUIET_DELETE_ON_CREATE_COPY", true)) { - // Someone issuing CreateCopy("foo.tif") on a - // memory driver doesn't expect files with those names to be deleted - // on a file system... - // This is somewhat messy. Ideally there should be a way for the - // driver to overload the default behavior - if (!EQUAL(GetDescription(), "MEM") && - !EQUAL(GetDescription(), "Memory")) - { - /* -------------------------------------------------------------------- - */ - /* Establish list of files of output dataset if it already - * exists. */ - /* -------------------------------------------------------------------- - */ - std::set oSetExistingDestFiles; - { - CPLPushErrorHandler(CPLQuietErrorHandler); - const char *const apszAllowedDrivers[] = {GetDescription(), - nullptr}; - auto poExistingOutputDS = - std::unique_ptr(GDALDataset::Open( - pszFilename, GDAL_OF_RASTER, apszAllowedDrivers)); - if (poExistingOutputDS) - { - char **papszFileList = poExistingOutputDS->GetFileList(); - for (char **papszIter = papszFileList; - papszIter && *papszIter; ++papszIter) - { - oSetExistingDestFiles.insert( - CPLString(*papszIter).replaceAll('\\', '/')); - } - CSLDestroy(papszFileList); - } - CPLPopErrorHandler(); - } - - /* -------------------------------------------------------------------- - */ - /* Check if the source dataset shares some files with the dest - * one.*/ - /* -------------------------------------------------------------------- - */ - std::set oSetExistingDestFilesFoundInSource; - if (!oSetExistingDestFiles.empty()) - { - CPLPushErrorHandler(CPLQuietErrorHandler); - // We need to reopen in a temporary dataset for the particular - // case of overwritten a .tif.ovr file from a .tif - // If we probe the file list of the .tif, it will then open the - // .tif.ovr ! - const char *const apszAllowedDrivers[] = { - poSrcDS->GetDriver() - ? poSrcDS->GetDriver()->GetDescription() - : nullptr, - nullptr}; - auto poSrcDSTmp = std::unique_ptr( - GDALDataset::Open(poSrcDS->GetDescription(), GDAL_OF_RASTER, - apszAllowedDrivers)); - if (poSrcDSTmp) - { - char **papszFileList = poSrcDSTmp->GetFileList(); - for (char **papszIter = papszFileList; - papszIter && *papszIter; ++papszIter) - { - CPLString osFilename(*papszIter); - osFilename.replaceAll('\\', '/'); - if (oSetExistingDestFiles.find(osFilename) != - oSetExistingDestFiles.end()) - { - oSetExistingDestFilesFoundInSource.insert( - osFilename); - } - } - CSLDestroy(papszFileList); - } - CPLPopErrorHandler(); - } - - // If the source file(s) and the dest one share some files in - // common, only remove the files that are *not* in common - if (!oSetExistingDestFilesFoundInSource.empty()) - { - for (const std::string &osFilename : oSetExistingDestFiles) - { - if (oSetExistingDestFilesFoundInSource.find(osFilename) == - oSetExistingDestFilesFoundInSource.end()) - { - VSIUnlink(osFilename.c_str()); - } - } - } - - QuietDelete(pszFilename); - } + QuietDeleteForCreateCopy(pszFilename, poSrcDS); } int iIdxQuietDeleteOnCreateCopy = - CSLPartialFindString(papszOptions, "QUIET_DELETE_ON_CREATE_COPY="); + CSLPartialFindString(papszOptions, "@QUIET_DELETE_ON_CREATE_COPY="); if (iIdxQuietDeleteOnCreateCopy >= 0) { if (papszOptionsToDelete == nullptr) diff --git a/deps/libgdal/gdal/ogr/ogr_proj_p.cpp b/deps/libgdal/gdal/ogr/ogr_proj_p.cpp index a27a84fe1..f47691e76 100644 --- a/deps/libgdal/gdal/ogr/ogr_proj_p.cpp +++ b/deps/libgdal/gdal/ogr/ogr_proj_p.cpp @@ -35,14 +35,6 @@ #include "proj.h" -#if defined(__MACH__) && defined(__APPLE__) && defined(HAVE_PTHREAD_ATFORK) -// Works around a weird issue with GDAL, numpy and python threading on -// Mac. There is definitely something not understood, but as using pthread_atfork() -// is just an optimization, just disable it. -// Cf https://github.com/OSGeo/gdal/issues/8497 for details -#undef HAVE_PTHREAD_ATFORK -#endif - #ifndef _WIN32 #include #include @@ -121,7 +113,17 @@ struct OSRPJContextHolder #endif { #if HAVE_PTHREAD_ATFORK - pthread_atfork(nullptr, nullptr, ForkOccurred); + static std::once_flag flag; + std::call_once( + flag, + []() + { + if (pthread_atfork(nullptr, nullptr, ForkOccurred) != 0) + { + CPLError(CE_Failure, CPLE_OutOfMemory, + "pthread_atfork() in ogr_proj_p failed"); + } + }); #endif init(); } diff --git a/deps/libgdal/gdal/ogr/ogr_wkb.cpp b/deps/libgdal/gdal/ogr/ogr_wkb.cpp index 3847c2b03..f2e2aa5ff 100644 --- a/deps/libgdal/gdal/ogr/ogr_wkb.cpp +++ b/deps/libgdal/gdal/ogr/ogr_wkb.cpp @@ -642,84 +642,48 @@ static bool OGRWKBIntersectsPessimistic(const GByte *data, const size_t size, const int nDim = 2 + (OGR_GT_HasZ(eGeometryType) ? 1 : 0) + (OGR_GT_HasM(eGeometryType) ? 1 : 0); - if (eFlatType == wkbLineString || eFlatType == wkbCircularString) - { - return OGRWKBIntersectsPointSequencePessimistic( - data, size, eByteOrder, nDim, iOffsetInOut, sEnvelope, bErrorOut); - } - - if (eFlatType == wkbPolygon) - { - return OGRWKBIntersectsRingSequencePessimistic( - data, size, eByteOrder, nDim, iOffsetInOut, sEnvelope, bErrorOut); - } - - if (eFlatType == wkbMultiLineString) + if (eFlatType == wkbPoint) { - const uint32_t nParts = - OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); - if (nParts > (size - iOffsetInOut) / MIN_WKB_SIZE) - { - bErrorOut = true; + if (size - iOffsetInOut < nDim * sizeof(double)) return false; - } - for (uint32_t k = 0; k < nParts; k++) + double dfX = 0; + double dfY = 0; + memcpy(&dfX, data + iOffsetInOut, sizeof(double)); + memcpy(&dfY, data + iOffsetInOut + sizeof(double), sizeof(double)); + iOffsetInOut += nDim * sizeof(double); + if (OGR_SWAP(eByteOrder)) { - if (iOffsetInOut + MIN_WKB_SIZE > size) - { - bErrorOut = true; - return false; - } - iOffsetInOut += WKB_PREFIX_SIZE; - if (OGRWKBIntersectsPointSequencePessimistic(data, size, eByteOrder, - nDim, iOffsetInOut, - sEnvelope, bErrorOut)) - { - return true; - } - else if (bErrorOut) - { - return false; - } + CPL_SWAP64PTR(&dfX); + CPL_SWAP64PTR(&dfY); } - return false; - } - - if (eFlatType == wkbMultiPolygon) - { - const uint32_t nParts = - OGRWKBReadUInt32AtOffset(data, eByteOrder, iOffsetInOut); - if (nParts > (size - iOffsetInOut) / MIN_WKB_SIZE) + if (std::isnan(dfX)) { - bErrorOut = true; return false; } - for (uint32_t k = 0; k < nParts; k++) + else { - if (iOffsetInOut + MIN_WKB_SIZE > size) - { - bErrorOut = true; - return false; - } - CPLAssert(data[iOffsetInOut] == eByteOrder); - iOffsetInOut += WKB_PREFIX_SIZE; - if (OGRWKBIntersectsRingSequencePessimistic(data, size, eByteOrder, - nDim, iOffsetInOut, - sEnvelope, bErrorOut)) - { - return true; - } - else if (bErrorOut) - { - return false; - } + return dfX >= sEnvelope.MinX && dfX <= sEnvelope.MaxX && + dfY >= sEnvelope.MinY && dfY <= sEnvelope.MaxY; } - return false; } - if (eFlatType == wkbGeometryCollection || eFlatType == wkbCompoundCurve || - eFlatType == wkbCurvePolygon || eFlatType == wkbMultiCurve || - eFlatType == wkbMultiSurface) + if (eFlatType == wkbLineString || eFlatType == wkbCircularString) + { + return OGRWKBIntersectsPointSequencePessimistic( + data, size, eByteOrder, nDim, iOffsetInOut, sEnvelope, bErrorOut); + } + + if (eFlatType == wkbPolygon || eFlatType == wkbTriangle) + { + return OGRWKBIntersectsRingSequencePessimistic( + data, size, eByteOrder, nDim, iOffsetInOut, sEnvelope, bErrorOut); + } + + if (eFlatType == wkbMultiPoint || eFlatType == wkbMultiLineString || + eFlatType == wkbMultiPolygon || eFlatType == wkbGeometryCollection || + eFlatType == wkbCompoundCurve || eFlatType == wkbCurvePolygon || + eFlatType == wkbMultiCurve || eFlatType == wkbMultiSurface || + eFlatType == wkbPolyhedralSurface || eFlatType == wkbTIN) { if (nRec == 128) { diff --git a/deps/libgdal/gdal/ogr/ogrgeometryfactory.cpp b/deps/libgdal/gdal/ogr/ogrgeometryfactory.cpp index 1f61fca15..d8db826c0 100644 --- a/deps/libgdal/gdal/ogr/ogrgeometryfactory.cpp +++ b/deps/libgdal/gdal/ogr/ogrgeometryfactory.cpp @@ -2916,6 +2916,22 @@ static void AddSimpleGeomToMulti(OGRGeometryCollection *poMulti, } #endif // #ifdef HAVE_GEOS +/************************************************************************/ +/* WrapPointDateLine() */ +/************************************************************************/ + +static void WrapPointDateLine(OGRPoint *poPoint) +{ + if (poPoint->getX() > 180) + { + poPoint->setX(fmod(poPoint->getX() + 180, 360) - 180); + } + else if (poPoint->getX() < -180) + { + poPoint->setX(-(fmod(-poPoint->getX() + 180, 360) - 180)); + } +} + /************************************************************************/ /* CutGeometryOnDateLineAndAddToMulti() */ /************************************************************************/ @@ -2927,6 +2943,14 @@ static void CutGeometryOnDateLineAndAddToMulti(OGRGeometryCollection *poMulti, const OGRwkbGeometryType eGeomType = wkbFlatten(poGeom->getGeometryType()); switch (eGeomType) { + case wkbPoint: + { + auto poPoint = poGeom->toPoint()->clone(); + WrapPointDateLine(poPoint); + poMulti->addGeometryDirectly(poPoint); + break; + } + case wkbPolygon: case wkbLineString: { @@ -3913,19 +3937,18 @@ OGRGeometry *OGRGeometryFactory::transformWithOptions( } // TODO and we should probably also test that the axis order + data axis // mapping is long-lat... - const OGRwkbGeometryType eType = wkbFlatten(poDstGeom->getGeometryType()); if (eType == wkbPoint) { OGRPoint *poDstPoint = poDstGeom->toPoint(); - if (poDstPoint->getX() > 180) - { - poDstPoint->setX(fmod(poDstPoint->getX() + 180, 360) - 180); - } - else if (poDstPoint->getX() < -180) + WrapPointDateLine(poDstPoint); + } + else if (eType == wkbMultiPoint) + { + for (auto *poDstPoint : *(poDstGeom->toMultiPoint())) { - poDstPoint->setX(-(fmod(-poDstPoint->getX() + 180, 360) - 180)); + WrapPointDateLine(poDstPoint); } } else diff --git a/deps/libgdal/gdal/ogr/ogrsf_frmts/pmtiles/ogrpmtileswriterdataset.cpp b/deps/libgdal/gdal/ogr/ogrsf_frmts/pmtiles/ogrpmtileswriterdataset.cpp index ffb07ccee..2562c641c 100644 --- a/deps/libgdal/gdal/ogr/ogrsf_frmts/pmtiles/ogrpmtileswriterdataset.cpp +++ b/deps/libgdal/gdal/ogr/ogrsf_frmts/pmtiles/ogrpmtileswriterdataset.cpp @@ -92,11 +92,12 @@ bool OGRPMTilesWriterDataset::Create(const char *pszFilename, // a way that corresponds to the "clustered" mode, that is // "offsets are either contiguous with the previous offset+length, or // refer to a lesser offset, when writing with deduplication." - std::string osTmpFile(std::string(pszFilename) + ".tmp.mbtiles"); + std::string osTmpFile(pszFilename); if (!VSIIsLocal(pszFilename)) { osTmpFile = CPLGenerateTempFilename(CPLGetFilename(pszFilename)); } + osTmpFile += ".tmp.mbtiles"; if (!aosOptions.FetchNameValue("NAME")) aosOptions.SetNameValue("NAME", CPLGetBasename(pszFilename)); diff --git a/deps/libgdal/gdal/ogr/ogrspatialreference.cpp b/deps/libgdal/gdal/ogr/ogrspatialreference.cpp index 41c8e8077..256b06e51 100644 --- a/deps/libgdal/gdal/ogr/ogrspatialreference.cpp +++ b/deps/libgdal/gdal/ogr/ogrspatialreference.cpp @@ -4120,12 +4120,34 @@ OGRErr OGRSpatialReference::importFromUrl(const char *pszUrl) /* -------------------------------------------------------------------- */ CPLErrorReset(); - const char *pszHeaders = "HEADERS=Accept: application/x-ogcwkt"; + std::string osUrl(pszUrl); + // We have historically supported "http://spatialreference.org/ref/AUTHNAME/CODE/" + // as a valid URL since we used a "Accept: application/x-ogcwkt" header + // to query WKT. To allow a static server to be used, rather append a + // "ogcwkt/" suffix. + for (const char *pszPrefix : {"https://spatialreference.org/ref/", + "http://spatialreference.org/ref/"}) + { + if (STARTS_WITH(pszUrl, pszPrefix)) + { + const CPLStringList aosTokens( + CSLTokenizeString2(pszUrl + strlen(pszPrefix), "/", 0)); + if (aosTokens.size() == 2) + { + osUrl = "https://spatialreference.org/ref/"; + osUrl += aosTokens[0]; // authority + osUrl += '/'; + osUrl += aosTokens[1]; // code + osUrl += "/ogcwkt/"; + } + break; + } + } + const char *pszTimeout = "TIMEOUT=10"; - char *apszOptions[] = {const_cast(pszHeaders), - const_cast(pszTimeout), nullptr}; + char *apszOptions[] = {const_cast(pszTimeout), nullptr}; - CPLHTTPResult *psResult = CPLHTTPFetch(pszUrl, apszOptions); + CPLHTTPResult *psResult = CPLHTTPFetch(osUrl.c_str(), apszOptions); /* -------------------------------------------------------------------- */ /* Try to handle errors. */ diff --git a/deps/libgdal/gdal/port/cpl_aws.cpp b/deps/libgdal/gdal/port/cpl_aws.cpp index 9aaa8178e..73451232f 100644 --- a/deps/libgdal/gdal/port/cpl_aws.cpp +++ b/deps/libgdal/gdal/port/cpl_aws.cpp @@ -838,10 +838,21 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2( const CPLString osEC2RootURL(VSIGetPathSpecificOption( osPathForOption.c_str(), "CPL_AWS_EC2_API_ROOT_URL", osEC2DefaultURL)); // coverity[tainted_data] - const CPLString osECSRelativeURI(VSIGetPathSpecificOption( - osPathForOption.c_str(), "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "")); + const CPLString osECSFullURI(VSIGetPathSpecificOption( + osPathForOption.c_str(), "AWS_CONTAINER_CREDENTIALS_FULL_URI", "")); + // coverity[tainted_data] + const CPLString osECSRelativeURI( + osECSFullURI.empty() ? VSIGetPathSpecificOption( + osPathForOption.c_str(), + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "") + : std::string()); CPLString osToken; - if (osEC2RootURL == osEC2DefaultURL && !osECSRelativeURI.empty()) + if (!osECSFullURI.empty()) + { + // Cf https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html + osURLRefreshCredentials = osECSFullURI; + } + else if (osEC2RootURL == osEC2DefaultURL && !osECSRelativeURI.empty()) { // See // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html diff --git a/deps/libgdal/gdal/port/cpl_safemaths.hpp b/deps/libgdal/gdal/port/cpl_safemaths.hpp index d37b757c9..05e2da8d5 100644 --- a/deps/libgdal/gdal/port/cpl_safemaths.hpp +++ b/deps/libgdal/gdal/port/cpl_safemaths.hpp @@ -126,7 +126,7 @@ inline CPLSafeInt CPLSM(GUInt64 x) inline CPLSafeInt CPLSM_TO_UNSIGNED(GUInt64 x); #endif -#if defined(_MSC_VER) +#if !defined(BUILTIN_OVERFLOW_CHECK_AVAILABLE) && defined(_MSC_VER) class CPLMSVCSafeIntException : public msl::utilities::SafeIntException { public: diff --git a/deps/libgdal/gdal/port/cpl_vsisimple.cpp b/deps/libgdal/gdal/port/cpl_vsisimple.cpp index 61d807bfc..0b89674a3 100644 --- a/deps/libgdal/gdal/port/cpl_vsisimple.cpp +++ b/deps/libgdal/gdal/port/cpl_vsisimple.cpp @@ -1458,6 +1458,9 @@ GIntBig CPLGetPhysicalRAM(void) { // cgroup V1 // Read memory.limit_in_byte in the whole szGroupName hierarchy + // Make sure to end up by reading + // /sys/fs/cgroup/memory/memory.limit_in_bytes itself, for + // scenarios like https://github.com/OSGeo/gdal/issues/8968 while (true) { snprintf(szFilename, sizeof(szFilename), @@ -1477,7 +1480,7 @@ GIntBig CPLGetPhysicalRAM(void) std::min(static_cast(nVal), nLimit)); } char *pszLastSlash = strrchr(szGroupName, '/'); - if (!pszLastSlash || pszLastSlash == szGroupName) + if (!pszLastSlash) break; *pszLastSlash = '\0'; }