From d047feca9b258ef0b378748962041ccff0755c84 Mon Sep 17 00:00:00 2001 From: Quyen Ly Date: Thu, 12 Sep 2024 19:57:13 +0700 Subject: [PATCH] VAL-403 Re-use the previous version and dependency databases if no changes found --- .../core/service/MysqlValidationService.java | 6 ++- .../rvf/core/service/ReleaseDataManager.java | 25 +++++++++- .../core/service/ValidationVersionLoader.java | 49 ++++++++++--------- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/ihtsdo/rvf/core/service/MysqlValidationService.java b/src/main/java/org/ihtsdo/rvf/core/service/MysqlValidationService.java index 586d8cd1..b33d24eb 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/MysqlValidationService.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/MysqlValidationService.java @@ -64,9 +64,11 @@ public ValidationStatusReport runRF2MysqlValidations(ValidationRunConfig validat String reportStorage = validationConfig.getStorageLocation(); try { // prepare release data for testing - if (StringUtils.hasLength(executionConfig.getPreviousVersion())) { - releaseVersionLoader.loadPreviousVersion(executionConfig); + releaseVersionLoader.loadPreviousVersion(executionConfig); + if (releaseVersionLoader.isUnknownVersion(executionConfig.getPreviousVersion())) { + statusReport.addFailureMessage("Failed to load previous release " + executionConfig.getPreviousVersion()); } + // load dependency release releaseVersionLoader.loadDependencyVersion(executionConfig); if (releaseVersionLoader.isUnknownVersion(executionConfig.getExtensionDependencyVersion())) { diff --git a/src/main/java/org/ihtsdo/rvf/core/service/ReleaseDataManager.java b/src/main/java/org/ihtsdo/rvf/core/service/ReleaseDataManager.java index 46d04b8d..59ca77dc 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/ReleaseDataManager.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/ReleaseDataManager.java @@ -128,8 +128,8 @@ private void fetchRvfSchemasFromDb() { } } - public void dropDatabaseIfExist(String schemaName) throws BusinessServiceException{ - logger.info("Dropping schema: {}", schemaName); + public void dropSchema(String schemaName) throws BusinessServiceException{ + logger.info("Dropping schema {}", schemaName); //clean database try (Statement statement = dataSource.getConnection().createStatement()) { String dropStr = "drop database if exists " + schemaName + ";"; @@ -722,4 +722,25 @@ private void insertIntoProspectiveDeltaTablesFromDependency(MysqlExecutionConfig } } } + + public long getPublishedReleaseLastModifiedDate(String publishedRelease) { + ResourceManager resourceManager = new ResourceManager(releaseStorageConfig, cloudResourceLoader); + try { + return resourceManager.getResourceLastModifiedDate(publishedRelease); + } catch (Exception e) { + logger.warn("Failed to find the last modified for resource {}", publishedRelease, e); + return 0L; + } + } + + public long getBinaryArchiveSchemaLastModifiedDate(String schemaName) { + ResourceManager resourceManager = new ResourceManager(mysqlBinaryStorageConfig, cloudResourceLoader); + String archiveFileName = schemaName + ZIP_FILE_EXTENSION; + try { + return resourceManager.getResourceLastModifiedDate(archiveFileName); + } catch (Exception e) { + logger.warn("Failed to find the last modified for resource {}", schemaName, e); + return 0L; + } + } } diff --git a/src/main/java/org/ihtsdo/rvf/core/service/ValidationVersionLoader.java b/src/main/java/org/ihtsdo/rvf/core/service/ValidationVersionLoader.java index 2e54869a..a0457fb8 100644 --- a/src/main/java/org/ihtsdo/rvf/core/service/ValidationVersionLoader.java +++ b/src/main/java/org/ihtsdo/rvf/core/service/ValidationVersionLoader.java @@ -80,8 +80,6 @@ public void init() { } public void loadPreviousVersion(MysqlExecutionConfig executionConfig) throws BusinessServiceException, IOException { - String schemaName = constructRVFSchema(executionConfig.getPreviousVersion()); - releaseDataManager.dropDatabaseIfExist(schemaName); if (executionConfig.getPreviousVersion().endsWith(ZIP_FILE_EXTENSION)) { String rvfDbSchema = loadRelease(executionConfig.getPreviousVersion()); executionConfig.setPreviousVersion(rvfDbSchema); @@ -92,8 +90,6 @@ public void loadPreviousVersion(MysqlExecutionConfig executionConfig) throws Bus } public void loadDependencyVersion(MysqlExecutionConfig executionConfig) throws IOException, BusinessServiceException { - String schemaName = constructRVFSchema(executionConfig.getExtensionDependencyVersion()); - releaseDataManager.dropDatabaseIfExist(schemaName); if (executionConfig.getExtensionDependencyVersion().endsWith(ZIP_FILE_EXTENSION)) { String dependencyVersion = loadRelease(executionConfig.getExtensionDependencyVersion()); executionConfig.setExtensionDependencyVersion(dependencyVersion); @@ -145,34 +141,41 @@ private boolean checkDeltaFilesExist(File localProspectiveFile) throws ReleaseIm return false; } - private String constructRVFSchema(String releaseVersion) { - if (releaseVersion != null) { - if (releaseVersion.endsWith(ZIP_FILE_EXTENSION)) { - return RvfReleaseDbSchemaNameGenerator.generate(releaseVersion); - } - return releaseVersion.startsWith(RVF_DB_PREFIX) ? releaseVersion : RVF_DB_PREFIX + releaseVersion; - } else { - return null; - } - } - private String loadRelease(String releaseVersion) throws IOException, BusinessServiceException { if (releaseVersion != null && releaseVersion.endsWith(ZIP_FILE_EXTENSION)) { String schemaName = RvfReleaseDbSchemaNameGenerator.generate(releaseVersion); - if (!releaseDataManager.isKnownRelease(schemaName) && (!releaseDataManager.restoreReleaseFromBinaryArchive(schemaName))) { + long publishedReleaseLastModifiedDate = releaseDataManager.getPublishedReleaseLastModifiedDate(releaseVersion); + long binaryArchiveSchemaLastModifiedDate = releaseDataManager.getBinaryArchiveSchemaLastModifiedDate(schemaName); + + // If the binary archive has been deleted (- or it has not been generated yet), OR the release file has been changed, + // then the schema and the binary archive schema need to be re-generated + if (binaryArchiveSchemaLastModifiedDate == 0 || publishedReleaseLastModifiedDate > binaryArchiveSchemaLastModifiedDate) { + logger.info("The Binary Archive file was deleted (- or it has not been generated yet), OR a new version of published release has been detected."); + if (releaseDataManager.isKnownRelease(schemaName)) { + releaseDataManager.dropSchema(schemaName); + } + uploadPublishedReleaseThenGenerateBinaryArchive(releaseVersion, schemaName); + } else { + // Restore schema from binary archive file + if (!releaseDataManager.isKnownRelease(schemaName) && !releaseDataManager.restoreReleaseFromBinaryArchive(schemaName)) { logger.info("No existing mysql binary release available."); - releaseDataManager.uploadPublishedReleaseFromStore(releaseVersion, schemaName); - if (generateBinaryArchive) { - String archiveFilename = releaseDataManager.generateBinaryArchive(schemaName); - logger.info("Release mysql binary archive is generated: {}", archiveFilename); - } + uploadPublishedReleaseThenGenerateBinaryArchive(releaseVersion, schemaName); + } + } - } - return schemaName; + return schemaName; } return releaseVersion; } + private void uploadPublishedReleaseThenGenerateBinaryArchive(String releaseVersion, String schemaName) throws BusinessServiceException { + releaseDataManager.uploadPublishedReleaseFromStore(releaseVersion, schemaName); + if (generateBinaryArchive) { + String archiveFilename = releaseDataManager.generateBinaryArchive(schemaName); + logger.info("Release mysql binary archive is generated: {}", archiveFilename); + } + } + public MysqlExecutionConfig createExecutionConfig(ValidationRunConfig validationConfig) throws BusinessServiceException { MysqlExecutionConfig executionConfig = new MysqlExecutionConfig(validationConfig.getRunId(), validationConfig.isFirstTimeRelease()); executionConfig.setProspectiveVersion(RVF_DB_PREFIX + getProspectiveVersionFromFileNames(validationConfig)