diff --git a/build.gradle b/build.gradle index 36395c76e..724c551dd 100644 --- a/build.gradle +++ b/build.gradle @@ -103,7 +103,7 @@ task functionalLegacy(type: Test) { systemProperty "cucumber.filter.tags", "@Legacy and not @Smoke and not @Ignore" systemProperty "test.mode", "legacy" } -task copyFunctionalReport(type: Copy){ +task copyFunctionalReport(type: Copy) { from("${rootDir}/target/site/serenity") into("${rootDir}/functional-test-report") logger.quiet("Functional Test Report available at - file://${rootDir}/functional-test-report/index.html") @@ -113,7 +113,7 @@ task functional() { description = "Runs functional tests" group = "Verification" gradle.startParameter.continueOnFailure = true - dependsOn('clearReports','functionalOpal', 'functionalLegacy', 'aggregate', 'copyFunctionalReport') + dependsOn('clearReports', 'functionalOpal', 'functionalLegacy', 'aggregate', 'copyFunctionalReport') tasks.functionalOpal.mustRunAfter clearReports tasks.functionalLegacy.mustRunAfter functionalOpal tasks.aggregate.mustRunAfter functionalLegacy @@ -130,7 +130,7 @@ task smokeOpal(type: Test) { gradle.startParameter.continueOnFailure = true systemProperty "cucumber.filter.tags", "@Smoke and not @Ignore" } -task copySmokeReport(type: Copy){ +task copySmokeReport(type: Copy) { from("${rootDir}/target/site/serenity") into("${rootDir}/smoke-test-report") logger.quiet("Smoke Test Report available at - file://${rootDir}/smoke-test-report/index.html") @@ -140,7 +140,7 @@ task smoke() { description = "Runs Smoke Tests" group = "Verification" gradle.startParameter.continueOnFailure = true - dependsOn('clearReports','smokeOpal', 'aggregate', 'copySmokeReport') + dependsOn('clearReports', 'smokeOpal', 'aggregate', 'copySmokeReport') tasks.smokeOpal.mustRunAfter clearReports tasks.aggregate.mustRunAfter smokeOpal tasks.copySmokeReport.mustRunAfter aggregate @@ -164,17 +164,16 @@ jacocoTestReport { } afterEvaluate { classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, exclude: jacocoExclusionArray(coverageExclusions) + fileTree(dir: it, exclude: jacocoExclusionArray(coverageExclusions) ) })) } } -static String[] jacocoExclusionArray(ArrayList exclusions) -{ +static String[] jacocoExclusionArray(ArrayList exclusions) { final def lst = new ArrayList() - exclusions.stream().forEach {it.endsWith(".java") ? lst.add(it.replace(".java", ".class")) : lst.add(it)} + exclusions.stream().forEach { it.endsWith(".java") ? lst.add(it.replace(".java", ".class")) : lst.add(it) } return lst.toArray() } @@ -191,7 +190,7 @@ sonarqube { property "sonar.projectKey", "uk.gov.hmcts:opal-fines-service" property "sonar.gradle.skipCompile", "true" property "sonar.exclusions", coverageExclusions.join(', ') - property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/repository/jpa/*,**/opal/dto/*" + property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/sftp/*,**/repository/jpa/*,**/opal/dto/*" } } @@ -212,6 +211,10 @@ dependencyManagement { mavenBom 'org.springframework.cloud:spring-cloud-dependencies:2023.0.1' } + imports { + mavenBom "org.springframework.integration:spring-integration-bom:6.2.1" + } + dependencies { dependency group: 'com.google.guava', name: 'guava', version: '33.1.0-jre' } @@ -246,6 +249,11 @@ dependencies { implementation group: 'org.springframework.boot', name: 'spring-boot-starter-oauth2-client' implementation group: 'org.springframework.boot', name: 'spring-boot-starter-quartz' + implementation group: 'org.springframework.boot', name: 'spring-boot-starter-integration' + implementation group: 'org.springframework.integration', name: 'spring-integration-sftp' + implementation group: 'org.springframework.integration', name: 'spring-integration-file' + + implementation group: 'org.springframework.security', name: 'spring-security-oauth2-authorization-server', version: '1.2.4' implementation group: 'org.springframework', name: 'spring-aspects' @@ -274,7 +282,6 @@ dependencies { testCompileOnly 'org.projectlombok:lombok:1.18.32' testAnnotationProcessor 'org.projectlombok:lombok:1.18.32' - testImplementation 'com.github.fge:json-schema-validator:2.2.14' testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' diff --git a/charts/opal-fines-service/Chart.yaml b/charts/opal-fines-service/Chart.yaml index 19c0158ac..abe52ee1c 100644 --- a/charts/opal-fines-service/Chart.yaml +++ b/charts/opal-fines-service/Chart.yaml @@ -3,7 +3,7 @@ appVersion: "1.0" description: A Helm chart for opal-fines-service app name: opal-fines-service home: https://github.com/hmcts/opal-fines-service -version: 0.0.31 +version: 0.0.32 maintainers: - name: HMCTS Opal Team dependencies: diff --git a/charts/opal-fines-service/values.dev.template.yaml b/charts/opal-fines-service/values.dev.template.yaml index 9ac712e9e..4a9a2a556 100644 --- a/charts/opal-fines-service/values.dev.template.yaml +++ b/charts/opal-fines-service/values.dev.template.yaml @@ -21,6 +21,14 @@ java: alias: OPAL_LEGACY_GATEWAY_USERNAME - name: OpalLegacyGatewayPassword alias: OPAL_LEGACY_GATEWAY_PASSWORD + - name: inbound-user + alias: OPAL_SFTP_INBOUND_USER + - name: inbound-password + alias: OPAL_SFTP_INBOUND_PASSWORD + - name: outbound-user + alias: OPAL_SFTP_OUTBOUND_USER + - name: outbound-password + alias: OPAL_SFTP_OUTBOUND_PASSWORD environment: OPAL_FINES_DB_HOST: "{{ .Release.Name }}-postgresql" OPAL_FINES_DB_NAME: "{{ .Values.postgresql.auth.database}}" diff --git a/charts/opal-fines-service/values.yaml b/charts/opal-fines-service/values.yaml index 84276a2a1..eebe8d2b9 100644 --- a/charts/opal-fines-service/values.yaml +++ b/charts/opal-fines-service/values.yaml @@ -31,6 +31,14 @@ java: alias: OPAL_LEGACY_GATEWAY_USERNAME - name: OpalLegacyGatewayPassword alias: OPAL_LEGACY_GATEWAY_PASSWORD + - name: inbound-user + alias: OPAL_SFTP_INBOUND_USER + - name: inbound-password + alias: OPAL_SFTP_INBOUND_PASSWORD + - name: outbound-user + alias: OPAL_SFTP_OUTBOUND_USER + - name: outbound-password + alias: OPAL_SFTP_OUTBOUND_PASSWORD environment: RUN_DB_MIGRATION_ON_STARTUP: true OPAL_FRONTEND_URL: https://opal-frontend.{{ .Values.global.environment }}.platform.hmcts.net diff --git a/config/owasp/suppressions.xml b/config/owasp/suppressions.xml index 5fd8f1dda..2a80506e5 100644 --- a/config/owasp/suppressions.xml +++ b/config/owasp/suppressions.xml @@ -9,6 +9,7 @@ Disputed CVE-2023-39017 + CVE-2023-48795 diff --git a/src/main/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJob.java b/src/main/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJob.java index a9e90a2df..d7e67d547 100644 --- a/src/main/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJob.java +++ b/src/main/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJob.java @@ -1,13 +1,22 @@ package uk.gov.hmcts.opal.scheduler.job; import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.quartz.DisallowConcurrentExecution; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import uk.gov.hmcts.opal.scheduler.model.CronJob; +import uk.gov.hmcts.opal.sftp.SftpService; + +import java.io.IOException; +import java.io.InputStream; + +import static java.lang.String.format; +import static java.time.LocalTime.now; @Component @Getter @@ -18,18 +27,46 @@ public class FileHandlerJob implements CronJob { @Value("${opal.schedule.file-handler-job.cron}") private String cronExpression; + @Autowired + private SftpService sftpService; + + @SneakyThrows @Override public void execute(JobExecutionContext context) throws JobExecutionException { log.info("Job ** {} ** starting @ {}", context.getJobDetail().getKey().getName(), context.getFireTime()); - //TODO: Some logic here to perform the actual task + + String fileName = format("test-file-%s.txt", now()); + this.uploadFile("My file contents here...", fileName); + + sftpService.downloadOutboundFile("", fileName, this::logInputStream); + log.info( "Job ** {} ** completed. Next job scheduled @ {}", context.getJobDetail().getKey().getName(), + context.getNextFireTime() ); + } + + public void uploadFile(String contents, String fileName) { + sftpService.uploadOutboundFile(format("%s %s", contents, now()).getBytes(), "", fileName); + } + + public void logInputStream(InputStream inputStream) { + try { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + // Do something with the read bytes + String contents = new String(buffer, 0, bytesRead); + log.info(contents); + } + } catch (IOException exception) { + log.error(exception.getMessage(), exception); + } } } diff --git a/src/main/java/uk/gov/hmcts/opal/sftp/SftpService.java b/src/main/java/uk/gov/hmcts/opal/sftp/SftpService.java new file mode 100644 index 000000000..0103beb67 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/opal/sftp/SftpService.java @@ -0,0 +1,65 @@ +package uk.gov.hmcts.opal.sftp; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.function.Consumer; + +import static java.lang.String.format; + +@Slf4j +@Service +@RequiredArgsConstructor +public class SftpService { + + private final DefaultSftpSessionFactory inboundSessionFactory; + private final DefaultSftpSessionFactory outboundSessionFactory; + + public void uploadOutboundFile(byte[] fileBytes, String path, String fileName) { + uploadFile(outboundSessionFactory, fileBytes, path, fileName); + } + + public void uploadFile(DefaultSftpSessionFactory sessionFactory, byte[] fileBytes, String path, String fileName) { + var template = new RemoteFileTemplate<>(sessionFactory); + template.execute(session -> { + session.write(new ByteArrayInputStream(fileBytes), path + "/" + fileName); + log.info(format("File %s uploaded successfully.", fileName)); + return true; + }); + } + + public boolean downloadInboundFile(String path, String fileName, Consumer fileProcessor) { + return downloadFile(inboundSessionFactory, path, fileName, fileProcessor); + } + + public boolean downloadOutboundFile(String path, String fileName, Consumer fileProcessor) { + return downloadFile(outboundSessionFactory, path, fileName, fileProcessor); + } + + public boolean downloadFile(DefaultSftpSessionFactory sessionFactory, + String path, + String fileName, + Consumer fileProcessor) { + var template = new RemoteFileTemplate<>(sessionFactory); + return template.get(path + "/" + fileName, fileProcessor::accept); + } + + public boolean deleteOutboundFile(String path, String fileName) { + return deleteFile(outboundSessionFactory, path, fileName); + } + + public boolean deleteInboundFile(String path, String fileName) { + return deleteFile(inboundSessionFactory, path, fileName); + } + + public boolean deleteFile(DefaultSftpSessionFactory sessionFactory, String path, String fileName) { + var template = new RemoteFileTemplate<>(sessionFactory); + return template.execute(session -> session.remove(path + "/" + fileName)); + } + +} diff --git a/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConfiguration.java b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConfiguration.java new file mode 100644 index 000000000..b16a90cc5 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConfiguration.java @@ -0,0 +1,38 @@ +package uk.gov.hmcts.opal.sftp.config; + +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; + +@Configuration +@RequiredArgsConstructor +public class SftpConfiguration { + + private final SftpProperties sftpProperties; + + @Bean + public DefaultSftpSessionFactory inboundSessionFactory() { + DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory(); + sessionFactory.setHost(sftpProperties.getInbound().getHost()); + sessionFactory.setPort(sftpProperties.getInbound().getPort()); + sessionFactory.setUser(sftpProperties.getInbound().getUser()); + sessionFactory.setPassword(sftpProperties.getInbound().getPassword()); + sessionFactory.setAllowUnknownKeys(true); + + return sessionFactory; + } + + @Bean + public DefaultSftpSessionFactory outboundSessionFactory() { + DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory(); + sessionFactory.setHost(sftpProperties.getOutbound().getHost()); + sessionFactory.setPort(sftpProperties.getOutbound().getPort()); + sessionFactory.setUser(sftpProperties.getOutbound().getUser()); + sessionFactory.setPassword(sftpProperties.getOutbound().getPassword()); + sessionFactory.setAllowUnknownKeys(true); + + return sessionFactory; + } + +} diff --git a/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConnection.java b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConnection.java new file mode 100644 index 000000000..49ff97d67 --- /dev/null +++ b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConnection.java @@ -0,0 +1,15 @@ +package uk.gov.hmcts.opal.sftp.config; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class SftpConnection { + + private String host; + private int port; + private String user; + private String password; + private String location; +} diff --git a/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpProperties.java b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpProperties.java new file mode 100644 index 000000000..9a32768eb --- /dev/null +++ b/src/main/java/uk/gov/hmcts/opal/sftp/config/SftpProperties.java @@ -0,0 +1,15 @@ +package uk.gov.hmcts.opal.sftp.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Data +@ConfigurationProperties(prefix = "opal.sftp") +public class SftpProperties { + + private SftpConnection inbound; + private SftpConnection outbound; + +} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index b9d888056..3b5666487 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -110,6 +110,19 @@ quartzProperties: threadCount: 8 opal: + sftp: + inbound: + host: ${OPAL_SFTP_INBOUND_HOST:opalsftpstg.blob.core.windows.net} + port: ${OPAL_SFTP_INBOUND_PORT:22} + user: ${OPAL_SFTP_INBOUND_USER:-} + password: ${OPAL_SFTP_INBOUND_PASSWORD:-} + location: ${OPAL_SFTP_INBOUND_LOCATION:inbound} + outbound: + host: ${OPAL_SFTP_OUTBOUND_HOST:opalsftpstg.blob.core.windows.net} + port: ${OPAL_SFTP_OUTBOUND_PORT:22} + user: ${OPAL_SFTP_OUTBOUND_USER:-} + password: ${OPAL_SFTP_OUTBOUND_PASSWORD:-} + location: ${OPAL_SFTP_OUTBOUND_LOCATION:outbound} schedule: log-retention-job: cron: ${OPAL_LOG_RETENTION_JOB_CRON:0 0 * * * ?} diff --git a/src/test/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJobTest.java b/src/test/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJobTest.java index 9e3d2a7df..608138d0e 100644 --- a/src/test/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJobTest.java +++ b/src/test/java/uk/gov/hmcts/opal/scheduler/job/FileHandlerJobTest.java @@ -1,5 +1,6 @@ package uk.gov.hmcts.opal.scheduler.job; +import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -10,10 +11,17 @@ import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.slf4j.Logger; +import uk.gov.hmcts.opal.sftp.SftpService; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.util.Date; import static org.assertj.core.util.DateUtil.now; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class FileHandlerJobTest { @@ -22,10 +30,13 @@ class FileHandlerJobTest { private FileHandlerJob fileHandlerJob; @Mock - private JobExecutionContext jobExecutionContext; + JobExecutionContext jobExecutionContext; @Mock - private Logger logger; + SftpService sftpService; + + @Mock + Logger logger; @BeforeEach void setUp() { @@ -34,9 +45,10 @@ void setUp() { @Test void testExecute() throws JobExecutionException { - JobDetail jobDetail = JobBuilder.newJob(FileHandlerJob.class) - .withIdentity("FileHandlerJob", "FileHandlerJob") - .build(); + JobDetail jobDetail = JobBuilder.newJob(FileHandlerJob.class).withIdentity( + "FileHandlerJob", + "FileHandlerJob" + ).build(); when(jobExecutionContext.getJobDetail()).thenReturn(jobDetail); Date now = now(); @@ -44,5 +56,17 @@ void testExecute() throws JobExecutionException { when(jobExecutionContext.getNextFireTime()).thenReturn(now); fileHandlerJob.execute(jobExecutionContext); + + verify(sftpService, times(1)).uploadOutboundFile(any(), anyString(), anyString()); + verify(sftpService, times(1)).downloadOutboundFile(anyString(), anyString(), any()); + } + + @Test + @SneakyThrows + void testProcessInputStream() { + String inputString = "Test input string"; + InputStream inputStream = new ByteArrayInputStream(inputString.getBytes()); + + fileHandlerJob.logInputStream(inputStream); } }