Skip to content

Commit

Permalink
PO-286: Add common services for sftp (#334)
Browse files Browse the repository at this point in the history
* PO-286: Initial code

* PO-286: update after local testing

* Supress CVE

* Supress CVE

* Configure secrets from key vault

* Bumping chart version/ fixing aliases

* Add job to test the sftp

* Add overloaded delte file methods

---------

Co-authored-by: hmcts-jenkins-cnp <60659747+hmcts-jenkins-cnp[bot]@users.noreply.github.com>
  • Loading branch information
sabahirfan and hmcts-jenkins-cnp[bot] authored Apr 30, 2024
1 parent add9b84 commit 650472c
Show file tree
Hide file tree
Showing 12 changed files with 248 additions and 17 deletions.
27 changes: 17 additions & 10 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ task functionalLegacy(type: Test) {
systemProperty "cucumber.filter.tags", "@Legacy and not @Smoke and not @Ignore"
systemProperty "test.mode", "legacy"
}
task copyFunctionalReport(type: Copy){
task copyFunctionalReport(type: Copy) {
from("${rootDir}/target/site/serenity")
into("${rootDir}/functional-test-report")
logger.quiet("Functional Test Report available at - file://${rootDir}/functional-test-report/index.html")
Expand All @@ -113,7 +113,7 @@ task functional() {
description = "Runs functional tests"
group = "Verification"
gradle.startParameter.continueOnFailure = true
dependsOn('clearReports','functionalOpal', 'functionalLegacy', 'aggregate', 'copyFunctionalReport')
dependsOn('clearReports', 'functionalOpal', 'functionalLegacy', 'aggregate', 'copyFunctionalReport')
tasks.functionalOpal.mustRunAfter clearReports
tasks.functionalLegacy.mustRunAfter functionalOpal
tasks.aggregate.mustRunAfter functionalLegacy
Expand All @@ -130,7 +130,7 @@ task smokeOpal(type: Test) {
gradle.startParameter.continueOnFailure = true
systemProperty "cucumber.filter.tags", "@Smoke and not @Ignore"
}
task copySmokeReport(type: Copy){
task copySmokeReport(type: Copy) {
from("${rootDir}/target/site/serenity")
into("${rootDir}/smoke-test-report")
logger.quiet("Smoke Test Report available at - file://${rootDir}/smoke-test-report/index.html")
Expand All @@ -140,7 +140,7 @@ task smoke() {
description = "Runs Smoke Tests"
group = "Verification"
gradle.startParameter.continueOnFailure = true
dependsOn('clearReports','smokeOpal', 'aggregate', 'copySmokeReport')
dependsOn('clearReports', 'smokeOpal', 'aggregate', 'copySmokeReport')
tasks.smokeOpal.mustRunAfter clearReports
tasks.aggregate.mustRunAfter smokeOpal
tasks.copySmokeReport.mustRunAfter aggregate
Expand All @@ -164,17 +164,16 @@ jacocoTestReport {
}
afterEvaluate {
classDirectories.setFrom(files(classDirectories.files.collect {
fileTree(dir: it, exclude: jacocoExclusionArray(coverageExclusions)
fileTree(dir: it, exclude: jacocoExclusionArray(coverageExclusions)
)
}))
}
}

static String[] jacocoExclusionArray(ArrayList<String> exclusions)
{
static String[] jacocoExclusionArray(ArrayList<String> exclusions) {
final def lst = new ArrayList<String>()

exclusions.stream().forEach {it.endsWith(".java") ? lst.add(it.replace(".java", ".class")) : lst.add(it)}
exclusions.stream().forEach { it.endsWith(".java") ? lst.add(it.replace(".java", ".class")) : lst.add(it) }

return lst.toArray()
}
Expand All @@ -191,7 +190,7 @@ sonarqube {
property "sonar.projectKey", "uk.gov.hmcts:opal-fines-service"
property "sonar.gradle.skipCompile", "true"
property "sonar.exclusions", coverageExclusions.join(', ')
property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/repository/jpa/*,**/opal/dto/*"
property 'sonar.coverage.exclusions', "**/entity/*,**/model/*,**/exception/*,**/sftp/*,**/repository/jpa/*,**/opal/dto/*"
}
}

Expand All @@ -212,6 +211,10 @@ dependencyManagement {
mavenBom 'org.springframework.cloud:spring-cloud-dependencies:2023.0.1'
}

imports {
mavenBom "org.springframework.integration:spring-integration-bom:6.2.1"
}

dependencies {
dependency group: 'com.google.guava', name: 'guava', version: '33.1.0-jre'
}
Expand Down Expand Up @@ -246,6 +249,11 @@ dependencies {
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-oauth2-client'
implementation group: 'org.springframework.boot', name: 'spring-boot-starter-quartz'

implementation group: 'org.springframework.boot', name: 'spring-boot-starter-integration'
implementation group: 'org.springframework.integration', name: 'spring-integration-sftp'
implementation group: 'org.springframework.integration', name: 'spring-integration-file'


implementation group: 'org.springframework.security', name: 'spring-security-oauth2-authorization-server', version: '1.2.4'

implementation group: 'org.springframework', name: 'spring-aspects'
Expand Down Expand Up @@ -274,7 +282,6 @@ dependencies {
testCompileOnly 'org.projectlombok:lombok:1.18.32'
testAnnotationProcessor 'org.projectlombok:lombok:1.18.32'


testImplementation 'com.github.fge:json-schema-validator:2.2.14'
testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'

Expand Down
2 changes: 1 addition & 1 deletion charts/opal-fines-service/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ appVersion: "1.0"
description: A Helm chart for opal-fines-service app
name: opal-fines-service
home: https://github.com/hmcts/opal-fines-service
version: 0.0.31
version: 0.0.32
maintainers:
- name: HMCTS Opal Team
dependencies:
Expand Down
8 changes: 8 additions & 0 deletions charts/opal-fines-service/values.dev.template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,14 @@ java:
alias: OPAL_LEGACY_GATEWAY_USERNAME
- name: OpalLegacyGatewayPassword
alias: OPAL_LEGACY_GATEWAY_PASSWORD
- name: inbound-user
alias: OPAL_SFTP_INBOUND_USER
- name: inbound-password
alias: OPAL_SFTP_INBOUND_PASSWORD
- name: outbound-user
alias: OPAL_SFTP_OUTBOUND_USER
- name: outbound-password
alias: OPAL_SFTP_OUTBOUND_PASSWORD
environment:
OPAL_FINES_DB_HOST: "{{ .Release.Name }}-postgresql"
OPAL_FINES_DB_NAME: "{{ .Values.postgresql.auth.database}}"
Expand Down
8 changes: 8 additions & 0 deletions charts/opal-fines-service/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ java:
alias: OPAL_LEGACY_GATEWAY_USERNAME
- name: OpalLegacyGatewayPassword
alias: OPAL_LEGACY_GATEWAY_PASSWORD
- name: inbound-user
alias: OPAL_SFTP_INBOUND_USER
- name: inbound-password
alias: OPAL_SFTP_INBOUND_PASSWORD
- name: outbound-user
alias: OPAL_SFTP_OUTBOUND_USER
- name: outbound-password
alias: OPAL_SFTP_OUTBOUND_PASSWORD
environment:
RUN_DB_MIGRATION_ON_STARTUP: true
OPAL_FRONTEND_URL: https://opal-frontend.{{ .Values.global.environment }}.platform.hmcts.net
Expand Down
1 change: 1 addition & 0 deletions config/owasp/suppressions.xml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
<suppress>
<notes>Disputed</notes>
<cve>CVE-2023-39017</cve>
<cve>CVE-2023-48795</cve>
</suppress>

<!--End of Temporary suppression section -->
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
package uk.gov.hmcts.opal.scheduler.job;

import lombok.Getter;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import uk.gov.hmcts.opal.scheduler.model.CronJob;
import uk.gov.hmcts.opal.sftp.SftpService;

import java.io.IOException;
import java.io.InputStream;

import static java.lang.String.format;
import static java.time.LocalTime.now;

@Component
@Getter
Expand All @@ -18,18 +27,46 @@ public class FileHandlerJob implements CronJob {
@Value("${opal.schedule.file-handler-job.cron}")
private String cronExpression;

@Autowired
private SftpService sftpService;


@SneakyThrows
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {

log.info("Job ** {} ** starting @ {}", context.getJobDetail().getKey().getName(), context.getFireTime());
//TODO: Some logic here to perform the actual task

String fileName = format("test-file-%s.txt", now());
this.uploadFile("My file contents here...", fileName);

sftpService.downloadOutboundFile("", fileName, this::logInputStream);

log.info(
"Job ** {} ** completed. Next job scheduled @ {}",
context.getJobDetail().getKey().getName(),

context.getNextFireTime()
);
}

public void uploadFile(String contents, String fileName) {
sftpService.uploadOutboundFile(format("%s %s", contents, now()).getBytes(), "", fileName);
}


public void logInputStream(InputStream inputStream) {
try {
byte[] buffer = new byte[1024];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
// Do something with the read bytes
String contents = new String(buffer, 0, bytesRead);
log.info(contents);
}
} catch (IOException exception) {
log.error(exception.getMessage(), exception);
}
}

}
65 changes: 65 additions & 0 deletions src/main/java/uk/gov/hmcts/opal/sftp/SftpService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package uk.gov.hmcts.opal.sftp;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.stereotype.Service;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.function.Consumer;

import static java.lang.String.format;

@Slf4j
@Service
@RequiredArgsConstructor
public class SftpService {

private final DefaultSftpSessionFactory inboundSessionFactory;
private final DefaultSftpSessionFactory outboundSessionFactory;

public void uploadOutboundFile(byte[] fileBytes, String path, String fileName) {
uploadFile(outboundSessionFactory, fileBytes, path, fileName);
}

public void uploadFile(DefaultSftpSessionFactory sessionFactory, byte[] fileBytes, String path, String fileName) {
var template = new RemoteFileTemplate<>(sessionFactory);
template.execute(session -> {
session.write(new ByteArrayInputStream(fileBytes), path + "/" + fileName);
log.info(format("File %s uploaded successfully.", fileName));
return true;
});
}

public boolean downloadInboundFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
return downloadFile(inboundSessionFactory, path, fileName, fileProcessor);
}

public boolean downloadOutboundFile(String path, String fileName, Consumer<InputStream> fileProcessor) {
return downloadFile(outboundSessionFactory, path, fileName, fileProcessor);
}

public boolean downloadFile(DefaultSftpSessionFactory sessionFactory,
String path,
String fileName,
Consumer<InputStream> fileProcessor) {
var template = new RemoteFileTemplate<>(sessionFactory);
return template.get(path + "/" + fileName, fileProcessor::accept);
}

public boolean deleteOutboundFile(String path, String fileName) {
return deleteFile(outboundSessionFactory, path, fileName);
}

public boolean deleteInboundFile(String path, String fileName) {
return deleteFile(inboundSessionFactory, path, fileName);
}

public boolean deleteFile(DefaultSftpSessionFactory sessionFactory, String path, String fileName) {
var template = new RemoteFileTemplate<>(sessionFactory);
return template.execute(session -> session.remove(path + "/" + fileName));
}

}
38 changes: 38 additions & 0 deletions src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConfiguration.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package uk.gov.hmcts.opal.sftp.config;

import lombok.RequiredArgsConstructor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;

@Configuration
@RequiredArgsConstructor
public class SftpConfiguration {

private final SftpProperties sftpProperties;

@Bean
public DefaultSftpSessionFactory inboundSessionFactory() {
DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory();
sessionFactory.setHost(sftpProperties.getInbound().getHost());
sessionFactory.setPort(sftpProperties.getInbound().getPort());
sessionFactory.setUser(sftpProperties.getInbound().getUser());
sessionFactory.setPassword(sftpProperties.getInbound().getPassword());
sessionFactory.setAllowUnknownKeys(true);

return sessionFactory;
}

@Bean
public DefaultSftpSessionFactory outboundSessionFactory() {
DefaultSftpSessionFactory sessionFactory = new DefaultSftpSessionFactory();
sessionFactory.setHost(sftpProperties.getOutbound().getHost());
sessionFactory.setPort(sftpProperties.getOutbound().getPort());
sessionFactory.setUser(sftpProperties.getOutbound().getUser());
sessionFactory.setPassword(sftpProperties.getOutbound().getPassword());
sessionFactory.setAllowUnknownKeys(true);

return sessionFactory;
}

}
15 changes: 15 additions & 0 deletions src/main/java/uk/gov/hmcts/opal/sftp/config/SftpConnection.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package uk.gov.hmcts.opal.sftp.config;

import lombok.Builder;
import lombok.Data;

@Data
@Builder
public class SftpConnection {

private String host;
private int port;
private String user;
private String password;
private String location;
}
15 changes: 15 additions & 0 deletions src/main/java/uk/gov/hmcts/opal/sftp/config/SftpProperties.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package uk.gov.hmcts.opal.sftp.config;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

@Component
@Data
@ConfigurationProperties(prefix = "opal.sftp")
public class SftpProperties {

private SftpConnection inbound;
private SftpConnection outbound;

}
13 changes: 13 additions & 0 deletions src/main/resources/application.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,19 @@ quartzProperties:
threadCount: 8

opal:
sftp:
inbound:
host: ${OPAL_SFTP_INBOUND_HOST:opalsftpstg.blob.core.windows.net}
port: ${OPAL_SFTP_INBOUND_PORT:22}
user: ${OPAL_SFTP_INBOUND_USER:-}
password: ${OPAL_SFTP_INBOUND_PASSWORD:-}
location: ${OPAL_SFTP_INBOUND_LOCATION:inbound}
outbound:
host: ${OPAL_SFTP_OUTBOUND_HOST:opalsftpstg.blob.core.windows.net}
port: ${OPAL_SFTP_OUTBOUND_PORT:22}
user: ${OPAL_SFTP_OUTBOUND_USER:-}
password: ${OPAL_SFTP_OUTBOUND_PASSWORD:-}
location: ${OPAL_SFTP_OUTBOUND_LOCATION:outbound}
schedule:
log-retention-job:
cron: ${OPAL_LOG_RETENTION_JOB_CRON:0 0 * * * ?}
Expand Down
Loading

0 comments on commit 650472c

Please sign in to comment.