From 198360e29cd027a45cbeb468ce96c1d643f720bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Reitmann?= Date: Tue, 19 Oct 2021 11:22:18 +0200 Subject: [PATCH 01/18] bump version (#3011) --- package-lock.json | 2 +- package.json | 2 +- pom.xml | 2 +- src/main/webapp/scripts/app.constants.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index edcedbf2a8..d8e32c7d02 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Metadatamanagement", - "version": "1.0.115", + "version": "1.0.116-SNAPSHOT", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7ad2aaeb0a..35aaa41948 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Metadatamanagement", - "version": "1.0.115", + "version": "1.0.116-SNAPSHOT", "description": "Data Search for Higher Education Research and Science Studies", "private": true, "dependencies": { diff --git a/pom.xml b/pom.xml index 529f250939..675bc758f2 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ eu.dzhw.fdz metadatamanagement - 1.0.115 + 1.0.116-SNAPSHOT war metadatamanagement diff --git a/src/main/webapp/scripts/app.constants.js b/src/main/webapp/scripts/app.constants.js index f4b6674f6d..bc8600e55b 100644 --- a/src/main/webapp/scripts/app.constants.js +++ b/src/main/webapp/scripts/app.constants.js @@ -6,7 +6,7 @@ angular.module('metadatamanagementApp') .constant('ENV', 'local') -.constant('VERSION', '1.0.115') +.constant('VERSION', '1.0.116-SNAPSHOT') .constant('ElasticSearchProperties', {apiVersion:'7.x',logLevel:'error',pageSize:10}) From 8c8f464d5ce3fc25abb8b77ad61c0d5d3f39fde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Reitmann?= Date: Tue, 19 Oct 2021 14:14:00 +0200 Subject: [PATCH 02/18] refine readme after freelancer setup --- README.md | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8721f8c743..9d672234b0 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,8 @@ On Windows, `patch.exe` has to exist in the PATH. It is distributed as part of g ## Running on your local machine Before starting the app on your local machine you need to start the following Document Stores: -1. Mongodb: Mongodb must be running on the default port, on ubuntu you should install it from [here](https://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/) -2. Elasticsearch (7.12.1): Elasticsearch must be running on its default port. You can download it from [here](https://www.elastic.co/downloads/elasticsearch) - -Make sure that you have read-write-access on the ***data*** directory (in your project directory) for Elasticsearch. +1. Mongodb: Mongodb must be running on the default port, on ubuntu you should install it from [here](https://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/). ***However, running it with docker is preferred.*** +2. Elasticsearch (7.12.1): Elasticsearch must be running on its default port. You can download it from [here](https://www.elastic.co/downloads/elasticsearch). ***However running it with docker is preferred.*** ***Alternatively*** you can run @@ -42,7 +40,16 @@ Make sure that you have read-write-access on the ***data*** directory (in your p # for later use once the containers are created docker-compose start -to start all services the metadatamanagement depends on. Mongodb and Elasticsearch will be listening on its default ports. +to start all services the metadatamanagement depends on. MongoDB and Elasticsearch will be listening on its default ports. + +Make sure that you have read-write-access on the ***data*** directory (in your project directory) for Elasticsearch. + +You can get a MongoDB dump and restore it locally: +``` +wget https://metadatamanagement-public.s3.eu-central-1.amazonaws.com/20211019_metadatamanagement.zip +unzip 20211019_metadatamanagement.zip +mongorestore ./metadatamanagement --db=metadatamanagement +``` You will need to setup your `~/.m2/settings.xml` so that maven can download a dependency from Github: @@ -71,6 +78,15 @@ auto-refreshes when files change on your hard drive. mvn grunt +If you run the backend on your machine for the first time or you have restored a mongodb dump then you need to setup the elasticsearch indices. Therefore go to http://localhost:8080/de/health and sign in with `localuser` and password `herlichwillkommen`. Then click the red button 'reindex'. Reindexing can take up to 1 hour. + +In order for all external services to work on your local machine, you need to set the following environment variables: +``` +DARA_ENDPOINT=http://labs.da-ra.de/dara/ +DARA_USERNAME={see s3://metadatamanagement-private/sensitive_variables.tf} +DARA_PASSWORD={see s3://metadatamanagement-private/sensitive_variables.tf} +``` + If you want to build a docker image for the metadatamanagement server app you can run mvn deploy @@ -90,23 +106,6 @@ these new files. We test our project continuously with the Robot Framework. Test Developers can get further info [here](https://github.com/dzhw/metadatamanagement/wiki/Robot-Framework). -## Technical Documentation - -### Domain Model -The following picture models the relationships and attributes of the domain objects which are managed by our system. -![Domain Model](https://github.com/dzhw/metadatamanagement/wiki/images/domain-model.png) - -Javadoc for our domain model can be found [here](https://dzhw.github.io/metadatamanagement/). - -### Architecture - -A (german) overview of the Systemarchitecture can be found [here](https://github.com/dzhw/metadatamanagement/wiki/Architektur). - -The following picture gives a rough overview: -![Architecture](https://github.com/dzhw/metadatamanagement/wiki/images/architecture/aws_components_overview.png) - -This project is currently built and deployed to AWS Fargate by [Github Actions][GithubActions] (not TravisCI anymore as shown in the picture above). You can test the latest version on [our dev stage.](https://dev.metadata.fdz.dzhw.eu/) - # Big Thanks Cross-browser Testing Platform and Open Source :heart: Provided by [Sauce Labs][saucelabs] From 882c467beccebda839a96a8170238da95e60cb77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Reitmann?= Date: Tue, 19 Oct 2021 14:22:55 +0200 Subject: [PATCH 03/18] add maildev and identity provider --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d672234b0..512f5ce38c 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Before starting the app on your local machine you need to start the following Do # for later use once the containers are created docker-compose start -to start all services the metadatamanagement depends on. MongoDB and Elasticsearch will be listening on its default ports. +to start all services the metadatamanagement depends on. MongoDB and Elasticsearch will be listening on its default ports. MailDev will show all locally sent email on 8081 and the identity-provider can be setup on port 8082. Make sure that you have read-write-access on the ***data*** directory (in your project directory) for Elasticsearch. From a19dc581bf3a6a973bae82cdd64577f9510a9500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Reitmann?= Date: Thu, 21 Oct 2021 12:27:39 +0200 Subject: [PATCH 04/18] add automatic deletion for script attachments, index tags (#2978) --- .../domain/Script.java | 2 +- .../AnalysisPackageChangesProvider.java | 37 ++++++- .../service/ScriptAttachmentService.java | 36 ++++++- .../analysis_packages/mapping.json | 29 ++++++ .../elasticsearch/indices_version.json | 2 +- .../rest/ScriptAttachmentResourceTest.java | 99 +++++++++++++++++++ 6 files changed, 201 insertions(+), 4 deletions(-) diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/domain/Script.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/domain/Script.java index 288855c628..f79f3051ed 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/domain/Script.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/domain/Script.java @@ -39,7 +39,7 @@ public class Script implements Serializable { * Client side generated id of the script. Used to reference script attachments. Not unique in the * DB cause shadow copies of {@link AnalysisPackage}s do not changes this id. * - * Must not be empty. + * Must not be empty and must be unique within the {@link AnalysisPackage}. */ @NotEmpty(message = "analysis-package-management.error.script.uuid.not-empty") private String uuid; diff --git a/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/service/AnalysisPackageChangesProvider.java b/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/service/AnalysisPackageChangesProvider.java index 88cba0adf5..38911edff9 100644 --- a/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/service/AnalysisPackageChangesProvider.java +++ b/src/main/java/eu/dzhw/fdz/metadatamanagement/analysispackagemanagement/service/AnalysisPackageChangesProvider.java @@ -1,14 +1,19 @@ package eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.service; +import java.util.ArrayList; +import java.util.List; + import org.springframework.stereotype.Component; import org.springframework.web.context.annotation.RequestScope; import eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.domain.AnalysisPackage; +import eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.domain.Script; +import eu.dzhw.fdz.metadatamanagement.analysispackagemanagement.domain.ScriptAttachmentMetadata; import eu.dzhw.fdz.metadatamanagement.common.service.DomainObjectChangesProvider; /** * Remember the previous version of a analysis package per request in order to update elasticsearch - * correctly. + * correctly or in order to delete {@link ScriptAttachmentMetadata}. * * @author René Reitmann */ @@ -16,4 +21,34 @@ @RequestScope public class AnalysisPackageChangesProvider extends DomainObjectChangesProvider { + /** + * Return a list of all {@link Script}s which have been removed from the given + * {@link AnalysisPackage}. + * + * @param analysisPackageId The id of the {@link AnalysisPackage} + * @return list of deleted {@link Script}s + */ + public List