Merge remote-tracking branch 'refs/remotes/origin/develop'

# Conflicts: # build.gradle
AtlasOfLivingAustralia · Aug 3, 2024 · de38b6b · de38b6b
2 parents 47ed309 + dd8d3ce
commit de38b6b
Show file tree

Hide file tree

Showing 42 changed files with 3,722 additions and 2,869 deletions.
diff --git a/build.gradle b/build.gradle
@@ -8,7 +8,7 @@ import java.time.Instant
 //        mavenCentral()
 //    }
 //    dependencies {
-//        classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.5.7'
+//        classpath 'org.springframework.boot:spring-boot-gradle-plugin:2.7.0'
 //    }
 //}
 
@@ -40,7 +40,7 @@ repositories {
 }
 
 group = 'au.org.ala'
-version = '3.4.1'
+version = '3.5.0-SNAPSHOT'
 
 
 boolean inplace = false
@@ -55,15 +55,15 @@ dependencies {
 
     compileOnly 'org.projectlombok:lombok:1.18.24'
     annotationProcessor 'org.projectlombok:lombok:1.18.24'
-    developmentOnly 'org.springframework.boot:spring-boot-devtools:2.5.7'
+    developmentOnly 'org.springframework.boot:spring-boot-devtools:2.7.0'
 
-//    implementation 'org.springframework.boot:spring-boot-starter-parent:2.5.7'
+//    implementation 'org.springframework.boot:spring-boot-starter-parent:2.7.0'
 
-    implementation 'org.springframework.boot:spring-boot-starter-web:2.5.7'
+    implementation 'org.springframework.boot:spring-boot-starter-web:2.7.0'
     implementation 'org.apache.tomcat.embed:tomcat-embed-jasper:9.0.55'
-    implementation 'org.springframework.boot:spring-boot-starter-cache:2.5.7'
-    implementation 'org.springframework.boot:spring-boot-starter-actuator:2.5.7'
-    implementation 'org.springframework.boot:spring-boot-starter-validation:2.5.7'
+    implementation 'org.springframework.boot:spring-boot-starter-cache:2.7.0'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator:2.7.0'
+    implementation 'org.springframework.boot:spring-boot-starter-validation:2.7.0'
     implementation 'org.springframework:spring-oxm:5.3.13'
 
     implementation 'org.ehcache:ehcache:3.8.1'
@@ -94,16 +94,16 @@ dependencies {
     implementation 'taglibs:standard:1.1.2'
     implementation 'com.jayway.jsonpath:json-path:2.4.0'
     implementation 'com.sun.mail:jakarta.mail:2.0.1'
-    implementation 'org.springframework.boot:spring-boot-starter-oauth2-client:2.5.7'
-    implementation 'org.springframework.boot:spring-boot-starter-security:2.5.7'
-    implementation 'org.springframework.boot:spring-boot-devtools:2.5.7'
+    implementation 'org.springframework.boot:spring-boot-starter-oauth2-client:2.7.0'
+    implementation 'org.springframework.boot:spring-boot-starter-security:2.7.0'
+    implementation 'org.springframework.boot:spring-boot-devtools:2.7.0'
     implementation 'commons-collections:commons-collections:3.2.2'
     implementation 'io.reactivex.rxjava3:rxjava:3.0.6'
     implementation 'opensymphony:sitemesh:2.4.2'
     implementation 'com.thoughtworks.xstream:xstream:1.4.18'
-    implementation 'org.slf4j:slf4j-api:1.7.24'
-    implementation 'org.slf4j:jul-to-slf4j:1.7.24'
-    implementation 'org.slf4j:slf4j-log4j12:1.7.24'
+    implementation 'org.slf4j:slf4j-api:1.7.36'
+    implementation 'org.slf4j:jul-to-slf4j:1.7.36'
+    implementation 'org.slf4j:slf4j-log4j12:1.7.36'
     implementation 'org.apache.velocity:velocity:1.5'
     implementation 'org.json:json:20210307'
     implementation 'atg.taglib.json:json-taglib:0.4.1'
@@ -122,7 +122,7 @@ dependencies {
     implementation 'org.gbif:dwc-api:1.47'
     implementation 'org.apache.commons:commons-compress:1.21'
     implementation 'com.google.code.findbugs:jsr305:3.0.2'
-    implementation 'org.springframework.boot:spring-boot-autoconfigure:2.5.7'
+    implementation 'org.springframework.boot:spring-boot-autoconfigure:2.7.0'
 
     if (inplace) {
 
@@ -131,7 +131,7 @@ dependencies {
 
     } else {
 
-        implementation 'au.org.ala:ala-ws-spring-security:6.2.0'
+        implementation 'au.org.ala:ala-ws-spring-security:6.3.0-SNAPSHOT'
     }
 
     implementation 'org.codehaus.groovy:groovy-all:3.0.11'
@@ -149,7 +149,7 @@ dependencies {
     testImplementation 'net.bytebuddy:byte-buddy:1.10.13'
     testImplementation 'org.hamcrest:hamcrest:2.1'
     testImplementation 'org.hamcrest:hamcrest-library:2.1'
-    testImplementation 'org.springframework.security:spring-security-test:5.6.0'
+    testImplementation 'org.springframework.security:spring-security-test:5.7.1'
     testImplementation 'com.squareup.retrofit2:retrofit-mock:2.6.2'
     testImplementation 'org.codehaus.groovy:groovy:2.5.14'
     testImplementation 'org.springframework:spring-tx:5.3.9'

diff --git a/scripts/importUserProperties.py b/scripts/importUserProperties.py
@@ -0,0 +1,49 @@
+import csv, json, sys;
+from collections import defaultdict
+from cassandra.cluster import Cluster
+
+# Purpose:
+# To copy user properties from userdetails mysql table "profiles" to cassandra table "userproperty".
+#
+# Prerequisites:
+# 1. python and cassandra-driver
+#   pip install cassandra-driver
+# 2. cassandra table "userproperties". biocache-service will create it on the first addition of a property,
+#    but you can create it manually with cqlsh:
+#   CREATE TABLE biocache.userproperty (
+#        key text PRIMARY KEY,
+#        value text
+#    );
+#
+# Steps:
+# 1. dump userdetails "profiles" table, but only for custom properties, e.g.
+#   mysql userdetailsdb -e "select * from profiles where profiles.property like 'ala-hub.%' or profiles.property like 'null.%' or profiles.property like 'biocache-hub.%';" > userProperties.tsv
+# 2. import userProperties.tsv to cassandra at localhost:9042, keyspace "biocache", table "userproperty"
+#   python3 convertTsvToJson.py < userProperties.tsv
+
+data = [dict(r) for r in csv.DictReader(sys.stdin, delimiter="\t")]
+
+aggregatedByUserId = defaultdict(dict)
+
+# Iterate over the list of dictionaries
+for r in data:
+    # Aggregate the data by r.userid into a map
+    aggregatedByUserId[r['userid']][r['property']] = r['value']
+
+# connect to cassandra
+cluster = Cluster(['localhost'])
+session = cluster.connect()
+
+# insert query
+query = """
+INSERT INTO biocache.userproperty (key, value)
+VALUES (%s, %s)
+"""
+
+# write to cassandra
+for k, v in aggregatedByUserId.items():
+    text = json.dumps({**{'alaId': k}, **{'properties': v} })
+    session.execute(query, (k, text))
+
+session.shutdown()
+cluster.shutdown()
diff --git a/src/main/java/au/org/ala/biocache/config/AppConfig.java b/src/main/java/au/org/ala/biocache/config/AppConfig.java
@@ -1,8 +1,6 @@
 package au.org.ala.biocache.config;
 
-import au.org.ala.biocache.service.NameMatchSpeciesLookupService;
-import au.org.ala.biocache.service.RestartDataService;
-import au.org.ala.biocache.service.SpeciesLookupService;
+import au.org.ala.biocache.service.*;
 import au.org.ala.biocache.util.converter.FqConverter;
 import au.org.ala.dataquality.api.QualityServiceRpcApi;
 import au.org.ala.dataquality.client.ApiClient;
@@ -101,6 +99,17 @@ protected SpeciesLookupService getNameMatchSpeciesLookupService() {
         return getNameMatchSpeciesLookupService();
     }
 
+    protected SpeciesSearchService getNameMatchSpeciesSearchService() {
+        logger.info("Initialising name match species lookup services.");
+        NameMatchSpeciesSearchService service = new NameMatchSpeciesSearchService();
+        return service;
+    }
+
+    public @Bean(name = "speciesSearchService") SpeciesSearchService speciesSearchServiceBean() {
+        logger.info("Initialising species lookup services.");
+        return getNameMatchSpeciesSearchService();
+    }
+
     @Bean("dataQualityApiClient")
     public ApiClient dataQualityApiClient() {
         ApiClient apiClient = new ApiClient();

diff --git a/src/main/java/au/org/ala/biocache/config/Application.java b/src/main/java/au/org/ala/biocache/config/Application.java
@@ -15,7 +15,6 @@
 @SpringBootApplication
 @ImportResource("classpath:spring.xml")
 @ComponentScan( { "au.org.ala.biocache.*" , "au.org.ala.ws.config" })
-@PropertySource(value="file:///data/biocache/config/biocache-config.properties", ignoreResourceNotFound=true)
 public class Application extends SpringBootServletInitializer {
 
     @Override
@@ -41,4 +40,4 @@ public OpenAPI customOpenAPI(
     public static void main(String[] args) {
         SpringApplication.run(Application.class, args);
     }
-}
+}
diff --git a/src/main/java/au/org/ala/biocache/config/SecurityConfig.java b/src/main/java/au/org/ala/biocache/config/SecurityConfig.java
@@ -4,13 +4,15 @@
 
 import org.pac4j.core.config.Config;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.annotation.Order;
 import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
 import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.core.GrantedAuthorityDefaults;
 import org.springframework.security.web.authentication.www.BasicAuthenticationFilter;
 
 import javax.inject.Inject;
@@ -37,4 +39,9 @@ protected void configure(HttpSecurity http) throws Exception {
                 ).permitAll()
                 .and().csrf().disable();
     }
+
+    @Bean
+    public GrantedAuthorityDefaults grantedAuthorityDefaults() {
+        return  new GrantedAuthorityDefaults("");
+    }
 }
diff --git a/src/main/java/au/org/ala/biocache/dao/QidCacheDAO.java b/src/main/java/au/org/ala/biocache/dao/QidCacheDAO.java
@@ -73,7 +73,7 @@ public interface QidCacheDAO {
 
     String[] getFq(SpatialSearchRequestDTO requestParams);
 
-    String generateQid(SpatialSearchRequestDTO requestParams, String bbox, String title, Long maxage, String source);
+    String generateQid(SpatialSearchRequestDTO requestParams, double[] bbox, String title, Long maxage, String source);
 
     String fixWkt(String wkt) throws Exception;
 

diff --git a/src/main/java/au/org/ala/biocache/dao/QidCacheDAOImpl.java b/src/main/java/au/org/ala/biocache/dao/QidCacheDAOImpl.java
@@ -19,7 +19,6 @@
 import au.org.ala.biocache.service.DataQualityService;
 import au.org.ala.biocache.util.QidMissingException;
 import au.org.ala.biocache.util.QidSizeException;
-import au.org.ala.biocache.util.QueryFormatUtils;
 import au.org.ala.biocache.util.SpatialUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -74,9 +73,6 @@ public class QidCacheDAOImpl implements QidCacheDAO {
     @Inject
     private DataQualityService dataQualityService;
 
-    @Inject
-    protected QueryFormatUtils queryFormatUtils;
-
     /**
      * in memory store of params
      */
@@ -98,9 +94,6 @@ public class QidCacheDAOImpl implements QidCacheDAO {
      */
     private Thread cacheCleaner;
 
-    @Inject
-    private SearchDAO searchDAO;
-
     @Inject
     private StoreDAO storeDao;
 
@@ -411,25 +404,14 @@ public String[] getFq(SpatialSearchRequestDTO requestParams) {
 
     @Cacheable("qidGeneration")
     @Override
-    public String generateQid(SpatialSearchRequestDTO requestParams, String bbox, String title, Long maxage, String source) {
+    public String generateQid(SpatialSearchRequestDTO requestParams, double[] bbox, String title, Long maxage, String source) {
         try {
             //simplify wkt
             requestParams.setWkt(fixWkt(requestParams.getWkt()));
 
-            //get bbox (also cleans up Q)
-            double[] bb = null;
-            if (bbox != null && bbox.equals("true")) {
-                try {
-                    bb = searchDAO.getBBox(requestParams);
-                } catch (Exception e) {
-                    // When there are no occurrences for the query return a usable bounding box
-                    bb = new double []{-180, -90, 180, 90};
-                }
-            } else {
+            if (bbox == null) {
                 requestParams.setPageSize(0);
                 requestParams.setFacet(false);
-                //get a formatted Q
-                queryFormatUtils.formatSearchQuery(requestParams);
             }
 
             //store the title if necessary
@@ -440,7 +422,7 @@ public String generateQid(SpatialSearchRequestDTO requestParams, String bbox, St
             if (fqs.length == 0 || (fqs.length == 1 && fqs[0].length() == 0)) {
                 fqs = null;
             }
-            String qid = put(requestParams.getQ(), title, requestParams.getWkt(), bb, fqs, maxage, source);
+            String qid = put(requestParams.getQ(), title, requestParams.getWkt(), bbox, fqs, maxage, source);
 
             return qid;
         } catch (Exception e) {

diff --git a/src/main/java/au/org/ala/biocache/dao/SearchDAO.java b/src/main/java/au/org/ala/biocache/dao/SearchDAO.java
@@ -19,17 +19,17 @@
 import au.org.ala.biocache.util.LegendItem;
 import au.org.ala.biocache.util.QidMissingException;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.response.FacetField;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 
 import javax.servlet.ServletOutputStream;
 import java.io.OutputStream;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * DAO for searching occurrence records held in the biocache.
@@ -179,7 +179,7 @@ public interface SearchDAO {
      * @return
      * @throws Exception
      */
-    void findAllSpeciesJSON(SpatialSearchRequestDTO requestParams, OutputStream outputStream) throws Exception;
+    void findAllSpeciesJSON(SpatialSearchRequestDTO requestParams, Boolean includeRank, OutputStream outputStream) throws Exception;
 
     /**
      * Find all species (and counts) for a given query and stream as CSV
@@ -188,7 +188,7 @@ public interface SearchDAO {
      * @return
      * @throws Exception
      */
-    void findAllSpeciesCSV(SpatialSearchRequestDTO requestParams, OutputStream outputStream) throws Exception;
+    void findAllSpeciesCSV(SpatialSearchRequestDTO requestParams, Boolean includeRank,OutputStream outputStream) throws Exception;
 
     /**
      * Find all occurrences for a given query as SolrDocumentList
@@ -375,4 +375,17 @@ HeatmapDTO getHeatMap(String query, String[] filterQueries, Double minx, Double
     SolrQuery initSolrQuery(SpatialSearchRequestDTO searchParams, boolean substituteDefaultFacetOrder, Map<String, String[]> extraSolrParams) throws QidMissingException;
 
     boolean isInitialized() throws InterruptedException;
+
+    /**
+     * Retrieve an OccurrencePoint (distinct list of points - lat-long to 4 decimal places) for a given search
+     * without convertion to List of OccurrencePoint
+     *
+     * @param searchParams
+     * @param pointType
+     * @return
+     * @throws Exception
+     */
+    FacetField getFacetPointsShort(SpatialSearchRequestDTO searchParams, String pointType, Double minx, Double miny, Double maxx, Double maxy) throws Exception;
+
+    SolrDocument getOcc(String recordUuid);
 }