catalog: implement quota restrictions, #TASK-6446

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java

@@ -25,6 +25,7 @@
 import org.opencb.opencga.catalog.exceptions.CatalogException;
 import org.opencb.opencga.catalog.exceptions.CatalogParameterException;
 import org.opencb.opencga.core.api.ParamConstants;
+import org.opencb.opencga.core.models.job.ExecutionTime;
 import org.opencb.opencga.core.models.job.Job;
 import org.opencb.opencga.core.response.OpenCGAResult;
 
@@ -99,6 +100,9 @@ OpenCGAResult<Job> getAllInStudy(long studyId, QueryOptions options)
      */
     OpenCGAResult unmarkPermissionRule(long studyId, String permissionRuleId) throws CatalogException;
 
+    OpenCGAResult<ExecutionTime> executionTimeByMonth(Query query)
+            throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException;
+
     enum QueryParams implements QueryParam {
         ID("id", TEXT, ""),
         UID("uid", LONG, ""),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java

@@ -17,7 +17,9 @@
 package org.opencb.opencga.catalog.db.mongodb;
 
 import com.mongodb.client.ClientSession;
+import com.mongodb.client.model.Aggregates;
 import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Projections;
 import org.apache.commons.lang3.NotImplementedException;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.StopWatch;
@@ -44,10 +46,7 @@
 import org.opencb.opencga.core.config.Configuration;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
-import org.opencb.opencga.core.models.job.Job;
-import org.opencb.opencga.core.models.job.JobInternalWebhook;
-import org.opencb.opencga.core.models.job.JobPermissions;
-import org.opencb.opencga.core.models.job.ToolInfo;
+import org.opencb.opencga.core.models.job.*;
 import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.slf4j.LoggerFactory;
@@ -294,6 +293,45 @@ OpenCGAResult<Object> privateUpdate(ClientSession clientSession, Job job, Object
         return endWrite(tmpStartTime, 1, 1, events);
     }
 
+    @Override
+    public OpenCGAResult<ExecutionTime> executionTimeByMonth(Query query)
+            throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException {
+        long startTime = startQuery();
+
+        Bson bsonQuery = parseQuery(query, QueryOptions.empty());
+        List<Bson> aggregation = new ArrayList<>();
+        aggregation.add(Aggregates.match(bsonQuery));
+        aggregation.add(Aggregates.project(Projections.fields(
+                Projections.computed("date", "$" + PRIVATE_MODIFICATION_DATE),
+                Projections.computed("difference", new Document("$toDouble",
+                        new Document("$subtract",
+                                Arrays.asList("$" + QueryParams.EXECUTION_END.key(), "$" + QueryParams.EXECUTION_START.key()))))
+                )));
+        aggregation.add(new Document("$group", new Document()
+                .append("_id", new Document()
+                        .append("month", new Document("$month", "$date"))
+                        .append("year", new Document("$year", "$date")))
+                .append("sum", new Document("$sum", "$difference"))));
+
+        DataResult<Document> aggregate = jobCollection.aggregate(aggregation, QueryOptions.empty());
+        // Result comes in this format:
+        // { "_id" : { "month" : 5, "year" : 2024 }, "sum" : 13196 }
+
+        // Parse result
+        List<ExecutionTime> executionTimeList = new ArrayList<>(aggregate.getNumResults());
+        for (Document result : aggregate.getResults()) {
+            Document id = result.get("_id", Document.class);
+            String month = id.getInteger("month").toString();
+            String year = id.getInteger("year").toString();
+            double seconds = result.get("sum", Number.class).doubleValue() / 1000; // convert milliseconds to seconds
+            double minutes = seconds / 60.0;
+            double hours = minutes / 60.0;
+            ExecutionTime.Time time = new ExecutionTime.Time(hours, minutes, seconds);
+            executionTimeList.add(new ExecutionTime(month, year, time));
+        }
+        return endQuery(startTime, executionTimeList);
+    }
+
     @Override
     public OpenCGAResult delete(Job job) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException {
         try {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java

@@ -572,9 +572,11 @@ public OpenCGAResult<Job> submit(String studyStr, String toolId, Enums.Priority
         job.setAttributes(attributes);
         try {
             autoCompleteNewJob(organizationId, study, job, tokenPayload);
-
             authorizationManager.checkStudyPermission(organizationId, study.getUid(), userId, StudyPermissions.Permissions.EXECUTE_JOBS);
 
+            // Check if we have already reached the limit of job hours in the Organisation
+            checkExecutionLimitQuota(organizationId);
+
             // Check params
             ParamUtils.checkObj(params, "params");
             for (Map.Entry<String, Object> entry : params.entrySet()) {
@@ -613,6 +615,27 @@ public OpenCGAResult<Job> submit(String studyStr, String toolId, Enums.Priority
         }
     }
 
+    public OpenCGAResult<ExecutionTime> getExecutionTimeByMonth(String organizationId, Query query, String token) throws CatalogException {
+        JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token);
+        ParamUtils.checkParameter(organizationId, "organizationId");
+        authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, tokenPayload.getUserId(organizationId));
+        return getJobDBAdaptor(organizationId).executionTimeByMonth(query);
+    }
+
+    private void checkExecutionLimitQuota(String organizationId) throws CatalogException {
+        // Get current year/month
+        String time = TimeUtils.getTime(TimeUtils.getDate(), TimeUtils.yyyyMM);
+        Query query = new Query(JobDBAdaptor.QueryParams.MODIFICATION_DATE.key(), time);
+        OpenCGAResult<ExecutionTime> result = getJobDBAdaptor(organizationId).executionTimeByMonth(query);
+        if (result.getNumResults() > 0) {
+            ExecutionTime executionTime = result.first();
+            if (executionTime.getTime().getHours() >= configuration.getQuota().getMaxNumJobHours()) {
+                throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of execution hours ("
+                        + configuration.getQuota().getMaxNumJobHours() + ") for the current month.");
+            }
+        }
+    }
+
     /**
      * Check if the job is eligible to be reused, and if so, try to find an equivalent job.
      * Eligible job:

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ProjectManager.java

@@ -168,6 +168,9 @@ public OpenCGAResult<Project> create(ProjectCreateParams projectCreateParams, Qu
             authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId);
             ParamUtils.checkObj(projectCreateParams, "ProjectCreateParams");
             project = projectCreateParams.toProject();
+
+            // Check if we have already reached the limit of projects in the Organisation
+            checkProjectLimitQuota(organizationId);
             validateProjectForCreation(organizationId, project);
 
             queryResult = getProjectDBAdaptor(organizationId).insert(project, options);
@@ -202,6 +205,14 @@ public OpenCGAResult<Project> create(ProjectCreateParams projectCreateParams, Qu
         return queryResult;
     }
 
+    private void checkProjectLimitQuota(String organizationId) throws CatalogException {
+        long numProjects = getProjectDBAdaptor(organizationId).count(new Query()).getNumMatches();
+        if (numProjects >= configuration.getQuota().getMaxNumProjects()) {
+            throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of projects ("
+                    + configuration.getQuota().getMaxNumProjects() + ").");
+        }
+    }
+
     private void validateProjectForCreation(String organizationId, Project project) throws CatalogParameterException {
         ParamUtils.checkParameter(project.getId(), ProjectDBAdaptor.QueryParams.ID.key());
         project.setName(ParamUtils.defaultString(project.getName(), project.getId()));

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java

@@ -194,6 +194,10 @@ public OpenCGAResult<User> create(User user, String password, String token) thro
             }
         }
 
+        // Check if we have already reached the limit of users in the Organisation
+        checkUserLimitQuota(organizationId);
+
+        // Check if the user already exists
         checkUserExists(organizationId, user.getId());
 
         try {
@@ -218,6 +222,15 @@ public OpenCGAResult<User> create(User user, String password, String token) thro
         }
     }
 
+    private void checkUserLimitQuota(String organizationId) throws CatalogException {
+        // Check if we have already reached the limit of users in the Organisation
+        long numUsers = getUserDBAdaptor(organizationId).count(new Query()).getNumMatches();
+        if (numUsers >= configuration.getQuota().getMaxNumUsers()) {
+            throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of allowed users ("
+                    + configuration.getQuota().getMaxNumUsers() + ").");
+        }
+    }
+
     /**
      * Create a new user.
      *

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java

@@ -30,6 +30,8 @@
 import org.opencb.commons.test.GenericTest;
 import org.opencb.opencga.TestParamConstants;
 import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager;
+import org.opencb.opencga.catalog.db.api.JobDBAdaptor;
+import org.opencb.opencga.catalog.db.api.ProjectDBAdaptor;
 import org.opencb.opencga.catalog.db.api.UserDBAdaptor;
 import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils;
 import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
@@ -481,8 +483,19 @@ protected CatalogManager mockCatalogManager() throws CatalogDBException {
         UserManager userManager = spy.getUserManager();
         UserManager userManagerSpy = Mockito.spy(userManager);
         Mockito.doReturn(userManagerSpy).when(spy).getUserManager();
-        MongoDBAdaptorFactory mongoDBAdaptorFactory = mockMongoDBAdaptorFactory();
-        Mockito.doReturn(mongoDBAdaptorFactory).when(userManagerSpy).getCatalogDBAdaptorFactory();
+        MongoDBAdaptorFactory mockMongoFactory = mockMongoDBAdaptorFactory();
+        Mockito.doReturn(mockMongoFactory).when(userManagerSpy).getCatalogDBAdaptorFactory();
+
+        JobManager jobManager = spy.getJobManager();
+        JobManager jobManagerSpy = Mockito.spy(jobManager);
+        Mockito.doReturn(jobManagerSpy).when(spy).getJobManager();
+        Mockito.doReturn(mockMongoFactory).when(jobManagerSpy).getCatalogDBAdaptorFactory();
+
+        ProjectManager projectManager = spy.getProjectManager();
+        ProjectManager projectManagerSpy = Mockito.spy(projectManager);
+        Mockito.doReturn(projectManagerSpy).when(spy).getProjectManager();
+        Mockito.doReturn(mockMongoFactory).when(projectManagerSpy).getCatalogDBAdaptorFactory();
+
         return spy;
     }
 
@@ -492,6 +505,15 @@ protected MongoDBAdaptorFactory mockMongoDBAdaptorFactory() throws CatalogDBExce
         UserDBAdaptor userDBAdaptor = dbAdaptorFactorySpy.getCatalogUserDBAdaptor(organizationId);
         UserDBAdaptor userDBAdaptorSpy = Mockito.spy(userDBAdaptor);
         Mockito.doReturn(userDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogUserDBAdaptor(organizationId);
+
+        JobDBAdaptor jobDBAdaptor = dbAdaptorFactorySpy.getCatalogJobDBAdaptor(organizationId);
+        JobDBAdaptor jobDBAdaptorSpy = Mockito.spy(jobDBAdaptor);
+        Mockito.doReturn(jobDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogJobDBAdaptor(organizationId);
+
+        ProjectDBAdaptor projectDBAdaptor = dbAdaptorFactorySpy.getCatalogProjectDbAdaptor(organizationId);
+        ProjectDBAdaptor projectDBAdaptorSpy = Mockito.spy(projectDBAdaptor);
+        Mockito.doReturn(projectDBAdaptorSpy).when(dbAdaptorFactorySpy).getCatalogProjectDbAdaptor(organizationId);
+
         return dbAdaptorFactorySpy;
     }
 

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerTest.java

@@ -22,6 +22,7 @@
 import org.apache.commons.lang3.time.StopWatch;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 import org.opencb.biodata.models.common.Status;
 import org.opencb.biodata.models.pedigree.IndividualProperty;
 import org.opencb.commons.datastore.core.DataResult;
@@ -970,6 +971,32 @@ public void visitJob() throws CatalogException {
         assertEquals(1, catalogManager.getJobManager().count(studyFqn, query, ownerToken).getNumMatches());
     }
 
+    @Test
+    public void testJobQuotaLimit() throws CatalogException {
+        // Submit a dummy job. This shouldn't raise any error
+        catalogManager.getJobManager().submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken);
+
+        OpenCGAResult<ExecutionTime> result = catalogManager.getJobManager().getExecutionTimeByMonth(organizationId, new Query(), ownerToken);
+        assertEquals(1, result.getNumResults());
+        assertEquals(0, result.first().getTime().getHours(), 0.0);
+        assertEquals(0, result.first().getTime().getMinutes(), 0.0);
+        assertEquals(0, result.first().getTime().getSeconds(), 0.0);
+
+        try (CatalogManager mockManager = mockCatalogManager()) {
+            // Mock check result
+            OpenCGAResult<ExecutionTime> results = new OpenCGAResult<>(0, Collections.singletonList(new ExecutionTime("1", "2024",
+                    new ExecutionTime.Time(1000.0, 1000 * 60.0, 1000.0 * 60 * 60))));
+            JobDBAdaptor jobDBAdaptor = mockManager.getJobManager().getJobDBAdaptor(organizationId);
+
+            Mockito.doReturn(results).when(jobDBAdaptor).executionTimeByMonth(Mockito.any(Query.class));
+
+            // Submit a job. This should raise an error
+            CatalogException exception = assertThrows(CatalogException.class, () -> mockManager.getJobManager()
+                    .submit(studyId, "command-subcommand", null, Collections.emptyMap(), ownerToken));
+            assertTrue(exception.getMessage().contains("quota"));
+        }
+    }
+
     /**
      * VariableSet methods ***************************
      */

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ProjectManagerTest.java

@@ -20,6 +20,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 import org.opencb.commons.datastore.core.DataResult;
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.Query;
@@ -52,6 +53,26 @@
 @Category(MediumTests.class)
 public class ProjectManagerTest extends AbstractManagerTest {
 
+    @Test
+    public void createProjectQuotaTest() throws CatalogException {
+        try (CatalogManager mockCatalogManager = mockCatalogManager()) {
+            ProjectDBAdaptor projectDBAdaptor = mockCatalogManager.getProjectManager().getProjectDBAdaptor(organizationId);
+
+            // Mock there already exists 50 projects
+            OpenCGAResult<Project> result = new OpenCGAResult<>(0, Collections.emptyList());
+            result.setNumMatches(50);
+            Mockito.doReturn(result).when(projectDBAdaptor).count();
+            Mockito.doReturn(result).when(projectDBAdaptor).count(Mockito.any(Query.class));
+
+            ProjectCreateParams projectCreateParams = new ProjectCreateParams()
+                    .setId("newProject")
+                    .setName("Project about some genomes");
+            CatalogException exception = assertThrows(CatalogException.class,
+                    () -> mockCatalogManager.getProjectManager().create(projectCreateParams, QueryOptions.empty(), ownerToken));
+            assertTrue(exception.getMessage().contains("quota"));
+        }
+    }
+
     @Test
     public void searchProjectByStudy() throws CatalogException {
         OpenCGAResult<Project> result = catalogManager.getProjectManager().search(organizationId, new Query(ProjectDBAdaptor.QueryParams.STUDY.key(), "phase1"), null, ownerToken);

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/UserManagerTest.java

@@ -40,7 +40,6 @@
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
 import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper;
 
 @Category(MediumTests.class)
@@ -507,6 +506,24 @@ public void loginUserPasswordExpiredTest() throws CatalogException {
         }
     }
 
+    @Test
+    public void createUserQuotaTest() throws CatalogException {
+        try (CatalogManager mockCatalogManager = mockCatalogManager()) {
+            UserDBAdaptor userDBAdaptor = mockCatalogManager.getUserManager().getUserDBAdaptor(organizationId);
+
+            // Mock there already exists 50 users
+            OpenCGAResult<User> result = new OpenCGAResult<>(0, Collections.emptyList());
+            result.setNumMatches(50);
+            Mockito.doReturn(result).when(userDBAdaptor).count();
+            Mockito.doReturn(result).when(userDBAdaptor).count(Mockito.any(Query.class));
+
+            User user = new User("newUser");
+            CatalogException exception = assertThrows(CatalogException.class,
+                    () -> mockCatalogManager.getUserManager().create(user, TestParamConstants.PASSWORD, ownerToken));
+            assertTrue(exception.getMessage().contains("quota"));
+        }
+    }
+
     @Test
     public void changePasswordTest() throws CatalogException {
         String newPassword = PasswordUtils.getStrongRandomPassword();

opencga-core/src/main/java/org/opencb/opencga/core/common/TimeUtils.java

@@ -30,9 +30,10 @@
 
 public class TimeUtils {
 
-    private static final String yyyyMMdd = "yyyyMMdd";
-    private static final String yyyyMMddHHmmss = "yyyyMMddHHmmss";
-    private static final String yyyyMMddHHmmssSSS = "yyyyMMddHHmmssSSS";
+    public static final String yyyyMM = "yyyyMM";
+    public static final String yyyyMMdd = "yyyyMMdd";
+    public static final String yyyyMMddHHmmss = "yyyyMMddHHmmss";
+    public static final String yyyyMMddHHmmssSSS = "yyyyMMddHHmmssSSS";
 
     private static final Logger logger = LoggerFactory.getLogger(TimeUtils.class);
 
@@ -49,6 +50,11 @@ public static String getTime(Date date) {
         return sdf.format(date);
     }
 
+    public static String getTime(Date date, String format) {
+        SimpleDateFormat sdf = new SimpleDateFormat(format);
+        return sdf.format(date);
+    }
+
     public static String getTimeMillis() {
         return getTimeMillis(new Date());
     }