sync only changed parts in jira sync

jira/src/main/kotlin/gropius/sync/jira/JiraDataService.kt

@@ -256,6 +256,31 @@ class JiraDataService(
         }
     }
 
+    /**
+     * Collect list of users to request with using
+     *
+     * @param imsProject the project to work on
+     * @param users the list of users specific to this TimelineItem
+     * @return all users sorted by most fitting first
+     */
+    suspend fun collectRequestUsers(
+        imsProject: IMSProject, users: List<User>
+    ): List<User> {
+        val imsConfig = IMSConfig(helper, imsProject.ims().value, imsProject.ims().value.template().value)
+        val rawUserList = users.toMutableList()
+        if (imsConfig.readUser != null) {
+            val imsUser = neoOperations.findById(imsConfig.readUser, IMSUser::class.java).awaitSingleOrNull()
+                ?: throw IllegalArgumentException("Read user not found")
+            if (imsUser.ims().value != imsProject.ims().value) {
+                TODO("Error handling")
+            }
+            rawUserList.add(imsUser)
+        }
+        val userList = rawUserList.distinct()
+        logger.info("Requesting with users: $userList")
+        return userList
+    }
+
     /**
      * Process a request for a given set of users
      *
@@ -275,18 +300,7 @@ class JiraDataService(
         body: T? = null,
         crossinline urlBuilder: URLBuilder .(URLBuilder) -> Unit
     ): Pair<IMSUser, HttpResponse> {
-        val imsConfig = IMSConfig(helper, imsProject.ims().value, imsProject.ims().value.template().value)
-        val rawUserList = users.toMutableList()
-        if (imsConfig.readUser != null) {
-            val imsUser = neoOperations.findById(imsConfig.readUser, IMSUser::class.java).awaitSingleOrNull()
-                ?: throw IllegalArgumentException("Read user not found")
-            if (imsUser.ims().value != imsProject.ims().value) {
-                TODO("Error handling")
-            }
-            rawUserList.add(imsUser)
-        }
-        val userList = rawUserList.distinct()
-        logger.info("Requesting with users: $userList")
+        val userList = collectRequestUsers(imsProject, users)
         return tokenManager.executeUntilWorking(imsProject, userList, owner) {
             sendRequest<T>(
                 imsProject, requestMethod, body, urlBuilder, it

jira/src/main/kotlin/gropius/sync/jira/JiraSync.kt

@@ -8,6 +8,7 @@ import gropius.model.template.IMSTemplate
 import gropius.model.template.IssueState
 import gropius.model.user.User
 import gropius.sync.*
+import gropius.sync.jira.config.IMSConfig
 import gropius.sync.jira.config.IMSConfigManager
 import gropius.sync.jira.config.IMSProjectConfig
 import gropius.sync.jira.model.*
@@ -17,6 +18,9 @@ import io.ktor.http.*
 import kotlinx.serialization.json.*
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Component
+import java.time.OffsetDateTime
+import java.time.ZoneId
+import java.time.format.DateTimeFormatter
 import kotlin.io.encoding.ExperimentalEncodingApi
 
 /**
@@ -28,6 +32,7 @@ import kotlin.io.encoding.ExperimentalEncodingApi
  * @param collectedSyncInfo the collected sync info
  * @param loadBalancedDataFetcher the load balanced data fetcher
  * @param issueDataService the data service for issues
+ * @param syncStatusService the sync status service
  */
 @Component
 final class JiraSync(
@@ -37,9 +42,17 @@ final class JiraSync(
     val imsConfigManager: IMSConfigManager,
     collectedSyncInfo: CollectedSyncInfo,
     val loadBalancedDataFetcher: LoadBalancedDataFetcher = LoadBalancedDataFetcher(),
-    val issueDataService: IssueDataService
+    val issueDataService: IssueDataService,
+    val syncStatusService: SyncStatusService
 ) : AbstractSync(collectedSyncInfo) {
 
+    companion object {
+        /**
+         * Formatter for JQL dates
+         */
+        val JQL_FORMATTER = DateTimeFormatter.ofPattern("\"yyyy-MM-dd HH:mm\"")
+    }
+
     /**
      * Logger used to print notifications
      */
@@ -92,15 +105,22 @@ final class JiraSync(
         }
 
         for (imsProject in imsProjects) {
-            val issueList = mutableListOf<String>()
-            fetchIssueList(imsProject, issueList)
+            val (issueList, lastSeenTime) = fetchIssueList(imsProject)
             fetchIssueContent(issueList, imsProject)
+            if (lastSeenTime != null) {
+                syncStatusService.updateTime(imsProject.rawId!!, lastSeenTime)
+            }
         }
     }
 
+    /**
+     * Fetch the changelog of the issues
+     * @param issueList the list of issues
+     * @param imsProject the IMS project
+     */
     @OptIn(ExperimentalEncodingApi::class)
     private suspend fun fetchIssueContentChangelog(
-        issueList: MutableList<String>, imsProject: IMSProject
+        issueList: List<String>, imsProject: IMSProject
     ) {
         for (issueId in issueList) {
             var startAt = 0
@@ -122,9 +142,14 @@ final class JiraSync(
         }
     }
 
+    /**
+     * Fetch the comments of the issues
+     * @param issueList the list of issues
+     * @param imsProject the IMS project
+     */
     @OptIn(ExperimentalEncodingApi::class)
     private suspend fun fetchIssueContentComments(
-        issueList: MutableList<String>, imsProject: IMSProject
+        issueList: List<String>, imsProject: IMSProject
     ) {
         for (issueId in issueList) {
             var startAt = 0
@@ -147,35 +172,101 @@ final class JiraSync(
         }
     }
 
+    /**
+     * Fetch the content of the issues
+     * @param issueList the list of issues
+     * @param imsProject the IMS project
+     */
     @OptIn(ExperimentalEncodingApi::class)
     private suspend fun fetchIssueContent(
-        issueList: MutableList<String>, imsProject: IMSProject
+        issueList: List<String>, imsProject: IMSProject
     ) {
         logger.info("ISSUE LIST $issueList")
-        fetchIssueContentChangelog(issueList, imsProject)
+        val imsConfig = IMSConfig(helper, imsProject.ims().value, imsProject.ims().value.template().value)
         fetchIssueContentComments(issueList, imsProject)
+        if (imsConfig.isCloud) {
+            fetchIssueContentChangelog(issueList, imsProject)
+        }
     }
 
+    /**
+     * Fetch the list of changed issues
+     * @param imsProject the IMS project
+     * @return issueList the list of issues
+     */
     @OptIn(ExperimentalEncodingApi::class)
     private suspend fun fetchIssueList(
-        imsProject: IMSProject, issueList: MutableList<String>
-    ) {
+        imsProject: IMSProject
+    ): Pair<List<String>, OffsetDateTime?> {
+        val issueList = mutableListOf<String>()
         var startAt = 0
+        val lastSuccessfulSync: OffsetDateTime? =
+            syncStatusService.findByImsProject(imsProject.rawId!!)?.lastSuccessfulSync
+        val times = mutableListOf<OffsetDateTime>()
         while (true) {
             val imsProjectConfig = IMSProjectConfig(helper, imsProject)
-            val issueResponse = jiraDataService.request<Unit>(imsProject, listOf(), HttpMethod.Get, listOf()) {
-                appendPathSegments("search")
-                parameters.append("jql", "project=${imsProjectConfig.repo}")
-                parameters.append("expand", "names,schema,editmeta,changelog")
-                parameters.append("startAt", "$startAt")
+            val userList = jiraDataService.collectRequestUsers(imsProject, listOf())
+            val issueResponse = jiraDataService.tokenManager.executeUntilWorking(imsProject, userList, listOf()) {
+                val userTimeZone = ZoneId.of(
+                    jiraDataService.sendRequest<Unit>(
+                        imsProject, HttpMethod.Get, null, {
+                            appendPathSegments("myself")
+                        }, it
+                    ).get().body<UserQuery>().timeZone
+                )
+                var query = "project=${imsProjectConfig.repo}"
+                if (lastSuccessfulSync != null) {
+                    query = "project=${imsProjectConfig.repo} AND updated > ${
+                        lastSuccessfulSync.atZoneSameInstant(userTimeZone).format(JQL_FORMATTER)
+                    }"
+                }
+                logger.info("With $lastSuccessfulSync, ${imsProjectConfig.repo} and $userTimeZone, the query is '$query'")
+                jiraDataService.sendRequest<Unit>(
+                    imsProject, HttpMethod.Get, null, {
+                        appendPathSegments("search")
+                        parameters.append("jql", query)
+                        parameters.append("expand", "names,schema,editmeta,changelog")
+                        parameters.append("startAt", "$startAt")
+                    }, it
+                )
             }.second.body<ProjectQuery>()
             issueResponse.issues(imsProject).forEach {
                 issueList.add(it.jiraId)
                 issueDataService.insertIssue(imsProject, it)
+                for (comment in it.comments.values) {
+                    times.add(
+                        OffsetDateTime.parse(
+                            comment.created, IssueData.formatter
+                        )
+                    )
+                    times.add(
+                        OffsetDateTime.parse(
+                            comment.updated, IssueData.formatter
+                        )
+                    )
+                }
+                for (history in it.changelog.histories) {
+                    times.add(
+                        OffsetDateTime.parse(
+                            history.created, IssueData.formatter
+                        )
+                    )
+                }
+                val updated = it.fields["updated"]
+                if (updated != null) {
+                    times.add(
+                        OffsetDateTime.parse(
+                            updated.jsonPrimitive.content, IssueData.formatter
+                        )
+                    )
+                }
             }
             startAt = issueResponse.startAt + issueResponse.issues.size
-            if (startAt >= issueResponse.total) break
+            if (startAt >= issueResponse.total) {
+                break
+            }
         }
+        return issueList to times.maxOrNull()
     }
 
     override suspend fun findUnsyncedIssues(imsProject: IMSProject): List<IncomingIssue> {

jira/src/main/kotlin/gropius/sync/jira/SyncStatus.kt

@@ -0,0 +1,64 @@
+package gropius.sync.jira
+
+import kotlinx.coroutines.reactor.awaitSingle
+import org.bson.types.ObjectId
+import org.springframework.data.annotation.Id
+import org.springframework.data.mongodb.core.index.Indexed
+import org.springframework.data.mongodb.core.mapping.Document
+import org.springframework.data.mongodb.repository.ReactiveMongoRepository
+import org.springframework.stereotype.Repository
+import org.springframework.stereotype.Service
+import org.springframework.transaction.annotation.Transactional
+import java.time.OffsetDateTime
+
+/**
+ * Base class for storing information about general sync status
+ * @param imsProject IMS project ID
+ * @param lastSuccessfulSync Timestamp of the last successful sync. nullable for future use
+ */
+@Document
+data class SyncStatus(
+    @Indexed
+    val imsProject: String,
+    @Indexed
+    var lastSuccessfulSync: OffsetDateTime?,
+) {
+    /**
+     * MongoDB ID
+     */
+    @Id
+    var id: ObjectId? = null
+}
+
+/**
+ * Repository for accessing the sync status
+ */
+@Repository
+interface SyncStatusRepository : ReactiveMongoRepository<SyncStatus, ObjectId> {
+    /**
+     * Find using the IMSProject ID
+     * @param imsProject IMS project ID
+     */
+    suspend fun findByImsProject(
+        imsProject: String
+    ): SyncStatus?
+}
+
+/**
+ * Service for modifying the sync status
+ */
+@Service
+class SyncStatusService(val syncStatusRepository: SyncStatusRepository) : SyncStatusRepository by syncStatusRepository {
+
+    /**
+     * Update the Time saved in the SyncStatusService
+     * @param imsProject IMS project ID of the SyncStatusService
+     * @param lastSuccessfulSync New Timestamp of the last successful sync
+     */
+    @Transactional
+    suspend fun updateTime(imsProject: String, lastSuccessfulSync: OffsetDateTime) {
+        val status = syncStatusRepository.findByImsProject(imsProject) ?: SyncStatus(imsProject, null)
+        status.lastSuccessfulSync = lastSuccessfulSync
+        syncStatusRepository.save(status).awaitSingle()
+    }
+}

jira/src/main/kotlin/gropius/sync/jira/config/IMSConfig.kt

@@ -15,14 +15,16 @@ import java.net.URI
  * @param imsTemplate the template of the current IMS
  * @param defaultType the type of newly created issues
  * @param defaultTemplate the template of newly created issues
+ * @param isCloud true if cloud, false if data center
  */
 data class IMSConfig(
     val botUser: String,
     val readUser: String?,
     val rootUrl: URI,
     val imsTemplate: IMSTemplate,
     val defaultType: String?,
-    val defaultTemplate: String?
+    val defaultTemplate: String?,
+    val isCloud: Boolean
 ) {
     /**
      * @param ims the Gropius ims to use as input
@@ -37,7 +39,8 @@ data class IMSConfig(
         rootUrl = URI(helper.parseString(ims.templatedFields["root-url"])!!),
         imsTemplate = imsTemplate,
         defaultType = helper.parseString(ims.templatedFields["default-type"]),
-        defaultTemplate = helper.parseString(ims.templatedFields["default-template"])
+        defaultTemplate = helper.parseString(ims.templatedFields["default-template"]),
+        isCloud = helper.parseString(ims.templatedFields["jira-edition"])?.let { it == "CLOUD" } ?: true,
     )
 
     companion object {

jira/src/main/kotlin/gropius/sync/jira/model/ProjectQuery.kt

@@ -165,4 +165,13 @@ data class ProjectQuery(
 ) {
     fun issues(imsProject: IMSProject): List<IssueData> =
         issues.map { it.data(imsProject, names ?: JsonObject(mapOf()), schema ?: JsonObject(mapOf())) }
-}
+}
+
+/**
+ * Kotlin representation of the UserQuery JSON
+ * @param timeZone The timeZone of the user
+ */
+@Serializable
+data class UserQuery(
+    val timeZone: String
+) {}

sync/src/main/kotlin/gropius/sync/TokenManager.kt

@@ -205,7 +205,7 @@ abstract class TokenManager<ResponseType : BaseResponseType>(
                 }
             }
         }
-        TODO("Error Message")
+        TODO("Error Message for no working users")
     }
 
     /**