feat(indexing): schedule indexing based on cron trigger

Author: filipowm

ambassador-application/build.gradle.kts

@@ -12,6 +12,7 @@ plugins {
 val springdocVersion: String by extra
 val kotlinCoroutinesVersion: String by extra
 val springApiVersion: String by extra
+val shedlockVersion: String by extra
 
 dependencies {
     implementation(project(":ambassador-model"))
@@ -32,6 +33,9 @@ dependencies {
     implementation("com.github.ben-manes.caffeine:caffeine")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
 
+    implementation("net.javacrumbs.shedlock:shedlock-spring:$shedlockVersion")
+    implementation("net.javacrumbs.shedlock:shedlock-provider-jdbc-template:$shedlockVersion")
+
     implementation("io.github.filipowm:spring-api-starter:$springApiVersion")
 
     implementation("org.springdoc:springdoc-openapi-webflux-ui:$springdocVersion")

ambassador-application/src/main/kotlin/com/roche/ambassador/configuration/properties/IndexerProperties.kt

@@ -37,7 +37,11 @@ data class IndexerProperties(
 
     @NestedConfigurationProperty
     @Valid
-    val subscription: SubscriptionProperties = SubscriptionProperties()
+    val subscription: SubscriptionProperties = SubscriptionProperties(),
+
+    @NestedConfigurationProperty
+    @Valid
+    val scheduler: SchedulerProperties = SchedulerProperties(),
 
 )
 

ambassador-application/src/main/kotlin/com/roche/ambassador/configuration/properties/SchedulerProperties.kt

@@ -0,0 +1,9 @@
+package com.roche.ambassador.configuration.properties
+
+import java.time.Duration
+
+data class SchedulerProperties(
+    val enabled: Boolean = false,
+    val cron: String = "0 0 14 ? * SUN",
+    val lockFor: Duration = Duration.ofMinutes(30)
+)

ambassador-application/src/main/kotlin/com/roche/ambassador/indexing/IndexingConfiguration.kt

@@ -5,15 +5,23 @@ import com.roche.ambassador.configuration.properties.IndexingLockType
 import com.roche.ambassador.extensions.LoggerDelegate
 import com.roche.ambassador.extensions.toPrettyString
 import com.roche.ambassador.storage.indexing.IndexingRepository
+import net.javacrumbs.shedlock.core.LockProvider
+import net.javacrumbs.shedlock.provider.jdbctemplate.JdbcTemplateLockProvider
+import net.javacrumbs.shedlock.spring.annotation.EnableSchedulerLock
+import org.springframework.beans.factory.InitializingBean
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
+import org.springframework.jdbc.core.JdbcTemplate
+import org.springframework.scheduling.annotation.EnableScheduling
+import javax.sql.DataSource
 
 @Configuration
-internal class IndexingConfiguration(
-    private val indexerProperties: IndexerProperties
-) {
+internal class IndexingConfiguration(private val indexerProperties: IndexerProperties) {
 
-    private val log by LoggerDelegate()
+    companion object {
+        private val log by LoggerDelegate()
+    }
 
     @Bean
     fun indexingLock(indexingRepository: IndexingRepository): IndexingLock {
@@ -23,4 +31,31 @@ internal class IndexingConfiguration(
             IndexingLockType.DATABASE -> IndexingLock.createDatabaseLock(indexingRepository)
         }
     }
+
+    @Configuration
+    @ConditionalOnProperty(prefix = "ambassador.indexer.scheduler", name = ["enabled"], matchIfMissing = false, havingValue = "true")
+    @EnableScheduling
+    @EnableSchedulerLock(defaultLockAtMostFor = "10m")
+    inner class SchedulerConfiguration : InitializingBean {
+
+        override fun afterPropertiesSet() {
+            log.info("Initialized indexing scheduler with cron: ${indexerProperties.scheduler.cron}")
+        }
+
+        @Bean
+        fun lockProvider(dataSource: DataSource): LockProvider {
+            return JdbcTemplateLockProvider(
+                JdbcTemplateLockProvider.Configuration.builder()
+                    .withJdbcTemplate(JdbcTemplate(dataSource))
+                    .usingDbTime()
+                    .build()
+            )
+        }
+
+        @Bean
+        fun indexingScheduler(indexingService: IndexingService): ScheduledIndexingInitializer {
+           return ScheduledIndexingInitializer(indexingService)
+        }
+    }
+
 }

ambassador-application/src/main/kotlin/com/roche/ambassador/indexing/ScheduledIndexingInitializer.kt

@@ -0,0 +1,29 @@
+package com.roche.ambassador.indexing
+
+import com.roche.ambassador.extensions.LoggerDelegate
+import com.roche.ambassador.security.RunAsTechnicalUser
+import kotlinx.coroutines.runBlocking
+import net.javacrumbs.shedlock.spring.annotation.SchedulerLock
+import org.springframework.scheduling.annotation.Scheduled
+
+internal open class ScheduledIndexingInitializer(private val indexingService: IndexingService) {
+
+    companion object {
+        private val log by LoggerDelegate()
+    }
+
+    @Scheduled(cron = "\${ambassador.indexer.scheduler.cron}")
+    @SchedulerLock(name = " indexing", lockAtLeastFor = "\${ambassador.indexer.scheduler.lockFor}", lockAtMostFor = "1h")
+    @RunAsTechnicalUser
+    open fun triggerScheduling() {
+        log.info("Triggering scheduled indexing")
+        // purposely run blocking, cause it will get async in downstream when indexing is triggered successfully
+        runBlocking {
+            try {
+                indexingService.reindex()
+            } catch (ex: IndexingAlreadyStartedException) {
+                log.warn("Unable to start scheduled indexing because it is already running")
+            }
+        }
+    }
+}

ambassador-application/src/main/kotlin/com/roche/ambassador/security/RunAsTechnicalUser.kt

@@ -0,0 +1,5 @@
+package com.roche.ambassador.security
+
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION)
+@Retention(AnnotationRetention.RUNTIME)
+annotation class RunAsTechnicalUser

ambassador-application/src/main/kotlin/com/roche/ambassador/security/RunAsTechnicalUserAspect.kt

@@ -0,0 +1,44 @@
+package com.roche.ambassador.security
+
+import org.aspectj.lang.ProceedingJoinPoint
+import org.aspectj.lang.annotation.Around
+import org.aspectj.lang.annotation.Aspect
+import org.springframework.security.authentication.AbstractAuthenticationToken
+import org.springframework.security.core.context.ReactiveSecurityContextHolder
+import org.springframework.stereotype.Component
+
+@Aspect
+@Component
+internal open class RunAsTechnicalUserAspect {
+
+    companion object {
+        val technicalUser = AmbassadorUser(
+            "Technical User",
+            "_admin", "admin@admin.com",
+            mapOf(), listOf("ROLE_ADMIN", "ROLE_USER")
+        )
+        private val technicalUserToken = TechnicalUserToken(technicalUser)
+    }
+
+    private class TechnicalUserToken(private val ambassadorUser: AmbassadorUser) : AbstractAuthenticationToken(ambassadorUser.authorities) {
+        override fun getCredentials(): Any = "__none__"
+
+        override fun getPrincipal(): AmbassadorUser = ambassadorUser
+
+    }
+
+    @Around("@annotation(com.roche.ambassador.security.RunAsTechnicalUser)")
+    open fun logExecutionTime(joinPoint: ProceedingJoinPoint): Any? {
+        val savedContext = ReactiveSecurityContextHolder.getContext()
+        ReactiveSecurityContextHolder.withAuthentication(technicalUserToken)
+        try {
+            return joinPoint.proceed()
+        } finally {
+            if (savedContext != null) {
+                ReactiveSecurityContextHolder.withSecurityContext(savedContext)
+            } else {
+                ReactiveSecurityContextHolder.clearContext()
+            }
+        }
+    }
+}

ambassador-application/src/main/resources/application.yml

@@ -77,6 +77,10 @@ ambassador:
   security:
     enabled: true
   indexer:
+    scheduler:
+      enabled: true
+      cron: "0 0 14 ? * SUN"
+      lockFor: 30m
     features:
       requireVisibility:
         - public

ambassador-gitlab-client/src/main/kotlin/com/roche/gitlab/api/project/model/FeatureAccessLevel.kt

@@ -2,12 +2,13 @@ package com.roche.gitlab.api.project.model
 
 enum class FeatureAccessLevel {
 
+    PUBLIC,
     DISABLED,
     PRIVATE,
     ENABLED
     ;
 
-    fun canEveryoneAccess(): Boolean = this == ENABLED
+    fun canEveryoneAccess(): Boolean = this == ENABLED || this == PUBLIC
     fun canNooneAccess(): Boolean = this == DISABLED
     fun canOnlyProjectMembersAccess(): Boolean = this == PRIVATE
 }

ambassador-storage/src/main/resources/db/migration/V202204181912__add_shedlock_tables.sql

@@ -0,0 +1,7 @@
+CREATE TABLE shedlock(
+    name VARCHAR(64) NOT NULL,
+    lock_until TIMESTAMP NOT NULL,
+    locked_at TIMESTAMP NOT NULL,
+    locked_by VARCHAR(255) NOT NULL,
+    PRIMARY KEY (name)
+);