Added endpoints to manage foundry instances

gradle/libs.versions.toml

@@ -2,7 +2,7 @@
 ktor = "3.1.2"
 logback = "1.4.14"
 kfs = "1.3.0"
-koin = "4.0.4"
+koin = "4.1.0-Beta7"
 caffeine = "3.1.0"
 tike = "2.9.1"
 jbcrypt = "0.4"
@@ -22,21 +22,34 @@ ktor-server-netty = { module = "io.ktor:ktor-server-netty-jvm", version.ref = "k
 ktor-server-auth = { module = "io.ktor:ktor-server-auth", version.ref = "ktor" }
 ktor-server-auth-jwt = { module = "io.ktor:ktor-server-auth-jwt", version.ref = "ktor" }
 ktor-server-status-pages = { module = "io.ktor:ktor-server-status-pages", version.ref = "ktor" }
+ktor-server-sse = { module = "io.ktor:ktor-server-sse", version.ref = "ktor" }
+ktor-server-rate-limit = { module = "io.ktor:ktor-server-rate-limit", version.ref = "ktor"}
 ktor-serialization-kotlinx = { module = "io.ktor:ktor-serialization-kotlinx-json", version.ref = "ktor" }
 ktor-serialization-jackson = { module = "io.ktor:ktor-serialization-jackson", version.ref = "ktor" }
 ktor-client-cio = { module = "io.ktor:ktor-client-cio-jvm", version.ref = "ktor" }
 ktor-client-core = { module = "io.ktor:ktor-client-core-jvm", version.ref = "ktor" }
 ktor-client-content-negotiation = { module = "io.ktor:ktor-client-content-negotiation-jvm", version.ref = "ktor" }
 kfswatch = { module = "io.github.irgaly.kfswatch:kfswatch", version.ref = "kfs" }
-koin-ktor = { module = "io.insert-koin:koin-ktor", version.ref = "koin" }
+koin-ktor = { module = "io.insert-koin:koin-ktor3", version.ref = "koin" }
 koin-logger = { module = "io.insert-koin:koin-logger-slf4j", version.ref = "koin" }
 logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
 jbcrypt = { module = "org.mindrot:jbcrypt", version.ref = "jbcrypt" }
 krontab = { module = "dev.inmo:krontab", version.ref = "krontab" }
 jupyter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "jupyter" }
 
 [bundles]
-ktor-server = ["ktor-server-core", "ktor-server-cors", "ktor-server-content-negotiations", "ktor-server-logging", "ktor-server-netty", "ktor-server-auth", "ktor-server-auth-jwt", "ktor-server-status-pages"]
+ktor-server = [
+    "ktor-server-core",
+    "ktor-server-cors",
+    "ktor-server-content-negotiations",
+    "ktor-server-logging",
+    "ktor-server-netty",
+    "ktor-server-auth",
+    "ktor-server-auth-jwt",
+    "ktor-server-status-pages",
+    "ktor-server-sse",
+    "ktor-server-rate-limit"
+]
 ktor-serialization = ["ktor-serialization-kotlinx", "ktor-serialization-jackson"]
 ktor-client = ["ktor-client-cio", "ktor-client-core", "ktor-client-content-negotiation"]
 koin = ["koin-ktor", "koin-logger"]

src/main/kotlin/org/wagham/kabotapi/KabotApiApplication.kt

@@ -5,6 +5,7 @@ import org.wagham.kabotapi.configuration.configureExceptions
 import org.wagham.kabotapi.configuration.configureHTTP
 import org.wagham.kabotapi.configuration.configureKoin
 import org.wagham.kabotapi.configuration.configureRouting
+import org.wagham.kabotapi.configuration.configureThrottling
 
 fun main(args: Array<String>) {
 	io.ktor.server.netty.EngineMain.main(args)
@@ -15,5 +16,6 @@ fun Application.module() {
 	configureHTTP()
 	configureKoin()
 	configureExceptions()
+	configureThrottling()
 	configureRouting()
 }

src/main/kotlin/org/wagham/kabotapi/components/InstanceConfigManager.kt

@@ -1,5 +1,6 @@
 package org.wagham.kabotapi.components
 
+import dev.inmo.krontab.doInfinity
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import io.github.irgaly.kfswatch.KfsDirectoryWatcher
@@ -9,6 +10,7 @@ import kotlinx.coroutines.launch
 import kotlinx.serialization.json.Json
 import org.wagham.kabotapi.entities.foundry.FoundryOptions
 import java.io.File
+import java.nio.file.Files
 
 class InstanceConfigManager(
 	private val baseFolder: String
@@ -22,6 +24,8 @@ class InstanceConfigManager(
 	private val scope = CoroutineScope(Dispatchers.IO)
 	private val watcher = KfsDirectoryWatcher(scope)
 	private val instancesByUrl = mutableMapOf<String, InstanceInfo>()
+	private val urlById = mutableMapOf<String, String>()
+	private val sizeByInstanceId = mutableMapOf<String, Long>()
 
 	fun startWatching() {
 		logger.info("Starting instance manager")
@@ -51,17 +55,48 @@ class InstanceConfigManager(
 				}
 			}
 		}
+		startWatchingInstanceSize()
 	}
 
-	fun getInfoByUrl(url: String): InstanceInfo? = instancesByUrl[url]
+	fun getConfigByUrl(url: String): InstanceInfo? = instancesByUrl[url]
+
+	fun getConfigById(id: String): InstanceInfo? = urlById[id]?.let {
+		getConfigByUrl(it)
+	}
+
+	fun getFolderSizeById(id: String): Long? = sizeByInstanceId[id]
+
+	private fun getFolderSize(folder: File): Long = folder.walkTopDown().filter {
+		it.isFile && !Files.isSymbolicLink(it.toPath())
+	}.sumOf { it.length() }
+
+	private fun computeInstancesSize() {
+		File(baseFolder).listFiles().filter {
+			it.isDirectory
+		}.onEach {
+			val size = getFolderSize(it)
+			sizeByInstanceId[it.name] = size
+		}
+	}
+
+	private fun startWatchingInstanceSize() {
+		computeInstancesSize()
+		scope.launch {
+			doInfinity("0 0 * * * *") {
+				computeInstancesSize()
+			}
+		}
+	}
 
 	private fun updateInstancesWith(optionsFile: File) {
 		val options = Json.decodeFromString<FoundryOptions>(optionsFile.readText())
 		val info = InstanceInfo(
 			id = options.dataPath.split("/").last(),
 			url = options.routePrefix,
 			name = options.masterName ?: "unknown",
+			domain = options.domain
 		)
+		urlById[info.id] = info.url
 		instancesByUrl[info.url] = info.also {
 			logger.info("Updating ${info.url} with $it")
 		}
@@ -70,7 +105,8 @@ class InstanceConfigManager(
 	data class InstanceInfo(
 		val id: String,
 		val url: String,
-		val name: String
+		val name: String,
+		val domain: String?
 	)
 
 }

src/main/kotlin/org/wagham/kabotapi/components/InstanceInactivityManager.kt

@@ -27,7 +27,7 @@ class InstanceInactivityManager(
 	private val enableLogging: Boolean,
 ) {
 
-	private val urlExtractingRegex = Regex(".* \"https://fnd\\.kaironbot\\.net/([^/]+).*")
+	private val urlExtractingRegex = Regex(".* \"https://fnd\\.[^/]+/([^/]+).*")
 	private val managerScope = CoroutineScope(Dispatchers.Default + SupervisorJob())
 	private val logger = KtorSimpleLogger(this.javaClass.simpleName)
 	private val instanceActivity = Caffeine.newBuilder()
@@ -54,12 +54,18 @@ class InstanceInactivityManager(
 		doInfinity("0 0 * * * *") {
 			try {
 				commandComponent.sendSocketCommand(Pm2ListCommand()).filter {
-					it.name !in excludedInstances
+					it.isActive && it.name !in excludedInstances
 				}.forEach {
 					if ((System.currentTimeMillis() - it.pm2Env.uptime).milliseconds > 1.hours) {
 						val lastActivity = instanceActivity.getIfPresent(it.name)
 						if (lastActivity == null) {
-							commandComponent.sendSocketCommand(Pm2StopCommand(it.name))
+							var retries = 5
+							do {
+								retries = runCatching {
+									commandComponent.sendSocketCommand(Pm2StopCommand(it.name))
+									0
+								}.getOrDefault(retries - 1)
+							} while(retries > 0)
 						}
 					}
 				}
@@ -89,11 +95,11 @@ class InstanceInactivityManager(
 		}
 	}
 
-
+	fun getLastActivityOf(instanceId: String): Long? = instanceActivity.getIfPresent(instanceId)
 
 	private fun getInstanceFromLog(log: String): InstanceConfigManager.InstanceInfo? =
 		urlExtractingRegex.find(log)?.groupValues?.get(1)?.let {
-			instanceConfigManager.getInfoByUrl(it)
+			instanceConfigManager.getConfigByUrl(it)
 		}?.takeIf {
 			it.id !in excludedInstances
 		}

src/main/kotlin/org/wagham/kabotapi/components/JWTManager.kt

@@ -5,7 +5,6 @@ import com.auth0.jwt.JWTVerifier
 import com.auth0.jwt.algorithms.Algorithm
 import com.auth0.jwt.interfaces.Payload
 import io.ktor.server.auth.jwt.*
-import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import org.wagham.kabotapi.entities.config.JWTConfig
 import org.wagham.kabotapi.entities.security.JWTClaims
@@ -52,14 +51,16 @@ class JWTManager(
         .sign(Algorithm.HMAC256(config.authSecret))
 
     /**
-     * @return a [JWTVerifier] for the authentication jwt.
+     * a [JWTVerifier] for the authentication jwt.
      */
-    fun authJWTVerifier(): JWTVerifier = JWT
+    val authJWTVerifier: JWTVerifier = JWT
         .require(Algorithm.HMAC256(config.authSecret))
         .withAudience(config.audience)
         .withIssuer(config.issuer)
         .build()
 
+    fun decodeAndGetClaims(token: String): JWTClaims = authJWTVerifier.verify(token).toJWTClaims()
+
     private fun Payload.isAuthJwtValid() =
         getClaim(USER_ID).asString().isNotBlank()
             && getClaim(GUILD_ID).asString().isNotBlank()

src/main/kotlin/org/wagham/kabotapi/components/socket/AbstractUdpListener.kt

@@ -8,7 +8,7 @@ import io.ktor.util.logging.*
 abstract class AbstractUdpListener(
 	listenPort: Int,
 	protected val logger: Logger,
-	private val enableLogging: Boolean
+	protected val enableLogging: Boolean
 ) {
 
 	private val receiveSocket = DatagramSocket(listenPort)
@@ -25,19 +25,24 @@ abstract class AbstractUdpListener(
 					try {
 						val packet = DatagramPacket(rcvBuffer, rcvBuffer.size)
 						socket.receive(packet)
-						val idx = packet.data.indexOf('\n'.code.toByte())
-						val received = if (idx == -1) null else buffer + String(packet.data, 0, idx)
-						buffer =
-							if (idx == -1) buffer + String(packet.data, 0, packet.length)
-							else String(packet.data, idx + 1, packet.length - idx - 1)
-						if (enableLogging) {
-							logger.info("Buffer: $buffer")
-						}
-						if (received != null) {
-							if (enableLogging) {
-								logger.info("Received: $received")
+						var data = packet.data.sliceArray(0 until packet.length)
+						while (data.isNotEmpty()) {
+							val idx = data.indexOf('\n'.code.toByte())
+
+							if (idx == -1) {
+								buffer += String(data, 0, data.size)
+								break
+							} else {
+								val line = buffer + String(data, 0, idx)
+								data = data.sliceArray(idx + 1 until data.size)
+								buffer = ""
+								if (line.isNotEmpty()) {
+									if (enableLogging) {
+										logger.info("Received: $line")
+									}
+									handlePacket(line)
+								}
 							}
-							handlePacket(received)
 						}
 					} catch (e: Exception) {
 						logger.error("Cannot receive packet", e)

src/main/kotlin/org/wagham/kabotapi/components/socket/CommandComponent.kt

@@ -1,6 +1,7 @@
 package org.wagham.kabotapi.components.socket
 
 import io.ktor.util.logging.*
+import kotlinx.coroutines.TimeoutCancellationException
 import kotlinx.coroutines.channels.BufferOverflow
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.launch
@@ -11,7 +12,7 @@ import org.wagham.kabotapi.data.PeekableChannel
 import java.net.DatagramPacket
 import java.net.DatagramSocket
 import java.net.InetAddress
-import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
 
 class CommandComponent(
 	private val sendPort: Int,
@@ -24,7 +25,7 @@ class CommandComponent(
 	private val socketMutex = Mutex()
 
 	override fun handlePacket(packet: String) {
-		val parts = packet.split('|')
+		val parts = packet.trim('\n').split('|', limit = 4)
 		packetChannel.trySend(
 			ParsedPacket(
 				parts[0].toLong(),
@@ -48,7 +49,7 @@ class CommandComponent(
 			val responseJob = launch {
 				do {
 					val hasNext = runCatching {
-						withTimeout(500.milliseconds) {
+						withTimeout(1.seconds) {
 							val next = packetChannel.peek()
 							when {
 								next.ts < command.ts -> {
@@ -63,6 +64,14 @@ class CommandComponent(
 								}
 							}
 						}
+					}.onFailure {
+						if (enableLogging) {
+							if(it is TimeoutCancellationException) {
+								logger.error("Timed out waiting for command $command")
+							} else {
+								logger.error("Error while waiting for command $command", it)
+							}
+						}
 					}.getOrDefault(false)
 				} while (hasNext)
 			}

src/main/kotlin/org/wagham/kabotapi/configuration/HttpConfig.kt

@@ -34,7 +34,7 @@ fun Application.configureHTTP() {
     install(Authentication) {
         jwt(AUTH_CTX) {
             realm = jwtManager.config.realm
-            verifier(jwtManager.authJWTVerifier())
+            verifier(jwtManager.authJWTVerifier)
 
             validate { credential ->
                 jwtManager.authCredentialToPrincipal(credential)

src/main/kotlin/org/wagham/kabotapi/configuration/KoinConfig.kt

@@ -20,13 +20,15 @@ import org.wagham.kabotapi.entities.config.MongoConfig
 import org.wagham.kabotapi.entities.config.SocketConfig
 import org.wagham.kabotapi.logic.CharacterLogic
 import org.wagham.kabotapi.logic.DiscordLogic
+import org.wagham.kabotapi.logic.FoundryLogic
 import org.wagham.kabotapi.logic.ItemLogic
 import org.wagham.kabotapi.logic.LabelLogic
 import org.wagham.kabotapi.logic.PlayerLogic
 import org.wagham.kabotapi.logic.SessionLogic
 import org.wagham.kabotapi.logic.UtilitiesLogic
 import org.wagham.kabotapi.logic.impl.CharacterLogicImpl
 import org.wagham.kabotapi.logic.impl.DiscordLogicImpl
+import org.wagham.kabotapi.logic.impl.FoundryLogicImpl
 import org.wagham.kabotapi.logic.impl.ItemLogicImpl
 import org.wagham.kabotapi.logic.impl.LabelLogicImpl
 import org.wagham.kabotapi.logic.impl.PlayerLogicImpl
@@ -64,6 +66,15 @@ fun applicationModules(
 		)
 	}
 
+	single<FoundryLogic> {
+		FoundryLogicImpl(
+			defaultDomain = foundryConfig.defaultDomain,
+			instanceConfigManager = get(),
+			instanceInactivityManager = get(),
+			commandComponent = get(),
+			excludedInstances = foundryConfig.excludedInstances
+		)
+	}
 	single<DiscordLogic> { DiscordLogicImpl(get(), discordConfig)}
 	single<CharacterLogic> { CharacterLogicImpl(get(), get()) }
 	single<LabelLogic> { LabelLogicImpl(get()) }

src/main/kotlin/org/wagham/kabotapi/configuration/RouteConfig.kt

@@ -4,6 +4,7 @@ import io.ktor.serialization.kotlinx.json.*
 import io.ktor.server.application.*
 import io.ktor.server.plugins.contentnegotiation.*
 import io.ktor.server.routing.*
+import io.ktor.server.sse.SSE
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.modules.SerializersModule
 import org.wagham.kabotapi.controllers.*
@@ -18,9 +19,12 @@ fun Application.configureRouting() {
 			}
 		})
 	}
+	install(SSE)
+
 	routing {
 		authController()
 		characterController()
+		foundryController()
 		guildController()
 		itemController()
 		labelController()