diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 261173ae4..9208accae 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -137,12 +137,14 @@ To avoid Sonar/ESLint code smells in `dashboard/`: ## Key Conventions -### Warnings and lint fixes (no suppressions) +### Warnings and lint fixes (suppress rarely) **CRITICAL:** When asked to fix a compiler warning, linter finding, or static analysis issue (Detekt, ESLint, Sonar, Kotlin compiler, etc.): -- ❌ **NEVER suppress** the finding — do not use `@Suppress`, `eslint-disable`, `// NOSONAR`, `SuppressWarnings`, file- or line-level ignores, or similar to silence the tool without fixing the underlying problem. +- ❌ **Do not suppress to defer work** — do not use `@Suppress`, `eslint-disable`, `// NOSONAR`, `SuppressWarnings`, file- or line-level ignores, or similar because fixing the finding properly is too large or inconvenient right now. That is not a valid reason to silence the tool. - ✅ **Fix the root cause** — refactor, correct types, satisfy the rule, or extract helpers so the violation is resolved properly. +- ✅ **Suppress only when it is appropriate forever** — e.g. a documented false positive, or a case where the rule genuinely does not apply on that line and a short comment explains why. **New feature code** should not ship with suppressions that are really “TODO fix later”; land the feature clean or fix the underlying issue before merge. +- ✅ **When the rule genuinely does not fit the file** — e.g. a demo data seeder may trigger hundreds of “magic number” findings: those literals *are* the seed data, not something that should become hundreds of named constants in one file. Prefer **configuration** (exclude that file or path in `detekt.yml`, ESLint `overrides`, etc.) so the tool skips that context, rather than line-level suppressions or a wall of constants that add no clarity. If a rule is wrong for the whole project, **adjust the tool configuration** (e.g. `detekt.yml`, ESLint config) deliberately — do not mask individual violations with suppressions. diff --git a/backend/detekt.yml b/backend/detekt.yml index cd496cad4..da07212d3 100644 --- a/backend/detekt.yml +++ b/backend/detekt.yml @@ -11,7 +11,14 @@ comments: # These can be enabled incrementally as the codebase is cleaned up. style: MagicNumber: - active: false + active: true + excludes: + - '**/test/**' + - '**/integrationTest/**' + - '**/DemoDataReseeder.kt' + ignorePropertyDeclaration: true + ignoreEnums: true + ignoreRanges: true WildcardImport: active: false ReturnCount: diff --git a/backend/src/main/kotlin/com/moneat/Application.kt b/backend/src/main/kotlin/com/moneat/Application.kt index 7f0cb5443..3bcf9b2fa 100644 --- a/backend/src/main/kotlin/com/moneat/Application.kt +++ b/backend/src/main/kotlin/com/moneat/Application.kt @@ -39,6 +39,8 @@ import io.ktor.server.netty.EngineMain import org.koin.ktor.plugin.Koin import org.koin.logger.slf4jLogger +private const val STACK_TRACE_DEPTH = 20 + fun main(args: Array) { // Add JVM shutdown hook to log when shutdown is triggered Runtime.getRuntime().addShutdownHook( @@ -70,7 +72,7 @@ fun Application.module() { // Log stack trace when shutdown is triggered to identify the cause monitor.subscribe(ApplicationStopping) { log.warn("APPLICATION STOPPING - Stack trace to identify trigger:") - Thread.currentThread().stackTrace.take(20).forEach { frame -> + Thread.currentThread().stackTrace.take(STACK_TRACE_DEPTH).forEach { frame -> log.warn(" at $frame") } } diff --git a/backend/src/main/kotlin/com/moneat/ai/AiChatService.kt b/backend/src/main/kotlin/com/moneat/ai/AiChatService.kt index 654d16174..8e094dd05 100644 --- a/backend/src/main/kotlin/com/moneat/ai/AiChatService.kt +++ b/backend/src/main/kotlin/com/moneat/ai/AiChatService.kt @@ -28,18 +28,22 @@ private val aiChatResponseJson = Json { ignoreUnknownKeys = true } */ class AiChatService { + companion object { + private const val MAX_USER_INPUT_LENGTH = 4000 + private const val MAX_HISTORY_MESSAGES = 20 + } + suspend fun chat(userId: Int, orgId: Int, request: ChatRequest): ChatApiResponse { return unavailable( operation = "chat", userId = userId, orgId = orgId, - context = "message=${request.message.take(50)}", + context = "messageLength=${request.message.length}", ) } /** Shared helpers for AI message shaping; `internal` for unit tests and enterprise module reuse. */ internal fun sanitizeUserInput(input: String): String { - val maxLength = 4000 val injectionPatterns = listOf( Regex("ignore previous instructions", RegexOption.IGNORE_CASE), Regex("system:", RegexOption.IGNORE_CASE), @@ -49,7 +53,7 @@ class AiChatService { for (pattern in injectionPatterns) { sanitized = pattern.replace(sanitized, "") } - return sanitized.take(maxLength) + return sanitized.take(MAX_USER_INPUT_LENGTH) } internal fun buildOpenAiMessages( @@ -59,7 +63,7 @@ class AiChatService { ): List { val systemContent = "$systemPrompt\n\n== API DOCUMENTATION ==\n$docBlock" val systemMessage = OpenAiMessage(role = "system", content = systemContent) - val recentHistory = history.takeLast(20) + val recentHistory = history.takeLast(MAX_HISTORY_MESSAGES) return listOf(systemMessage) + recentHistory } diff --git a/backend/src/main/kotlin/com/moneat/ai/OpenAiClient.kt b/backend/src/main/kotlin/com/moneat/ai/OpenAiClient.kt index 92f5aeb2b..4517cbf31 100644 --- a/backend/src/main/kotlin/com/moneat/ai/OpenAiClient.kt +++ b/backend/src/main/kotlin/com/moneat/ai/OpenAiClient.kt @@ -43,6 +43,11 @@ sealed class OpenAiError(override val message: String) : Exception(message) { } object OpenAiClient { + private const val HTTP_UNAUTHORIZED = 401 + private const val HTTP_TOO_MANY_REQUESTS = 429 + private const val HTTP_CLIENT_ERROR_MIN = 400 + private const val HTTP_CLIENT_ERROR_MAX = 499 + private val json = Json { ignoreUnknownKeys = true } private val client = @@ -92,9 +97,10 @@ object OpenAiClient { if (response.status != HttpStatusCode.OK) { logger.error { "OpenAI API error (${response.status}): $body" } throw when (response.status.value) { - 401 -> OpenAiError.AuthenticationError("Invalid or expired API key") - 429 -> OpenAiError.RateLimitError("Rate limit exceeded — please try again later") - in 400..499 -> OpenAiError.ModelError("OpenAI request error (${response.status.value}): $body") + HTTP_UNAUTHORIZED -> OpenAiError.AuthenticationError("Invalid or expired API key") + HTTP_TOO_MANY_REQUESTS -> OpenAiError.RateLimitError("Rate limit exceeded — please try again later") + in HTTP_CLIENT_ERROR_MIN..HTTP_CLIENT_ERROR_MAX -> + OpenAiError.ModelError("OpenAI request error (${response.status.value}): $body") else -> OpenAiError.ServerError("OpenAI server error (${response.status.value})") } } diff --git a/backend/src/main/kotlin/com/moneat/analytics/routes/AnalyticsRoutes.kt b/backend/src/main/kotlin/com/moneat/analytics/routes/AnalyticsRoutes.kt index faa8df557..8b5162ef4 100644 --- a/backend/src/main/kotlin/com/moneat/analytics/routes/AnalyticsRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/analytics/routes/AnalyticsRoutes.kt @@ -35,6 +35,12 @@ import java.time.format.DateTimeParseException private val logger = KotlinLogging.logger {} +private const val PERIOD_7D_OFFSET_DAYS = 6L +private const val PERIOD_30D_OFFSET_DAYS = 29L +private const val PERIOD_6MO_MONTHS = 6L +private const val PERIOD_12MO_MONTHS = 12L +private const val FILTER_PARTS_COUNT = 3 + /** * Authenticated dashboard API routes for product analytics. * All endpoints are under /v1/analytics/{projectId}/... @@ -253,11 +259,11 @@ private fun parseDateRange(call: io.ktor.server.application.ApplicationCall): Pa return when (period) { "today" -> now to now - "7d" -> now.minusDays(6) to now - "30d" -> now.minusDays(29) to now + "7d" -> now.minusDays(PERIOD_7D_OFFSET_DAYS) to now + "30d" -> now.minusDays(PERIOD_30D_OFFSET_DAYS) to now "month" -> now.withDayOfMonth(1) to now - "6mo" -> now.minusMonths(6) to now - "12mo" -> now.minusMonths(12) to now + "6mo" -> now.minusMonths(PERIOD_6MO_MONTHS) to now + "12mo" -> now.minusMonths(PERIOD_12MO_MONTHS) to now "custom" -> { val from = call.request.queryParameters["from"] ?: call.request.queryParameters["date_from"] val to = call.request.queryParameters["to"] ?: call.request.queryParameters["date_to"] @@ -268,7 +274,7 @@ private fun parseDateRange(call: io.ktor.server.application.ApplicationCall): Pa null } } - else -> now.minusDays(29) to now + else -> now.minusDays(PERIOD_30D_OFFSET_DAYS) to now } } @@ -300,7 +306,7 @@ private fun parseFilters(call: io.ktor.server.application.ApplicationCall): List ?: emptyList() return rawFilters.mapNotNull { filterStr -> val parts = filterStr.split(":", limit = 3) - if (parts.size == 3) { + if (parts.size == FILTER_PARTS_COUNT) { AnalyticsFilter(parts[0], parts[1], parts[2]) } else { null diff --git a/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsIngestionWorker.kt b/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsIngestionWorker.kt index c0ef722cf..bc77d982e 100644 --- a/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsIngestionWorker.kt +++ b/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsIngestionWorker.kt @@ -157,7 +157,7 @@ class AnalyticsIngestionWorker( val response = ClickHouseClient.execute(sql) val body = response.bodyAsText() if (response.isClickHouseError(body)) { - throw IOException("ClickHouse insert failed: ${body.take(500)}") + throw IOException("ClickHouse insert failed: ${body.take(ERROR_BODY_MAX_LENGTH)}") } } @@ -168,6 +168,7 @@ class AnalyticsIngestionWorker( private const val DEFAULT_WORKER_COUNT = 2 private const val BRPOP_TIMEOUT = 5L private const val ERROR_BACKOFF_MS = 1000L + private const val ERROR_BODY_MAX_LENGTH = 500 fun escapeCH(s: String): String = ClickHouseSqlUtils.escapeSql(s) } diff --git a/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsService.kt b/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsService.kt index c0d5acf4b..33ba49249 100644 --- a/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsService.kt +++ b/backend/src/main/kotlin/com/moneat/analytics/services/AnalyticsService.kt @@ -39,6 +39,10 @@ import java.time.format.DateTimeFormatter private val logger = KotlinLogging.logger {} private val jsonParser = Json { ignoreUnknownKeys = true } +private const val ERROR_TRUNCATE_LENGTH = 500 +private const val PERCENTAGE_MULTIPLIER = 100 +private const val DEFAULT_LIMIT = 100 + /** * Query builder for analytics dashboard endpoints. * Builds ClickHouse SQL queries with filters, date ranges, and comparison periods. @@ -159,7 +163,7 @@ class AnalyticsService { dateTo: LocalDate, filters: List, dimension: String, - limit: Int = 100, + limit: Int = DEFAULT_LIMIT, ): BreakdownResponse { val (column, table, alias) = resolveDimension(dimension) val timeColumn = if (alias == "s") "hour" else "timestamp" @@ -195,7 +199,7 @@ class AnalyticsService { dateFrom: LocalDate, dateTo: LocalDate, filters: List, - limit: Int = 100, + limit: Int = DEFAULT_LIMIT, ): BreakdownResponse = getBreakdown(projectId, dateFrom, dateTo, filters, "pathname", limit) suspend fun getEntryPages( @@ -203,7 +207,7 @@ class AnalyticsService { dateFrom: LocalDate, dateTo: LocalDate, filters: List, - limit: Int = 100, + limit: Int = DEFAULT_LIMIT, ): BreakdownResponse = getBreakdown(projectId, dateFrom, dateTo, filters, "entry_page", limit) suspend fun getExitPages( @@ -211,7 +215,7 @@ class AnalyticsService { dateFrom: LocalDate, dateTo: LocalDate, filters: List, - limit: Int = 100, + limit: Int = DEFAULT_LIMIT, ): BreakdownResponse = getBreakdown(projectId, dateFrom, dateTo, filters, "exit_page", limit) // --- Realtime --- @@ -221,7 +225,7 @@ class AnalyticsService { val count = suspendRunCatching { RedisConfig.sync().pfcount(key) }.getOrElse { e -> - val errorMsg = e.toString().take(500) + val errorMsg = e.toString().take(ERROR_TRUNCATE_LENGTH) logger.debug { "Failed to read realtime counter: $errorMsg" } 0L } @@ -284,7 +288,7 @@ class AnalyticsService { step } else { val prev = funnelSteps[i - 1].visitors - val dropoff = if (prev > 0) ((prev - step.visitors).toDouble() / prev * 100) else 0.0 + val dropoff = if (prev > 0) ((prev - step.visitors).toDouble() / prev * PERCENTAGE_MULTIPLIER) else 0.0 step.copy(dropoff = dropoff) } } @@ -298,7 +302,7 @@ class AnalyticsService { dateFrom: LocalDate, dateTo: LocalDate, filters: List, - limit: Int = 100, + limit: Int = DEFAULT_LIMIT, ): BreakdownResponse { val where = buildWhere(projectId, dateFrom, dateTo, filters, "e", "timestamp") val sql = """ diff --git a/backend/src/main/kotlin/com/moneat/auth/routes/AuthRoutes.kt b/backend/src/main/kotlin/com/moneat/auth/routes/AuthRoutes.kt index 1ed5e7e7c..4167581f9 100644 --- a/backend/src/main/kotlin/com/moneat/auth/routes/AuthRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/auth/routes/AuthRoutes.kt @@ -60,6 +60,7 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} private const val GITHUB_OAUTH_CALLBACK_ERROR = "GitHub OAuth callback error" private const val APPLE_OAUTH_CALLBACK_ERROR = "Apple OAuth callback error" +private const val USER_AGENT_MAX_LENGTH = 2048 fun Route.authRoutes( authService: AuthService = GlobalContext.get().get(), @@ -112,7 +113,7 @@ private suspend fun io.ktor.server.routing.RoutingContext.handleSignup(authServi call.request.headers["User-Agent"] ?.trim() ?.takeIf { it.isNotBlank() } - ?.take(2048) + ?.take(USER_AGENT_MAX_LENGTH) val context = SignupRequestContext( ipAddress = cfConnectingIp ?: forwardedFor ?: remoteHost, diff --git a/backend/src/main/kotlin/com/moneat/auth/services/AuthService.kt b/backend/src/main/kotlin/com/moneat/auth/services/AuthService.kt index 95cf899a6..5402b1b49 100644 --- a/backend/src/main/kotlin/com/moneat/auth/services/AuthService.kt +++ b/backend/src/main/kotlin/com/moneat/auth/services/AuthService.kt @@ -35,6 +35,7 @@ import com.moneat.shared.repositories.OrganizationRepository import com.moneat.shared.services.SidebarPreferenceService import com.moneat.utils.SentryUtils import io.ktor.server.config.ApplicationConfig +import org.jetbrains.exposed.v1.core.ResultRow import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.greater @@ -79,6 +80,12 @@ class AuthService( companion object { private const val VERIFICATION_TTL_MS = 24 * 60 * 60 * 1000L private const val PASSWORD_RESET_TTL_MS = 60 * 60 * 1000L + private const val MIN_PASSWORD_LENGTH = 8 + private const val ORG_SLUG_RANDOM_SUFFIX_LENGTH = 8 + private const val JWT_ACCESS_TOKEN_EXPIRY_MS = 3_600_000L + private const val DEMO_TOKEN_EXPIRY_MS = 86_400_000L + private const val TOKEN_ENTROPY_BYTES = 32 + private const val MAX_ORG_SLUG_LENGTH = 100 } private val config = ApplicationConfig("application.conf") @@ -97,9 +104,7 @@ class AuthService( if (request.email.isBlank()) { throw IllegalArgumentException("Email is required") } - if (request.password.length < 8) { - throw IllegalArgumentException("Password must be at least 8 characters") - } + require(request.password.length >= MIN_PASSWORD_LENGTH) { "Password must be at least 8 characters" } validateSignupLegalConsent(request) val normalizedEmail = request.email.lowercase().trim() @@ -127,52 +132,7 @@ class AuthService( val now = System.currentTimeMillis() // Resolve invite in a strict state-aware way. - val pendingInvite = - if (inviteToken != null) { - val inviteByToken = - OrgInvitations - .selectAll() - .where { OrgInvitations.token eq inviteToken } - .singleOrNull() - ?: throw IllegalArgumentException("Invitation not found") - - val inviteStatus = inviteByToken[OrgInvitations.status] - if (inviteStatus != "pending") { - throw IllegalArgumentException("Invitation is no longer valid") - } - - val inviteExpiresAt = inviteByToken[OrgInvitations.expires_at] - if (now > inviteExpiresAt) { - OrgInvitations.update({ OrgInvitations.id eq inviteByToken[OrgInvitations.id] }) { - it[OrgInvitations.status] = "expired" - } - throw IllegalArgumentException("Invitation has expired") - } - - if (inviteByToken[OrgInvitations.email].lowercase() != normalizedEmail) { - throw IllegalArgumentException("This invitation was sent to a different email address") - } - - inviteByToken - } else { - OrgInvitations.update({ - (OrgInvitations.email eq normalizedEmail) and - (OrgInvitations.status eq "pending") and - (OrgInvitations.expires_at lessEq now) - }) { - it[OrgInvitations.status] = "expired" - } - - OrgInvitations - .selectAll() - .where { - (OrgInvitations.email eq normalizedEmail) and - (OrgInvitations.status eq "pending") and - (OrgInvitations.expires_at greater now) - }.orderBy(OrgInvitations.created_at, org.jetbrains.exposed.v1.core.SortOrder.DESC) - .limit(1) - .singleOrNull() - } + val pendingInvite = resolvePendingInvite(inviteToken, normalizedEmail, now) // Detect first real user (excludes demo user which has id < 0) val isFirstUser = Users.selectAll().where { Users.id greater 0 }.count() == 0L @@ -202,62 +162,8 @@ class AuthService( it[email_verification_expires_at] = expiresAt }[Users.id] - val finalOrgId: Int - val finalOrgRole: String - // Join existing org if invited, otherwise create new org - if (pendingInvite != null) { - finalOrgId = pendingInvite[OrgInvitations.organization_id] - finalOrgRole = pendingInvite[OrgInvitations.role] - - // Create membership - Memberships.insert { - it[user_id] = id - it[organization_id] = finalOrgId - it[role] = finalOrgRole - } - - // Mark invitation as accepted - OrgInvitations.update( - { OrgInvitations.id eq pendingInvite[OrgInvitations.id] } - ) { - it[status] = "accepted" - } - - SentryUtils.breadcrumb( - "auth", - "User joined via invitation", - mapOf( - "user_id" to id, - "organization_id" to finalOrgId, - "role" to finalOrgRole - ) - ) - } else { - // Create default organization - finalOrgId = - Organizations.insert { - it[name] = "${request.name ?: request.email}'s Organization" - it[slug] = "org-${UUID.randomUUID().toString().take(8)}" - }[Organizations.id] - finalOrgRole = "owner" - - // Add membership - Memberships.insert { - it[user_id] = id - it[organization_id] = finalOrgId - it[role] = finalOrgRole - } - - SentryUtils.breadcrumb( - "auth", - "User created new org", - mapOf( - "user_id" to id, - "organization_id" to finalOrgId - ) - ) - } + val (finalOrgId, finalOrgRole) = createUserOrgMembership(id, pendingInvite, request) val acceptedAt = Clock.System.now() UserLegalAcceptances.insert { @@ -319,6 +225,103 @@ class AuthService( ) } + /** + * Resolve a pending invitation for signup: validate a token-based invite or find an email-based one. + * Must be called within a database transaction. + */ + private fun resolvePendingInvite(inviteToken: String?, normalizedEmail: String, now: Long): ResultRow? { + return if (inviteToken != null) { + val inviteByToken = + OrgInvitations + .selectAll() + .where { OrgInvitations.token eq inviteToken } + .singleOrNull() + ?: throw IllegalArgumentException("Invitation not found") + + val inviteStatus = inviteByToken[OrgInvitations.status] + require(inviteStatus == "pending") { "Invitation is no longer valid" } + + val inviteExpiresAt = inviteByToken[OrgInvitations.expires_at] + if (now > inviteExpiresAt) { + OrgInvitations.update({ OrgInvitations.id eq inviteByToken[OrgInvitations.id] }) { + it[OrgInvitations.status] = "expired" + } + throw IllegalArgumentException("Invitation has expired") + } + + require(inviteByToken[OrgInvitations.email].lowercase() == normalizedEmail) { + "This invitation was sent to a different email address" + } + + inviteByToken + } else { + OrgInvitations.update({ + (OrgInvitations.email eq normalizedEmail) and + (OrgInvitations.status eq "pending") and + (OrgInvitations.expires_at lessEq now) + }) { + it[OrgInvitations.status] = "expired" + } + + OrgInvitations + .selectAll() + .where { + (OrgInvitations.email eq normalizedEmail) and + (OrgInvitations.status eq "pending") and + (OrgInvitations.expires_at greater now) + }.orderBy(OrgInvitations.created_at, org.jetbrains.exposed.v1.core.SortOrder.DESC) + .limit(1) + .singleOrNull() + } + } + + /** + * Create an org membership for a new user: join via invite or create a new org. + * Must be called within a database transaction. + * Returns (orgId, orgRole). + */ + private fun createUserOrgMembership( + userId: Int, + pendingInvite: ResultRow?, + request: SignupRequest, + ): Pair { + return if (pendingInvite != null) { + val orgId = pendingInvite[OrgInvitations.organization_id] + val orgRole = pendingInvite[OrgInvitations.role] + Memberships.insert { + it[user_id] = userId + it[organization_id] = orgId + it[role] = orgRole + } + OrgInvitations.update({ OrgInvitations.id eq pendingInvite[OrgInvitations.id] }) { + it[status] = "accepted" + } + SentryUtils.breadcrumb( + "auth", + "User joined via invitation", + mapOf("user_id" to userId, "organization_id" to orgId, "role" to orgRole), + ) + orgId to orgRole + } else { + val orgId = + Organizations.insert { + it[name] = "${request.name ?: request.email}'s Organization" + it[slug] = "org-${UUID.randomUUID().toString().take(ORG_SLUG_RANDOM_SUFFIX_LENGTH)}" + }[Organizations.id] + Memberships.insert { + it[user_id] = userId + it[organization_id] = orgId + it[role] = "owner" + } + SentryUtils.breadcrumb( + "auth", + "User created new org", + mapOf("user_id" to userId, "organization_id" to orgId), + ) + orgId to "owner" + } + } + fun verifyEmail(token: String): Boolean { val user = userRepository.findByEmailVerificationToken(token) ?: return false val expiresAt = user.emailVerificationExpiresAt @@ -400,7 +403,7 @@ class AuthService( .withClaim("email", email) .withClaim("orgId", orgId) .withClaim("orgRole", orgRole) - .withExpiresAt(Date(System.currentTimeMillis() + 3600000)) + .withExpiresAt(Date(System.currentTimeMillis() + JWT_ACCESS_TOKEN_EXPIRY_MS)) .sign(Algorithm.HMAC256(jwtSecret)) } @@ -433,12 +436,12 @@ class AuthService( .withClaim("orgRole", "viewer") .withClaim("isDemo", true) .withClaim("demoEpochMs", demoEpochMs) - .withExpiresAt(Date(System.currentTimeMillis() + 86400000)) // 24 hours + .withExpiresAt(Date(System.currentTimeMillis() + DEMO_TOKEN_EXPIRY_MS)) // 24 hours .sign(Algorithm.HMAC256(jwtSecret)) } private fun generateVerificationToken(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(TOKEN_ENTROPY_BYTES) secureRandom.nextBytes(bytes) return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) } @@ -472,9 +475,7 @@ class AuthService( token: String, newPassword: String ): Boolean { - if (newPassword.length < 8) { - throw IllegalArgumentException("Password must be at least 8 characters") - } + require(newPassword.length >= MIN_PASSWORD_LENGTH) { "Password must be at least 8 characters" } val user = userRepository.findByPasswordResetToken(token) ?: return false val expiresAt = user.passwordResetExpiresAt @@ -509,7 +510,7 @@ class AuthService( .lowercase() .replace(Regex("[^a-z0-9]+"), "-") .trim('-') - .take(100) + .take(MAX_ORG_SLUG_LENGTH) val finalSlug = organizationRepository.updateOnboardingOrgAndMarkComplete( OrganizationRepository.OnboardingUpdate( diff --git a/backend/src/main/kotlin/com/moneat/auth/services/AuthTokenService.kt b/backend/src/main/kotlin/com/moneat/auth/services/AuthTokenService.kt index ffb0a7b01..1f25ee1a5 100644 --- a/backend/src/main/kotlin/com/moneat/auth/services/AuthTokenService.kt +++ b/backend/src/main/kotlin/com/moneat/auth/services/AuthTokenService.kt @@ -58,6 +58,8 @@ class AuthTokenService { // sentry-cli compatible org auth token format: sntrys_{base64_payload}_{base64_secret} private const val TOKEN_PREFIX = "sntrys_" private const val TOKEN_LENGTH = 32 // 32 bytes = 256 bits + private const val MILLIS_PER_SECOND = 1000L + private const val SECONDS_PER_DAY = 86_400 } /** @@ -70,7 +72,8 @@ class AuthTokenService { secretBytes: ByteArray ): String { val backendUrl = EnvConfig.get("BACKEND_URL", "https://api.moneat.io") - val iat = System.currentTimeMillis() / 1000 // Use Long instead of Double to avoid scientific notation + // Use Long instead of Double to avoid scientific notation + val iat = System.currentTimeMillis() / MILLIS_PER_SECOND val payloadJson = """{"iat":$iat,"url":"$backendUrl","region_url":"$backendUrl","org":"$orgSlug"}""" val payloadEncoded = Base64.getEncoder().encodeToString(payloadJson.toByteArray()) val secretEncoded = Base64.getEncoder().withoutPadding().encodeToString(secretBytes) @@ -117,7 +120,7 @@ class AuthTokenService { // Calculate expiration if specified val expiresAt = expiresInDays?.let { - Clock.System.now().plus(it * 24 * 60 * 60, DateTimeUnit.SECOND) + Clock.System.now().plus(it * SECONDS_PER_DAY, DateTimeUnit.SECOND) } val createdAt = Clock.System.now() diff --git a/backend/src/main/kotlin/com/moneat/auth/services/OAuthService.kt b/backend/src/main/kotlin/com/moneat/auth/services/OAuthService.kt index d23598a11..94f9e2be4 100644 --- a/backend/src/main/kotlin/com/moneat/auth/services/OAuthService.kt +++ b/backend/src/main/kotlin/com/moneat/auth/services/OAuthService.kt @@ -108,6 +108,13 @@ class OAuthService { private val backendUrl = EnvConfig.get("BACKEND_URL") ?: "https://api.moneat.io" private val frontendUrl = EnvConfig.get("FRONTEND_URL")!! + companion object { + private const val ORG_SLUG_SUFFIX_LENGTH = 8 + private const val ONE_HOUR_MILLIS = 3_600_000L + private const val STATE_BYTE_LENGTH = 32 + private val HTTP_SUCCESS_STATUS_RANGE = 200..299 + } + private val githubClientId = EnvConfig.get("GITHUB_OAUTH_CLIENT_ID") private val githubClientSecret = EnvConfig.get("GITHUB_OAUTH_CLIENT_SECRET") private val githubOauthBaseUrl = EnvConfig.get("GITHUB_OAUTH_BASE_URL", "https://github.com").trimEnd('/') @@ -168,7 +175,7 @@ class OAuthService { parameter("code", code) } - if (tokenResponse.status.value !in 200..299) { + if (tokenResponse.status.value !in HTTP_SUCCESS_STATUS_RANGE) { logger.error { "GitHub token exchange failed: ${tokenResponse.status}" } throw IllegalArgumentException("Failed to exchange code for token") } @@ -185,7 +192,7 @@ class OAuthService { } } - if (userResponse.status.value !in 200..299) { + if (userResponse.status.value !in HTTP_SUCCESS_STATUS_RANGE) { logger.error { "GitHub user fetch failed: ${userResponse.status}" } throw IllegalArgumentException("Failed to fetch user info") } @@ -205,7 +212,7 @@ class OAuthService { } } - if (emailsResponse.status.value in 200..299) { + if (emailsResponse.status.value in HTTP_SUCCESS_STATUS_RANGE) { val emails: List = emailsResponse.body() val primaryEmail = emails.firstOrNull { it.primary && it.verified } @@ -468,7 +475,7 @@ class OAuthService { val orgId = Organizations.insert { it[name] = "${userData.name ?: userData.email}'s Organization" - it[slug] = "org-${UUID.randomUUID().toString().take(8)}" + it[slug] = "org-${UUID.randomUUID().toString().take(ORG_SLUG_SUFFIX_LENGTH)}" }[Organizations.id] // Add membership @@ -509,12 +516,12 @@ class OAuthService { .withClaim("email", email) .withClaim("orgId", orgId) .withClaim("orgRole", orgRole) - .withExpiresAt(Date(System.currentTimeMillis() + 3600000)) + .withExpiresAt(Date(System.currentTimeMillis() + ONE_HOUR_MILLIS)) .sign(Algorithm.HMAC256(jwtSecret)) } fun generateState(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(STATE_BYTE_LENGTH) java.security.SecureRandom().nextBytes(bytes) return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) } diff --git a/backend/src/main/kotlin/com/moneat/auth/services/RefreshTokenService.kt b/backend/src/main/kotlin/com/moneat/auth/services/RefreshTokenService.kt index 139041b2c..c0e73f0e4 100644 --- a/backend/src/main/kotlin/com/moneat/auth/services/RefreshTokenService.kt +++ b/backend/src/main/kotlin/com/moneat/auth/services/RefreshTokenService.kt @@ -52,6 +52,8 @@ class RefreshTokenService { private const val REFRESH_TOKEN_LENGTH = 64 private const val REFRESH_TOKEN_EXPIRY_DAYS = 30L private const val ACCESS_TOKEN_EXPIRY_HOURS = 1L + private const val MILLIS_PER_DAY = 86_400_000L + private const val MILLIS_PER_HOUR = 3_600_000L private fun hashToken(token: String): String { val digest = MessageDigest.getInstance("SHA-256") @@ -79,7 +81,7 @@ class RefreshTokenService { val refreshToken = generateRandomToken() val tokenHash = hashToken(refreshToken) val now = System.currentTimeMillis() - val expiresAt = now + (REFRESH_TOKEN_EXPIRY_DAYS * 24 * 60 * 60 * 1000) + val expiresAt = now + (REFRESH_TOKEN_EXPIRY_DAYS * MILLIS_PER_DAY) transaction { RefreshTokens.insert { @@ -201,7 +203,7 @@ class RefreshTokenService { .withClaim("email", email) .withClaim("orgId", orgId) .withClaim("orgRole", effectiveRole) - .withExpiresAt(Date(issuedAt.time + (ACCESS_TOKEN_EXPIRY_HOURS * 3600000))) + .withExpiresAt(Date(issuedAt.time + (ACCESS_TOKEN_EXPIRY_HOURS * MILLIS_PER_HOUR))) if (isDemoIdentity) { jwtBuilder diff --git a/backend/src/main/kotlin/com/moneat/billing/routes/BillingRoutes.kt b/backend/src/main/kotlin/com/moneat/billing/routes/BillingRoutes.kt index 673c00e43..df630eae2 100644 --- a/backend/src/main/kotlin/com/moneat/billing/routes/BillingRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/billing/routes/BillingRoutes.kt @@ -32,6 +32,7 @@ import com.moneat.shared.services.UsageTrackingService import com.moneat.utils.BooleanResponse import com.moneat.utils.ErrorResponse import io.ktor.http.HttpStatusCode +import io.ktor.server.application.ApplicationCall import io.ktor.server.auth.jwt.JWTPrincipal import io.ktor.server.auth.principal import io.ktor.server.request.receive @@ -58,6 +59,10 @@ private const val FAILED_TO_CREATE_CHECKOUT_SESSION = "Failed to create checkout private const val FAILED_TO_CANCEL_SUBSCRIPTION = "Failed to cancel subscription" private const val FAILED_TO_UPDATE_ON_CALL_SEATS = "Failed to update on-call seats" private const val FAILED_TO_UPDATE_SEATS = "Failed to update seats" +private const val PAYG_INCREMENT_CENTS = 500 +private const val MAX_ONCALL_SEATS = 200 +private const val AUTH_REQUIRED = "Authentication required" +private const val NO_ORG_ACCESS = "No organization access" // Public billing endpoints (no auth required) fun Route.publicBillingRoutes( @@ -90,13 +95,13 @@ fun Route.billingRoutes( get("/usage") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@get } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@get } @@ -145,52 +150,28 @@ fun Route.billingRoutes( post("/checkout") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@post } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@post } - - val request = call.receive() - suspendRunCatching { - val response = - stripeService.createCheckoutSession( - organizationId = orgId, - tierName = request.tierName, - billingInterval = request.billingInterval, - successUrl = request.successUrl, - cancelUrl = request.cancelUrl, - oncallSeats = request.oncallSeats - ) - call.respond(response) - }.onFailure { e -> - when (e) { - is IllegalArgumentException -> call.respond( - HttpStatusCode.BadRequest, - ErrorResponse(e.message ?: "Invalid checkout request") - ) - else -> call.respond( - HttpStatusCode.InternalServerError, - ErrorResponse(e.message ?: FAILED_TO_CREATE_CHECKOUT_SESSION) - ) - } - } + handleBillingCheckout(call, stripeService, orgId) } get("/invoices") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@get } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@get } @@ -204,13 +185,13 @@ fun Route.billingRoutes( get("/payment-method") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@get } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@get } @@ -224,13 +205,13 @@ fun Route.billingRoutes( post("/setup-intent") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@post } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@post } @@ -244,13 +225,13 @@ fun Route.billingRoutes( post("/setup-intent/confirm") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@post } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@post } @@ -267,135 +248,188 @@ fun Route.billingRoutes( post("/cancel") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@post } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@post } - - suspendRunCatching { - call.respond(stripeService.cancelSubscription(orgId)) - }.onFailure { e -> - when (e) { - is IllegalStateException -> call.respond( - HttpStatusCode.BadRequest, - ErrorResponse(e.message ?: "No cancelable subscription found") - ) - else -> call.respond( - HttpStatusCode.InternalServerError, - ErrorResponse(e.message ?: FAILED_TO_CANCEL_SUBSCRIPTION) - ) - } - } + handleBillingCancel(call, stripeService, orgId) } put("/payg-budget") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@put } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@put } - val request = call.receive() - if (request.paygBudgetCents < 0 || request.paygBudgetCents % 500 != 0) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse("PAYG budget must be in $5 increments")) - return@put - } - - val tierContext = pricingTierService.getEffectiveTierForOrganization(orgId) - if (!tierContext.tier.paygEnabled || tierContext.tier.tierName.equals("FREE", ignoreCase = true)) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse("PAYG budget is only available on paid tiers")) - return@put - } - - val updated = - transaction { - val sub = - Subscriptions - .selectAll() - .where { - (Subscriptions.organization_id eq orgId) and - (Subscriptions.status inList listOf("active", "trialing", "past_due")) - }.orderBy(Subscriptions.id to SortOrder.DESC) - .firstOrNull() - ?: return@transaction false - Subscriptions.update({ Subscriptions.id eq sub[Subscriptions.id] }) { - it[payg_budget_cents] = request.paygBudgetCents - } - true - } - if (!updated) { - call.respond(HttpStatusCode.NotFound, ErrorResponse("No active subscription found")) - return@put - } - call.respond(UpdatePaygBudgetResponse(paygBudgetCents = request.paygBudgetCents)) + handleBillingPaygBudget(call, pricingTierService, orgId) } put("/oncall-seats") { val principal = call.principal() ?: run { - call.respond(HttpStatusCode.Unauthorized, ErrorResponse("Authentication required")) + call.respond(HttpStatusCode.Unauthorized, ErrorResponse(AUTH_REQUIRED)) return@put } val userId = principal.payload.getClaim("userId").asInt() val orgId = pricingTierService.getPrimaryOrganizationIdForUser(userId) ?: run { - call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization access")) + call.respond(HttpStatusCode.Forbidden, ErrorResponse(NO_ORG_ACCESS)) return@put } + handleBillingOnCallSeats(call, stripeService, orgId) + } + } +} - val request = call.receive() - if (request.seats < 0) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse("Seats must be non-negative")) - return@put - } - if (request.seats > 200) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse("Maximum 200 on-call seats allowed")) - return@put - } +private suspend fun handleBillingCheckout( + call: ApplicationCall, + stripeService: StripeService, + orgId: Int, +) { + val request = call.receive() + suspendRunCatching { + val response = + stripeService.createCheckoutSession( + organizationId = orgId, + tierName = request.tierName, + billingInterval = request.billingInterval, + successUrl = request.successUrl, + cancelUrl = request.cancelUrl, + oncallSeats = request.oncallSeats + ) + call.respond(response) + }.onFailure { e -> + when (e) { + is IllegalArgumentException -> call.respond( + HttpStatusCode.BadRequest, + ErrorResponse(e.message ?: "Invalid checkout request") + ) + else -> call.respond( + HttpStatusCode.InternalServerError, + ErrorResponse(e.message ?: FAILED_TO_CREATE_CHECKOUT_SESSION) + ) + } + } +} - // Check if seats >= currently used - val usedSeats = - suspendRunCatching { - val clazz = Class.forName("com.moneat.enterprise.services.oncall.OnCallScheduleService") - val instance = clazz.getDeclaredConstructor().newInstance() - val method = clazz.getMethod("getOnCallUsedSeats", Int::class.java) - method.invoke(instance, orgId) as? Int ?: 0 - }.getOrElse { _ -> - 0 - } - if (request.seats < usedSeats) { - call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Cannot reduce seats below currently assigned users ($usedSeats)") - ) - return@put - } +private suspend fun handleBillingCancel( + call: ApplicationCall, + stripeService: StripeService, + orgId: Int, +) { + suspendRunCatching { + call.respond(stripeService.cancelSubscription(orgId)) + }.onFailure { e -> + when (e) { + is IllegalStateException -> call.respond( + HttpStatusCode.BadRequest, + ErrorResponse(e.message ?: "No cancelable subscription found") + ) + else -> call.respond( + HttpStatusCode.InternalServerError, + ErrorResponse(e.message ?: FAILED_TO_CANCEL_SUBSCRIPTION) + ) + } + } +} + +private suspend fun handleBillingPaygBudget( + call: ApplicationCall, + pricingTierService: PricingTierService, + orgId: Int, +) { + val request = call.receive() + if (request.paygBudgetCents < 0 || request.paygBudgetCents % PAYG_INCREMENT_CENTS != 0) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("PAYG budget must be in $5 increments")) + return + } + + val tierContext = pricingTierService.getEffectiveTierForOrganization(orgId) + if (!tierContext.tier.paygEnabled || tierContext.tier.tierName.equals("FREE", ignoreCase = true)) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("PAYG budget is only available on paid tiers")) + return + } - try { - val response = stripeService.updateOnCallSeats(orgId, request.seats) - call.respond(response) - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse((e.message ?: "Invalid request"))) - } catch (e: SerializationException) { - logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } - call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) - } catch (e: IOException) { - logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } - call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) - } catch (e: IllegalStateException) { - logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } - call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) + val updated = + transaction { + val sub = + Subscriptions + .selectAll() + .where { + (Subscriptions.organization_id eq orgId) and + (Subscriptions.status inList listOf("active", "trialing", "past_due")) + }.orderBy(Subscriptions.id to SortOrder.DESC) + .firstOrNull() + ?: return@transaction false + Subscriptions.update({ Subscriptions.id eq sub[Subscriptions.id] }) { + it[payg_budget_cents] = request.paygBudgetCents } + true + } + if (!updated) { + call.respond(HttpStatusCode.NotFound, ErrorResponse("No active subscription found")) + return + } + call.respond(UpdatePaygBudgetResponse(paygBudgetCents = request.paygBudgetCents)) +} + +private suspend fun handleBillingOnCallSeats( + call: ApplicationCall, + stripeService: StripeService, + orgId: Int, +) { + val request = call.receive() + if (request.seats < 0) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Seats must be non-negative")) + return + } + if (request.seats > MAX_ONCALL_SEATS) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Maximum 200 on-call seats allowed")) + return + } + + // Check if seats >= currently used + val usedSeats = + suspendRunCatching { + val clazz = Class.forName("com.moneat.enterprise.services.oncall.OnCallScheduleService") + val instance = clazz.getDeclaredConstructor().newInstance() + val method = clazz.getMethod("getOnCallUsedSeats", Int::class.java) + method.invoke(instance, orgId) as? Int ?: 0 + }.getOrElse { _ -> + 0 } + if (request.seats < usedSeats) { + call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Cannot reduce seats below currently assigned users ($usedSeats)") + ) + return + } + + try { + val response = stripeService.updateOnCallSeats(orgId, request.seats) + call.respond(response) + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse((e.message ?: "Invalid request"))) + } catch (e: SerializationException) { + logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } + call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) + } catch (e: IOException) { + logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } + call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) + } catch (e: IllegalStateException) { + logger.error(e) { FAILED_TO_UPDATE_ON_CALL_SEATS } + call.respond(HttpStatusCode.InternalServerError, ErrorResponse(FAILED_TO_UPDATE_SEATS)) } } diff --git a/backend/src/main/kotlin/com/moneat/billing/services/BillingBackgroundService.kt b/backend/src/main/kotlin/com/moneat/billing/services/BillingBackgroundService.kt index e2d0d9360..541de7d9d 100644 --- a/backend/src/main/kotlin/com/moneat/billing/services/BillingBackgroundService.kt +++ b/backend/src/main/kotlin/com/moneat/billing/services/BillingBackgroundService.kt @@ -48,6 +48,10 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val BILLING_JOB_INTERVAL_MS = 60_000L +private const val QUOTA_WARNING_THRESHOLD = 0.8 +private const val CENTS_PER_DOLLAR = 100.0 + class BillingBackgroundService( private val stripeService: StripeService = StripeService( SubscriptionRepositoryImpl(), @@ -82,7 +86,7 @@ class BillingBackgroundService( val flushed = stripeService.flushPendingMeteredUsage() if (flushed > 0) logger.info { "Flushed pending metered usage for $flushed subscription(s)" } } - delay(60_000L) + delay(BILLING_JOB_INTERVAL_MS) } } @@ -96,7 +100,7 @@ class BillingBackgroundService( ) { stripeService.applyDunningDowngrade() } - delay(60_000L) + delay(BILLING_JOB_INTERVAL_MS) } } @@ -110,7 +114,7 @@ class BillingBackgroundService( ) { processQuotaThresholdNotifications() } - delay(60_000L) + delay(BILLING_JOB_INTERVAL_MS) } } } @@ -151,13 +155,13 @@ class BillingBackgroundService( 0.0 } - if (basePct >= 0.8) { + if (basePct >= QUOTA_WARNING_THRESHOLD) { maybeSendNotification(orgId, periodStart, "base_80", usage) } if (basePct >= 1.0) { maybeSendNotification(orgId, periodStart, "base_100", usage) } - if (usage.paygLimitUnits > 0 && paygPct >= 0.8) { + if (usage.paygLimitUnits > 0 && paygPct >= QUOTA_WARNING_THRESHOLD) { maybeSendNotification(orgId, periodStart, "payg_80", usage) } } @@ -229,8 +233,8 @@ class BillingBackgroundService( appendLine("Plan: ${usage.plan}") appendLine("Usage: ${usage.usedUnits}/${usage.totalLimitUnits} units") appendLine("Base limit: ${usage.baseLimitUnits} units") - appendLine("PAYG budget: $${"%.2f".format(usage.paygBudgetCents / 100.0)}") - appendLine("PAYG used estimate: $${"%.2f".format(usage.paygUsedCentsEstimate / 100.0)}") + appendLine("PAYG budget: $${"%.2f".format(usage.paygBudgetCents / CENTS_PER_DOLLAR)}") + appendLine("PAYG used estimate: $${"%.2f".format(usage.paygUsedCentsEstimate / CENTS_PER_DOLLAR)}") appendLine("Billing period: ${usage.periodStart} to ${usage.periodEnd}") } diff --git a/backend/src/main/kotlin/com/moneat/billing/services/BillingQuotaService.kt b/backend/src/main/kotlin/com/moneat/billing/services/BillingQuotaService.kt index 13d4c6ecc..dfc36938b 100644 --- a/backend/src/main/kotlin/com/moneat/billing/services/BillingQuotaService.kt +++ b/backend/src/main/kotlin/com/moneat/billing/services/BillingQuotaService.kt @@ -59,6 +59,10 @@ class BillingQuotaService( ) { companion object { private const val BYTES_PER_GB = 1_073_741_824L + private const val MICROS_PER_CENT = 10_000L + private const val UNITS_PER_THOUSAND = 1_000L + private const val UNITS_PER_MILLION = 1_000_000L + private const val UNITS_PER_HUNDRED_THOUSAND = 100_000L } fun isEnforcementEnabled(): Boolean { @@ -714,7 +718,7 @@ class BillingQuotaService( val paygEnabled = tier.paygEnabled val paygLimitUnits = if (paygEnabled && paygBudgetCents > 0 && paygRateMicros > 0) { - (paygBudgetCents.toLong() * 10_000L) / paygRateMicros + (paygBudgetCents.toLong() * MICROS_PER_CENT) / paygRateMicros } else { 0 } @@ -831,7 +835,7 @@ class BillingQuotaService( val errorOverageUnits = max(0, state.usedErrors - state.errorLimit) val errorOverageCents = if (state.errorOverageRateCentsPer1k > 0 && errorOverageUnits > 0) { - ((errorOverageUnits * state.errorOverageRateCentsPer1k) / 1000).toInt() + ((errorOverageUnits * state.errorOverageRateCentsPer1k) / UNITS_PER_THOUSAND).toInt() } else { 0 } @@ -863,7 +867,7 @@ class BillingQuotaService( val llmOverageUnits = max(0, state.usedLlmEvents - state.llmEventLimit) val llmOverageCents = if (state.llmOverageRateCentsPer1k > 0 && llmOverageUnits > 0) { - ((llmOverageUnits * state.llmOverageRateCentsPer1k) / 1000).toInt() + ((llmOverageUnits * state.llmOverageRateCentsPer1k) / UNITS_PER_THOUSAND).toInt() } else { 0 } @@ -878,7 +882,7 @@ class BillingQuotaService( ( analyticsPageviewOverageUnits * state.analyticsPageviewOverageRateCentsPer100k - ) / 100_000 + ) / UNITS_PER_HUNDRED_THOUSAND ).toInt() } else { 0 @@ -891,7 +895,7 @@ class BillingQuotaService( } val apmSpanOverageCents = if (state.apmSpanOverageRateCentsPer1m > 0 && apmSpanOverageUnits > 0) { - ((apmSpanOverageUnits * state.apmSpanOverageRateCentsPer1m) / 1_000_000).toInt() + ((apmSpanOverageUnits * state.apmSpanOverageRateCentsPer1m) / UNITS_PER_MILLION).toInt() } else { 0 } @@ -907,7 +911,7 @@ class BillingQuotaService( ( customMetricOverageUnits * state.customMetricOverageRateCentsPer100k - ) / 100_000 + ) / UNITS_PER_HUNDRED_THOUSAND ).toInt() } else { 0 diff --git a/backend/src/main/kotlin/com/moneat/billing/services/PricingTierService.kt b/backend/src/main/kotlin/com/moneat/billing/services/PricingTierService.kt index febdfe27a..f3a716859 100644 --- a/backend/src/main/kotlin/com/moneat/billing/services/PricingTierService.kt +++ b/backend/src/main/kotlin/com/moneat/billing/services/PricingTierService.kt @@ -56,6 +56,18 @@ data class EffectiveTierContext( val currentPeriodEnd: kotlinx.datetime.LocalDate? ) +private const val DEFAULT_ANALYTICS_RETENTION_DAYS = 1095 +private const val MAX_RETENTION_DAYS = 90 +private const val MAX_TRIAL_DAYS = 60 +private const val MAX_ANALYTICS_RETENTION_DAYS = 3650 +private const val PRO_MONTHLY_PRICE_CENTS = 2900 +private const val TEAM_MONTHLY_PRICE_CENTS = 7900 +private const val BUSINESS_MONTHLY_PRICE_CENTS = 19900 +private const val PRO_YEARLY_PRICE_CENTS = 28800 +private const val TEAM_YEARLY_PRICE_CENTS = 79200 +private const val BUSINESS_YEARLY_PRICE_CENTS = 199200 +private const val DEFAULT_TRIAL_DAYS_NON_FREE = 14 + class PricingTierService { private data class DefaultFeatureFlags( val statusPagesEnabled: Boolean, @@ -242,7 +254,9 @@ class PricingTierService { val resolvedOncallEnabled = request.oncallEnabled ?: currentConfig?.oncallEnabled ?: false val resolvedMaxAnalyticsSites = request.maxAnalyticsSites val resolvedAnalyticsRetentionDays = - request.analyticsRetentionDays ?: currentConfig?.analyticsRetentionDays ?: 1095 + request.analyticsRetentionDays + ?: currentConfig?.analyticsRetentionDays + ?: DEFAULT_ANALYTICS_RETENTION_DAYS val resolvedMonthlyAnalyticsPageviewLimit = request.monthlyAnalyticsPageviewLimit ?: currentConfig?.monthlyAnalyticsPageviewLimit ?: 0 val resolvedAnalyticsPageviewOverageRateCentsPer100k = @@ -397,9 +411,9 @@ class PricingTierService { } private fun validateCreateTierRequest(request: CreateTierVersionRequest) { - require(request.retentionDays in 1..90) { "Retention days must be between 1 and 90" } + require(request.retentionDays in 1..MAX_RETENTION_DAYS) { "Retention days must be between 1 and 90" } if (request.logRetentionDays != null) { - require(request.logRetentionDays in 1..90) { "Log retention days must be between 1 and 90" } + require(request.logRetentionDays in 1..MAX_RETENTION_DAYS) { "Log retention days must be between 1 and 90" } } require(request.monthlyUnitLimit >= 0) { "Monthly unit limit must be non-negative" } require(request.monthlyErrorLimit >= 0) { "Monthly error limit must be non-negative" } @@ -408,10 +422,12 @@ class PricingTierService { require(request.monthlyFeedbackLimit >= 0) { "Monthly feedback limit must be non-negative" } require(request.monthlyLlmEventLimit >= 0) { "Monthly LLM event limit must be non-negative" } if (request.replayRetentionDays != null) { - require(request.replayRetentionDays in 1..90) { "Replay retention days must be between 1 and 90" } + require(request.replayRetentionDays in 1..MAX_RETENTION_DAYS) { + "Replay retention days must be between 1 and 90" + } } if (request.llmRetentionDays != null) { - require(request.llmRetentionDays in 1..90) { "LLM retention days must be between 1 and 90" } + require(request.llmRetentionDays in 1..MAX_RETENTION_DAYS) { "LLM retention days must be between 1 and 90" } } if (request.monthlyGbLimit != null) { require(request.monthlyGbLimit >= 0) { "Monthly GB limit must be non-negative" } @@ -420,7 +436,7 @@ class PricingTierService { require(request.yearlyPriceCents >= 0) { "Yearly price cannot be negative" } } if (request.trialDays != null) { - require(request.trialDays in 0..60) { "Trial days must be between 0 and 60" } + require(request.trialDays in 0..MAX_TRIAL_DAYS) { "Trial days must be between 0 and 60" } } if (request.overageRateCentsPerGb != null) { require(request.overageRateCentsPerGb >= 0) { "Overage rate cannot be negative" } @@ -435,7 +451,9 @@ class PricingTierService { require(request.llmOverageRateCentsPer1k >= 0) { "LLM overage rate cannot be negative" } } if (request.analyticsRetentionDays != null) { - require(request.analyticsRetentionDays in 1..3650) { "Analytics retention days must be between 1 and 3650" } + require(request.analyticsRetentionDays in 1..MAX_ANALYTICS_RETENTION_DAYS) { + "Analytics retention days must be between 1 and 3650" + } } if (request.monthlyAnalyticsPageviewLimit != null) { require( @@ -575,21 +593,21 @@ class PricingTierService { val monthlyPrice = when (tier) { PricingTier.FREE -> 0 - PricingTier.PRO -> 2900 - PricingTier.TEAM -> 7900 - PricingTier.BUSINESS -> 19900 + PricingTier.PRO -> PRO_MONTHLY_PRICE_CENTS + PricingTier.TEAM -> TEAM_MONTHLY_PRICE_CENTS + PricingTier.BUSINESS -> BUSINESS_MONTHLY_PRICE_CENTS } val yearlyPrice = when (tier) { PricingTier.FREE -> 0 - PricingTier.PRO -> 28800 + PricingTier.PRO -> PRO_YEARLY_PRICE_CENTS // $288/yr - PricingTier.TEAM -> 79200 + PricingTier.TEAM -> TEAM_YEARLY_PRICE_CENTS // $792/yr - PricingTier.BUSINESS -> 199200 // $1992/yr + PricingTier.BUSINESS -> BUSINESS_YEARLY_PRICE_CENTS // $1992/yr } return PricingTierConfigResponse( id = 0, @@ -736,6 +754,6 @@ class PricingTierService { } private fun defaultTrialDaysForTier(tierName: String): Int { - return if (tierName.uppercase() == "FREE") 0 else 14 + return if (tierName.uppercase() == "FREE") 0 else DEFAULT_TRIAL_DAYS_NON_FREE } } diff --git a/backend/src/main/kotlin/com/moneat/billing/services/StripeService.kt b/backend/src/main/kotlin/com/moneat/billing/services/StripeService.kt index 4dbdf7e8d..74386fc2a 100644 --- a/backend/src/main/kotlin/com/moneat/billing/services/StripeService.kt +++ b/backend/src/main/kotlin/com/moneat/billing/services/StripeService.kt @@ -243,7 +243,7 @@ class StripeService( InvoiceListParams .builder() .setCustomer(customerId) - .setLimit(limit.coerceIn(1, 100)) + .setLimit(limit.coerceIn(1, MAX_INVOICES_LIMIT)) .build() ) return invoices.data.map { invoice -> @@ -770,7 +770,7 @@ class StripeService( it[plan] = "free" it[status] = "active" it[current_period_start] = Clock.System.now() - it[current_period_end] = addDays(Clock.System.now(), 30) + it[current_period_end] = addDays(Clock.System.now(), FREE_TIER_PERIOD_DAYS) it[payg_budget_cents] = 0 it[payg_used_units] = 0 it[payg_used_micros] = 0 @@ -827,9 +827,9 @@ class StripeService( // that cannot form a full unit remain in pending_overage_bytes for the // next flush cycle rather than being silently dropped. val pendingBytes = row[Subscriptions.pending_overage_bytes] - val drainableUnits = pendingBytes / (BYTES_PER_GB / 100) + val drainableUnits = pendingBytes / (BYTES_PER_GB / GB_METER_UNITS_PER_GB) if (drainableUnits > 0) { - val remainingBytes = pendingBytes - drainableUnits * (BYTES_PER_GB / 100) + val remainingBytes = pendingBytes - drainableUnits * (BYTES_PER_GB / GB_METER_UNITS_PER_GB) Subscriptions.update({ Subscriptions.id eq subscriptionId }) { it[pending_meter_units] = row[Subscriptions.pending_meter_units] + drainableUnits it[pending_overage_bytes] = remainingBytes @@ -1039,7 +1039,7 @@ class StripeService( it[plan] = "free" it[status] = "active" it[current_period_start] = now - it[current_period_end] = addDays(now, 30) + it[current_period_end] = addDays(now, FREE_TIER_PERIOD_DAYS) it[pricing_tier_config_id] = freeTier?.id?.takeIf { id -> id > 0 } it[payg_budget_cents] = 0 it[payg_used_units] = 0 @@ -1141,7 +1141,7 @@ class StripeService( instant: Instant, days: Int ): Instant { - return Instant.fromEpochSeconds(instant.epochSeconds + (days * 86_400L)) + return Instant.fromEpochSeconds(instant.epochSeconds + (days * SECONDS_PER_DAY)) } private fun epochSecondsToIso(epochSeconds: Long?): String? { @@ -1163,5 +1163,9 @@ class StripeService( companion object { private val TERMINAL_WEBHOOK_STATUSES = listOf("processed", "success", "skipped") private const val BYTES_PER_GB = 1_073_741_824L + private const val MAX_INVOICES_LIMIT = 100L + private const val FREE_TIER_PERIOD_DAYS = 30 + private const val GB_METER_UNITS_PER_GB = 100 + private const val SECONDS_PER_DAY = 86_400L } } diff --git a/backend/src/main/kotlin/com/moneat/config/ClickHouseClient.kt b/backend/src/main/kotlin/com/moneat/config/ClickHouseClient.kt index c13165f78..9b7f3fbcb 100644 --- a/backend/src/main/kotlin/com/moneat/config/ClickHouseClient.kt +++ b/backend/src/main/kotlin/com/moneat/config/ClickHouseClient.kt @@ -31,6 +31,13 @@ import com.moneat.utils.suspendRunCatching object ClickHouseClient { private const val MIGRATION_TIMEOUT_MS = 600_000L + private const val HTTP_MAX_CONNECTIONS = 100 + private const val KEEP_ALIVE_MS = 5_000L + private const val CONNECT_TIMEOUT_MS = 10_000L + private const val SOCKET_TIMEOUT_MS = 30_000L + private const val MIGRATION_MAX_CONNECTIONS = 4 + private const val QUERY_LOG_MAX_LEN = 200 + private const val ERROR_BODY_MAX_LEN = 500 @Volatile private var httpClient: HttpClient? = null @@ -56,27 +63,27 @@ object ClickHouseClient { this.httpClient = HttpClient(CIO) { engine { - maxConnectionsCount = 100 + maxConnectionsCount = HTTP_MAX_CONNECTIONS endpoint { - keepAliveTime = 5000 - connectTimeout = 10_000 - socketTimeout = 30_000 + keepAliveTime = KEEP_ALIVE_MS + connectTimeout = CONNECT_TIMEOUT_MS + socketTimeout = SOCKET_TIMEOUT_MS } } } this.migrationClient = HttpClient(CIO) { engine { - maxConnectionsCount = 4 + maxConnectionsCount = MIGRATION_MAX_CONNECTIONS endpoint { - keepAliveTime = 5000 - connectTimeout = 10_000 + keepAliveTime = KEEP_ALIVE_MS + connectTimeout = CONNECT_TIMEOUT_MS socketTimeout = MIGRATION_TIMEOUT_MS } } install(HttpTimeout) { requestTimeoutMillis = MIGRATION_TIMEOUT_MS - connectTimeoutMillis = 10_000 + connectTimeoutMillis = CONNECT_TIMEOUT_MS socketTimeoutMillis = MIGRATION_TIMEOUT_MS } } @@ -91,7 +98,7 @@ object ClickHouseClient { SentryUtils.withSpan(span, "db.clickhouse", "ClickHouse query") { childSpan -> childSpan?.setData("db.system", "clickhouse") childSpan?.setData("db.name", database) - childSpan?.setData("db.statement", query.take(200)) // Truncate long queries + childSpan?.setData("db.statement", query.take(QUERY_LOG_MAX_LEN)) // Truncate long queries client.post(baseUrl) { parameter("database", database) @@ -121,7 +128,7 @@ object ClickHouseClient { val response = execute(queryWithFormat, span) val body = response.bodyAsText() check(!response.isClickHouseError(body)) { - "ClickHouse query failed (${response.status.value}): ${body.take(500)}" + "ClickHouse query failed (${response.status.value}): ${body.take(ERROR_BODY_MAX_LEN)}" } return body } diff --git a/backend/src/main/kotlin/com/moneat/config/ClickHouseMigrations.kt b/backend/src/main/kotlin/com/moneat/config/ClickHouseMigrations.kt index 1b029dbe2..2910c2652 100644 --- a/backend/src/main/kotlin/com/moneat/config/ClickHouseMigrations.kt +++ b/backend/src/main/kotlin/com/moneat/config/ClickHouseMigrations.kt @@ -41,6 +41,7 @@ import com.moneat.utils.suspendRunCatching object ClickHouseMigrations { private const val MIGRATIONS_TABLE = "schema_migrations" private const val MIGRATIONS_PATH = "db/clickhouse_migration" + private const val ERROR_BODY_MAX_LEN = 500 data class Migration( val version: Int, @@ -180,7 +181,7 @@ object ClickHouseMigrations { val response = ClickHouseClient.executeMigration(query) val body = response.bodyAsText() if (response.isClickHouseError(body)) { - throw RuntimeException("ClickHouse error reading applied migrations: ${body.take(500)}") + throw RuntimeException("ClickHouse error reading applied migrations: ${body.take(ERROR_BODY_MAX_LEN)}") } if (body.isBlank()) return emptyList() diff --git a/backend/src/main/kotlin/com/moneat/config/SentryConfig.kt b/backend/src/main/kotlin/com/moneat/config/SentryConfig.kt index 74f4ee8eb..d6366968b 100644 --- a/backend/src/main/kotlin/com/moneat/config/SentryConfig.kt +++ b/backend/src/main/kotlin/com/moneat/config/SentryConfig.kt @@ -21,6 +21,10 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} +private const val DEFAULT_SAMPLE_RATE = 0.1 +private const val MAX_BREADCRUMBS = 100 +private const val DSN_LOG_LENGTH = 20 + object SentryConfig { fun initialize() { val dsn = EnvConfig.get("SENTRY_DSN") @@ -36,18 +40,20 @@ object SentryConfig { options.release = EnvConfig.get("RELEASE_VERSION") ?: "moneat@${System.getProperty("app.version", "dev")}" // Performance monitoring - val tracesSampleRate = EnvConfig.get("SENTRY_TRACES_SAMPLE_RATE", "0.1").toDoubleOrNull() ?: 0.1 + val tracesSampleRate = EnvConfig.get("SENTRY_TRACES_SAMPLE_RATE", "0.1").toDoubleOrNull() + ?: DEFAULT_SAMPLE_RATE options.tracesSampleRate = tracesSampleRate // Enable profiling (if supported) - val profilesSampleRate = EnvConfig.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1").toDoubleOrNull() ?: 0.1 + val profilesSampleRate = EnvConfig.get("SENTRY_PROFILES_SAMPLE_RATE", "0.1").toDoubleOrNull() + ?: DEFAULT_SAMPLE_RATE options.profilesSampleRate = profilesSampleRate // Set server name options.serverName = EnvConfig.get("HOSTNAME", "moneat-backend") // Enable breadcrumbs - options.maxBreadcrumbs = 100 + options.maxBreadcrumbs = MAX_BREADCRUMBS // Send default PII (Personal Identifiable Information) options.isSendDefaultPii = false @@ -64,7 +70,7 @@ object SentryConfig { logger.info { "Sentry initialized with DSN: ${dsn.take( - 20 + DSN_LOG_LENGTH )}..., traces: $tracesSampleRate, profiles: $profilesSampleRate" } } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardRepositoryImpl.kt b/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardRepositoryImpl.kt index 01ecdb930..2412edb4e 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardRepositoryImpl.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardRepositoryImpl.kt @@ -43,6 +43,9 @@ private val json = Json { } class DashboardRepositoryImpl : DashboardRepository { + companion object { + private const val RECENTLY_VIEWED_LIMIT = 10 + } override fun list(orgId: Long, projectId: Long?, userId: Int?): List = transaction { @@ -221,7 +224,7 @@ class DashboardRepositoryImpl : DashboardRepository { ) } .orderBy(Dashboards.updatedAt, SortOrder.DESC) - .limit(10) + .limit(RECENTLY_VIEWED_LIMIT) .map { row -> val did = row[Dashboards.id] val isFav = userId?.let { uid -> diff --git a/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardWidgetRepositoryImpl.kt b/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardWidgetRepositoryImpl.kt index da588d5c0..fa8261417 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardWidgetRepositoryImpl.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/repositories/DashboardWidgetRepositoryImpl.kt @@ -39,6 +39,10 @@ private val json = Json { } class DashboardWidgetRepositoryImpl : DashboardWidgetRepository { + companion object { + private const val DEFAULT_WIDGET_WIDTH = 6 + private const val DEFAULT_WIDGET_HEIGHT = 4 + } override fun listByDashboardId(dashboardId: Long): List = transaction { @@ -121,8 +125,8 @@ class DashboardWidgetRepositoryImpl : DashboardWidgetRepository { it[DashboardWidgets.widgetType] = widget.widgetType ?: "timeseries" it[DashboardWidgets.gridX] = widget.gridX ?: 0 it[DashboardWidgets.gridY] = widget.gridY ?: 0 - it[DashboardWidgets.gridW] = widget.gridW ?: 6 - it[DashboardWidgets.gridH] = widget.gridH ?: 4 + it[DashboardWidgets.gridW] = widget.gridW ?: DEFAULT_WIDGET_WIDTH + it[DashboardWidgets.gridH] = widget.gridH ?: DEFAULT_WIDGET_HEIGHT it[DashboardWidgets.queryConfig] = widget.queryConfigs ?.firstOrNull() ?.let { qc -> json.encodeToString(qc) } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/routes/DashboardRoutes.kt b/backend/src/main/kotlin/com/moneat/dashboards/routes/DashboardRoutes.kt index 1938e9145..0a1909935 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/routes/DashboardRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/routes/DashboardRoutes.kt @@ -42,6 +42,7 @@ import com.moneat.dashboards.services.CustomDataSourceExecutor import com.moneat.dashboards.services.CustomDataSourceService import com.moneat.dashboards.services.DashboardAlertService import com.moneat.dashboards.services.DashboardQueryEngine +import com.moneat.dashboards.translation.DashboardTranslator import com.moneat.dashboards.translation.DataDogTranslator import com.moneat.dashboards.translation.GrafanaTranslator import com.moneat.plugins.getDemoEpochMs @@ -74,6 +75,18 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} private val json = Json { ignoreUnknownKeys = true } +private const val QUERY_PREVIEW_LENGTH = 80 +private const val DEFAULT_RETENTION_DAYS = 90 +private const val MAX_QUERIES_PER_REQUEST = 10 + +private const val AUTH_JWT = "auth-jwt" +private const val ERR_NO_ORGANIZATION = "No organization found" +private const val ERR_INVALID_DASHBOARD_ID = "Invalid dashboard ID" +private const val ERR_DASHBOARD_NOT_FOUND = "Dashboard not found" +private const val ERR_DATA_SOURCE_NOT_FOUND = "Data source not found" +private const val ERR_UNKNOWN_SOURCE_TYPE = "Unknown source type" +private const val ERR_INVALID_DATA_SOURCE_ID = "Invalid data source ID" + private fun getOrgIdForUser(userId: Int): Long? { return transaction { Memberships.selectAll() @@ -109,760 +122,835 @@ private fun getDashboardScope(dashboardId: Long, orgId: Long): DashboardScope? { } } -fun Route.customDashboardRoutes( - dashboardService: CustomDashboardService = GlobalContext.get().get(), - queryEngine: DashboardQueryEngine = GlobalContext.get().get(), - retentionPolicyService: RetentionPolicyService = GlobalContext.get().get(), - dataDogTranslator: DataDogTranslator = DataDogTranslator(), - grafanaTranslator: GrafanaTranslator = GrafanaTranslator(), - dataSourceService: CustomDataSourceService = GlobalContext.get().get(), - dataSourceExecutor: CustomDataSourceExecutor = GlobalContext.get().get(), - dashboardAlertService: DashboardAlertService = GlobalContext.get().get(), -) { - // Resolve __prometheus marker to the org's first Prometheus custom datasource - fun resolvePrometheusDataSource(dsl: QueryDsl, orgId: Long): QueryDsl { - if (dsl.dataSource != "__prometheus") return dsl - val sources = dataSourceService.listDataSources(orgId) - val promSource = sources.firstOrNull { it.sourceType.equals("prometheus", ignoreCase = true) } - if (promSource == null) { - logger.warn { - val sourcesList = sources.map { "${it.id}:${it.sourceType}" } - val shortQuery = dsl.rawQuery?.take(80) ?: "" - "No Prometheus datasource found for org $orgId (${sources.size} sources: $sourcesList), " + - "cannot resolve __prometheus for rawQuery=$shortQuery" - } - return dsl +private fun resolvePrometheusDataSource( + dsl: QueryDsl, + orgId: Long, + dataSourceService: CustomDataSourceService, +): QueryDsl { + if (dsl.dataSource != "__prometheus") return dsl + val sources = dataSourceService.listDataSources(orgId) + val promSource = sources.firstOrNull { it.sourceType.equals("prometheus", ignoreCase = true) } + if (promSource == null) { + logger.warn { + val sourcesList = sources.map { "${it.id}:${it.sourceType}" } + val shortQuery = dsl.rawQuery?.take(QUERY_PREVIEW_LENGTH) ?: "" + "No Prometheus datasource found for org $orgId (${sources.size} sources: $sourcesList), " + + "cannot resolve __prometheus for rawQuery=$shortQuery" } - logger.debug { "Resolved __prometheus -> custom:${promSource.id} for rawQuery=${dsl.rawQuery?.take(80)}" } - return dsl.copy(dataSource = "custom:${promSource.id}") + return dsl } + val rawQueryPreview = dsl.rawQuery?.take(QUERY_PREVIEW_LENGTH) + logger.debug { "Resolved __prometheus -> custom:${promSource.id} for rawQuery=$rawQueryPreview" } + return dsl.copy(dataSource = "custom:${promSource.id}") +} - route("/v1/dashboards") { - authenticate("auth-jwt") { - // List dashboards for org - get { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val projectId = call.request.queryParameters["projectId"]?.toLongOrNull() - val dashboards = dashboardService.listDashboards(orgId, projectId, userId) - call.respond(dashboards) - } +private suspend fun io.ktor.server.routing.RoutingContext.handleListDashboards( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val projectId = call.request.queryParameters["projectId"]?.toLongOrNull() + val dashboards = dashboardService.listDashboards(orgId, projectId, userId) + call.respond(dashboards) +} - // Create dashboard - post { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleCreateDashboard( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val request = call.receive() + val dashboard = dashboardService.createDashboard(orgId, userId.toLong(), request) + call.respond(HttpStatusCode.Created, dashboard) +} - val request = call.receive() - val dashboard = dashboardService.createDashboard(orgId, userId.toLong(), request) - call.respond(HttpStatusCode.Created, dashboard) - } +private suspend fun io.ktor.server.routing.RoutingContext.handleListFolders( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val folders = dashboardService.listFolders(orgId) + call.respond(folders) +} - // Folder management (must be before /{id} to avoid "folders" matching as id) - route("/folders") { - get { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - val folders = dashboardService.listFolders(orgId) - call.respond(folders) - } - post { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - val request = call.receive() - val folder = dashboardService.createFolder(orgId, request) - call.respond(HttpStatusCode.Created, folder) - } - put("/{folderId}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@put call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - val folderId = call.parameters["folderId"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid folder ID")) - val request = call.receive() - val folder = dashboardService.updateFolder(folderId, orgId, request) - ?: return@put call.respond(HttpStatusCode.NotFound, ErrorResponse("Folder not found")) - call.respond(folder) - } - delete("/{folderId}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@delete call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - val folderId = call.parameters["folderId"]?.toLongOrNull() - ?: return@delete call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid folder ID")) - if (dashboardService.deleteFolder(folderId, orgId)) { - call.respond(HttpStatusCode.NoContent, "") - } else { - call.respond(HttpStatusCode.NotFound, ErrorResponse("Folder not found")) - } - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleCreateFolder( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val request = call.receive() + val folder = dashboardService.createFolder(orgId, request) + call.respond(HttpStatusCode.Created, folder) +} - // Get dashboard with widgets - get("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleUpdateFolder( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val folderId = call.parameters["folderId"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid folder ID")) + val request = call.receive() + val folder = dashboardService.updateFolder(folderId, orgId, request) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse("Folder not found")) + call.respond(folder) +} - val id = call.parameters["id"]?.toLongOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) +private suspend fun io.ktor.server.routing.RoutingContext.handleDeleteFolder( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val folderId = call.parameters["folderId"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid folder ID")) + if (dashboardService.deleteFolder(folderId, orgId)) { + call.respond(HttpStatusCode.NoContent, "") + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Folder not found")) + } +} - val dashboard = dashboardService.getDashboard(id, orgId, userId) - ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleGetDashboard( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val dashboard = dashboardService.getDashboard(id, orgId, userId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + call.respond(dashboard) +} - call.respond(dashboard) - } +private suspend fun io.ktor.server.routing.RoutingContext.handleToggleFavorite( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val isFavorited = dashboardService.toggleFavorite(userId, id, orgId) + call.respond(HttpStatusCode.OK, mapOf("is_favorited" to isFavorited)) +} - // Toggle favorite - post("/{id}/favorite") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleMoveDashboardToFolder( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val request = call.receive() + if (dashboardService.moveDashboardToFolder(id, orgId, request.folderId)) { + call.respond(HttpStatusCode.OK, mapOf("folder_id" to request.folderId)) + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + } +} - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) +private suspend fun io.ktor.server.routing.RoutingContext.handleUpdateDashboard( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val request = call.receive() + val updated = dashboardService.updateDashboard(id, orgId, request) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + call.respond(updated) +} - val isFavorited = dashboardService.toggleFavorite(userId, id, orgId) - call.respond(HttpStatusCode.OK, mapOf("is_favorited" to isFavorited)) - } +private suspend fun io.ktor.server.routing.RoutingContext.handleDeleteDashboard( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + if (dashboardService.deleteDashboard(id, orgId)) { + call.respond(HttpStatusCode.NoContent, "") + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + } +} - // Move dashboard to folder - put("/{id}/folder") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@put call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - - val request = call.receive() - if (dashboardService.moveDashboardToFolder(id, orgId, request.folderId)) { - call.respond(HttpStatusCode.OK, mapOf("folder_id" to request.folderId)) - } else { - call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleDashboardQuery( + queryEngine: DashboardQueryEngine, + retentionPolicyService: RetentionPolicyService, + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val dashboardScope = getDashboardScope(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + + val request = call.receive() + val demoEpochMs = call.getDemoEpochMs() + val isDemoUser = demoEpochMs != null + + // Demo users are scoped to demo projects; regular users must supply a projectId + val projectId: Long = if (isDemoUser) { + -1L // Queries against all 3 demo projects via ClickHouseQueryUtils.projectIdClause + } else { + call.request.queryParameters["projectId"]?.toLongOrNull() + ?: return call.respond( + HttpStatusCode.BadRequest, ErrorResponse("projectId query parameter required") + ) + } + if (!isDemoUser && !hasProjectAccess(orgId, projectId)) { + return call.respond(HttpStatusCode.Forbidden, ErrorResponse("Project access denied")) + } + if (!isDemoUser && dashboardScope.projectId != null && dashboardScope.projectId != projectId) { + return call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Dashboard is scoped to project ${dashboardScope.projectId}") + ) + } - // Update dashboard - put("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@put call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) + val retentionDays = if (isDemoUser) { + DEFAULT_RETENTION_DAYS + } else { + retentionPolicyService.getRetentionDaysForProject(projectId) ?: DEFAULT_RETENTION_DAYS + } + val withTimeRange = if (request.timeRange != null) { + request.queryConfig.copy(timeRange = request.timeRange) + } else { + request.queryConfig + } + val effectiveQuery = resolvePrometheusDataSource( + queryEngine.applyVariables(withTimeRange, request.variables), + orgId, + dataSourceService + ) + + try { + // Check if this is a custom data source query + if (queryEngine.isCustomDataSource(effectiveQuery.dataSource)) { + val sourceId = queryEngine.parseCustomDataSourceId(effectiveQuery.dataSource) + ?: return call.respond( + HttpStatusCode.BadRequest, ErrorResponse("Invalid custom data source ID") + ) + val source = dataSourceService.getDataSource(sourceId, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + val creds = dataSourceService.getDecryptedCredentials(sourceId, orgId) + ?: return call.respond( + HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") + ) + val sourceType = CustomDataSourceType.fromString(source.sourceType) + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_UNKNOWN_SOURCE_TYPE)) + val rawQuery = effectiveQuery.rawQuery + ?: return call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Custom data source queries require a rawQuery") + ) + val results = dataSourceExecutor.executeQuery( + sourceId, sourceType, source.host, source.port, + source.databaseName, creds, rawQuery, effectiveQuery.limit, effectiveQuery.timeRange + ) + call.respond(results) + } else { + val results = queryEngine.executeQuery(effectiveQuery, projectId, demoEpochMs, retentionDays) + call.respond(results) + } + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid query")) + } +} - val id = call.parameters["id"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) +private suspend fun executeSingleQuery( + effectiveQuery: QueryDsl, + orgId: Long, + projectId: Long, + demoEpochMs: Long?, + retentionDays: Int, + queryEngine: DashboardQueryEngine, + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +): List>? { + if (!queryEngine.isCustomDataSource(effectiveQuery.dataSource)) { + return queryEngine.executeQuery(effectiveQuery, projectId, demoEpochMs, retentionDays) + } + val sourceId = queryEngine.parseCustomDataSourceId(effectiveQuery.dataSource) ?: return null + val source = dataSourceService.getDataSource(sourceId, orgId) ?: return null + val creds = dataSourceService.getDecryptedCredentials(sourceId, orgId) ?: return null + val sourceType = CustomDataSourceType.fromString(source.sourceType) ?: return null + val rawQuery = effectiveQuery.rawQuery ?: return null + return dataSourceExecutor.executeQuery( + sourceId, sourceType, source.host, source.port, + source.databaseName, creds, rawQuery, effectiveQuery.limit, effectiveQuery.timeRange + ) +} - val request = call.receive() - val updated = dashboardService.updateDashboard(id, orgId, request) - ?: return@put call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleBatchDashboardQuery( + queryEngine: DashboardQueryEngine, + retentionPolicyService: RetentionPolicyService, + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val dashboardScope = getDashboardScope(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + + val request = call.receive() + val demoEpochMs = call.getDemoEpochMs() + val isDemoUser = demoEpochMs != null + + val projectId: Long = if (isDemoUser) { + -1L + } else { + call.request.queryParameters["projectId"]?.toLongOrNull() + ?: return call.respond( + HttpStatusCode.BadRequest, ErrorResponse("projectId query parameter required") + ) + } + if (!isDemoUser && !hasProjectAccess(orgId, projectId)) { + return call.respond(HttpStatusCode.Forbidden, ErrorResponse("Project access denied")) + } + if (!isDemoUser && dashboardScope.projectId != null && dashboardScope.projectId != projectId) { + return call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Dashboard is scoped to project ${dashboardScope.projectId}") + ) + } - call.respond(updated) - } + if (request.queries.size > MAX_QUERIES_PER_REQUEST) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Maximum 10 queries per batch")) + return + } - // Delete dashboard - delete("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@delete call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@delete call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - - if (dashboardService.deleteDashboard(id, orgId)) { - call.respond(HttpStatusCode.NoContent, "") - } else { - call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - } - } + val retentionDays = if (isDemoUser) { + DEFAULT_RETENTION_DAYS + } else { + retentionPolicyService.getRetentionDaysForProject(projectId) ?: DEFAULT_RETENTION_DAYS + } + val results = mutableMapOf>>() + + for ((index, query) in request.queries.withIndex()) { + val refId = query.refId ?: ('A' + index).toString() + val withTimeRange = if (request.timeRange != null) { + query.copy(timeRange = request.timeRange) + } else { + query + } + val effectiveQuery = resolvePrometheusDataSource( + queryEngine.applyVariables(withTimeRange, request.variables), + orgId, + dataSourceService + ) + suspendRunCatching { + executeSingleQuery( + effectiveQuery, + orgId, + projectId, + demoEpochMs, + retentionDays, + queryEngine, + dataSourceService, + dataSourceExecutor, + )?.let { results[refId] = it } + }.getOrElse { e -> + logger.warn(e) { "Batch query $refId failed" } + results[refId] = emptyList() + } + } - // Execute widget query - post("/{id}/query") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - val dashboardScope = getDashboardScope(id, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - - val request = call.receive() - val demoEpochMs = call.getDemoEpochMs() - val isDemoUser = demoEpochMs != null - - // Demo users are scoped to demo projects; regular users must supply a projectId - val projectId: Long = if (isDemoUser) { - -1L // Queries against all 3 demo projects via ClickHouseQueryUtils.projectIdClause - } else { - call.request.queryParameters["projectId"]?.toLongOrNull() - ?: return@post call.respond( - HttpStatusCode.BadRequest, ErrorResponse("projectId query parameter required") - ) - } - if (!isDemoUser && !hasProjectAccess(orgId, projectId)) { - return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("Project access denied")) - } - if (!isDemoUser && dashboardScope.projectId != null && dashboardScope.projectId != projectId) { - return@post call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Dashboard is scoped to project ${dashboardScope.projectId}") - ) - } - - val retentionDays = - if (isDemoUser) 90 else retentionPolicyService.getRetentionDaysForProject(projectId) ?: 90 - val withTimeRange = if (request.timeRange != null) { - request.queryConfig.copy(timeRange = request.timeRange) - } else { - request.queryConfig - } - val effectiveQuery = resolvePrometheusDataSource( - queryEngine.applyVariables(withTimeRange, request.variables), - orgId - ) + call.respond(BatchQueryResult(results)) +} - try { - // Check if this is a custom data source query - if (queryEngine.isCustomDataSource(effectiveQuery.dataSource)) { - val sourceId = queryEngine.parseCustomDataSourceId(effectiveQuery.dataSource) - ?: return@post call.respond( - HttpStatusCode.BadRequest, ErrorResponse("Invalid custom data source ID") - ) - - val source = dataSourceService.getDataSource(sourceId, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - - val creds = dataSourceService.getDecryptedCredentials(sourceId, orgId) - ?: return@post call.respond( - HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") - ) - - val sourceType = CustomDataSourceType.fromString(source.sourceType) - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Unknown source type")) - - val rawQuery = effectiveQuery.rawQuery - ?: return@post call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Custom data source queries require a rawQuery") - ) - - val results = dataSourceExecutor.executeQuery( - sourceId, sourceType, source.host, source.port, - source.databaseName, creds, rawQuery, effectiveQuery.limit, effectiveQuery.timeRange - ) - call.respond(results) - } else { - val results = queryEngine.executeQuery(effectiveQuery, projectId, demoEpochMs, retentionDays) - call.respond(results) - } - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid query")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleVariablesResolve( + dashboardService: CustomDashboardService, + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + getDashboardScope(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + + val dashboard = dashboardService.getDashboard(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + + val variables = dashboard.variables + val currentValues = call.receive>() + + // Find the org's Prometheus datasource + val promSource = dataSourceService.listDataSources(orgId) + .firstOrNull { it.sourceType.equals("prometheus", ignoreCase = true) } + + val resolved = mutableMapOf>() + for (v in variables) { + val query = v.query ?: continue + if (!query.startsWith("label_values(")) continue + if (promSource == null) continue + + // Substitute variable references in the query + var substituted = query + for ((name, value) in currentValues) { + substituted = substituted + .replace("\${$name}", value) + .replace("\$$name", value) + } - // Execute batch query (multiple queries keyed by refId) - post("/{id}/query/batch") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - val dashboardScope = getDashboardScope(id, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - - val request = call.receive() - val demoEpochMs = call.getDemoEpochMs() - val isDemoUser = demoEpochMs != null - - val projectId: Long = if (isDemoUser) { - -1L - } else { - call.request.queryParameters["projectId"]?.toLongOrNull() - ?: return@post call.respond( - HttpStatusCode.BadRequest, ErrorResponse("projectId query parameter required") - ) - } - if (!isDemoUser && !hasProjectAccess(orgId, projectId)) { - return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("Project access denied")) - } - if (!isDemoUser && dashboardScope.projectId != null && dashboardScope.projectId != projectId) { - return@post call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Dashboard is scoped to project ${dashboardScope.projectId}") - ) - } - - if (request.queries.size > 10) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse("Maximum 10 queries per batch")) - return@post - } - - val retentionDays = - if (isDemoUser) 90 else retentionPolicyService.getRetentionDaysForProject(projectId) ?: 90 - val results = mutableMapOf>>() - - for ((index, query) in request.queries.withIndex()) { - val refId = query.refId ?: ('A' + index).toString() - val withTimeRange = if (request.timeRange != null) { - query.copy(timeRange = request.timeRange) - } else { - query - } - val effectiveQuery = resolvePrometheusDataSource( - queryEngine.applyVariables(withTimeRange, request.variables), - orgId - ) - - suspendRunCatching { - if (queryEngine.isCustomDataSource(effectiveQuery.dataSource)) { - val sourceId = queryEngine.parseCustomDataSourceId(effectiveQuery.dataSource) - ?: continue - val source = dataSourceService.getDataSource(sourceId, orgId) ?: continue - val creds = dataSourceService.getDecryptedCredentials(sourceId, orgId) ?: continue - val sourceType = CustomDataSourceType.fromString(source.sourceType) ?: continue - val rawQuery = effectiveQuery.rawQuery ?: continue - - results[refId] = dataSourceExecutor.executeQuery( - sourceId, sourceType, source.host, source.port, - source.databaseName, creds, rawQuery, effectiveQuery.limit, effectiveQuery.timeRange - ) - } else { - results[refId] = queryEngine.executeQuery( - effectiveQuery, - projectId, - demoEpochMs, - retentionDays - ) - } - }.getOrElse { e -> - logger.warn(e) { "Batch query $refId failed" } - results[refId] = emptyList() - } - } - - call.respond(BatchQueryResult(results)) - } + val creds = dataSourceService.getDecryptedCredentials(promSource.id, orgId) ?: continue + val sourceType = CustomDataSourceType.fromString(promSource.sourceType) + ?: CustomDataSourceType.PROMETHEUS + val options = dataSourceExecutor.executeLabelValuesQuery( + sourceType, + promSource.host, + promSource.port, + creds, + substituted + ) + if (options.isNotEmpty()) { + resolved[v.name] = options + } + } - // Resolve variable options (e.g., Grafana label_values queries) - post("/{id}/variables/resolve") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - getDashboardScope(id, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - - val dashboard = dashboardService.getDashboard(id, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - - val variables = dashboard.variables - val currentValues = call.receive>() - - // Find the org's Prometheus datasource - val promSource = dataSourceService.listDataSources(orgId) - .firstOrNull { it.sourceType.equals("prometheus", ignoreCase = true) } - - val resolved = mutableMapOf>() - for (v in variables) { - val query = v.query ?: continue - if (!query.startsWith("label_values(")) continue - if (promSource == null) continue - - // Substitute variable references in the query - var substituted = query - for ((name, value) in currentValues) { - substituted = substituted - .replace("\${$name}", value) - .replace("\$$name", value) - } - - val creds = dataSourceService.getDecryptedCredentials(promSource.id, orgId) ?: continue - val sourceType = com.moneat.dashboards.models.CustomDataSourceType.fromString(promSource.sourceType) - ?: com.moneat.dashboards.models.CustomDataSourceType.PROMETHEUS - val options = dataSourceExecutor.executeLabelValuesQuery( - sourceType, - promSource.host, - promSource.port, - creds, - substituted - ) - if (options.isNotEmpty()) { - resolved[v.name] = options - } - } - - call.respond(resolved) - } + call.respond(resolved) +} - // Import dashboard from DataDog/Grafana JSON - post("/import") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val request = call.receive() - val jsonObj = suspendRunCatching { - json.parseToJsonElement(request.json) as JsonObject - }.getOrElse { _ -> - return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid JSON")) - } - - val translator = when (request.format.lowercase()) { - "datadog" -> dataDogTranslator - "grafana" -> grafanaTranslator - else -> return@post call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Unsupported format: ${request.format}. Use 'datadog' or 'grafana'") - ) - } - - suspendRunCatching { - val importResult = translator.import(jsonObj) - val createRequest = CreateDashboardRequest( - title = importResult.dashboard.title, - description = importResult.dashboard.description, - layoutType = importResult.dashboard.layoutType, - variables = importResult.variables, - widgets = importResult.dashboard.widgets.map { w -> - CreateWidgetRequest( - title = w.title, - widgetType = w.widgetType, - gridX = w.gridX, - gridY = w.gridY, - gridW = w.gridW, - gridH = w.gridH, - queryConfigs = w.queryConfigs, - displayConfig = w.displayConfig, - sortOrder = w.sortOrder - ) - } - ) - val created = dashboardService.createDashboard(orgId, userId.toLong(), createRequest) - call.respond( - HttpStatusCode.Created, - DashboardImportResult(created, importResult.warnings, importResult.variables) - ) - }.getOrElse { e -> - logger.error(e) { "Failed to import dashboard" } - call.respond(HttpStatusCode.BadRequest, ErrorResponse("Failed to import: ${e.message}")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleImportDashboard( + dashboardService: CustomDashboardService, + dataDogTranslator: DataDogTranslator, + grafanaTranslator: GrafanaTranslator, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + + val request = call.receive() + val jsonParseResult = suspendRunCatching { + json.parseToJsonElement(request.json) as JsonObject + } + if (jsonParseResult.isFailure) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid JSON")) + return + } + val jsonObj = jsonParseResult.getOrThrow() + + val translator: DashboardTranslator = when (request.format.lowercase()) { + "datadog" -> dataDogTranslator + "grafana" -> grafanaTranslator + else -> return call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Unsupported format: ${request.format}. Use 'datadog' or 'grafana'") + ) + } - // Export dashboard - get("/{id}/export/{format}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - - val format = call.parameters["format"] - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Format required")) - - val dashboard = dashboardService.getDashboard(id, orgId) - ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Dashboard not found")) - - when (format.lowercase()) { - "moneat" -> call.respond(dashboard) - "datadog" -> call.respond(dataDogTranslator.export(dashboard)) - "grafana" -> call.respond(grafanaTranslator.export(dashboard)) - else -> call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Unsupported format: $format. Use 'moneat', 'datadog', or 'grafana'") - ) - } + suspendRunCatching { + val importResult = translator.import(jsonObj) + val createRequest = CreateDashboardRequest( + title = importResult.dashboard.title, + description = importResult.dashboard.description, + layoutType = importResult.dashboard.layoutType, + variables = importResult.variables, + widgets = importResult.dashboard.widgets.map { w -> + CreateWidgetRequest( + title = w.title, + widgetType = w.widgetType, + gridX = w.gridX, + gridY = w.gridY, + gridW = w.gridW, + gridH = w.gridH, + queryConfigs = w.queryConfigs, + displayConfig = w.displayConfig, + sortOrder = w.sortOrder + ) } + ) + val created = dashboardService.createDashboard(orgId, userId.toLong(), createRequest) + call.respond( + HttpStatusCode.Created, + DashboardImportResult(created, importResult.warnings, importResult.variables) + ) + }.getOrElse { e -> + logger.error(e) { "Failed to import dashboard" } + call.respond(HttpStatusCode.BadRequest, ErrorResponse("Failed to import: ${e.message}")) + } +} - // Dashboard alert CRUD routes - get("/{id}/alerts") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) +private suspend fun io.ktor.server.routing.RoutingContext.handleExportDashboard( + dashboardService: CustomDashboardService, + dataDogTranslator: DataDogTranslator, + grafanaTranslator: GrafanaTranslator, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val format = call.parameters["format"] + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Format required")) + val dashboard = dashboardService.getDashboard(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DASHBOARD_NOT_FOUND)) + + when (format.lowercase()) { + "moneat" -> call.respond(dashboard) + "datadog" -> call.respond(dataDogTranslator.export(dashboard)) + "grafana" -> call.respond(grafanaTranslator.export(dashboard)) + else -> call.respond( + HttpStatusCode.BadRequest, + ErrorResponse("Unsupported format: $format. Use 'moneat', 'datadog', or 'grafana'") + ) + } +} - val id = call.parameters["id"]?.toLongOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) +private suspend fun io.ktor.server.routing.RoutingContext.handleListAlerts( + dashboardAlertService: DashboardAlertService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + call.respond(dashboardAlertService.listAlerts(id, orgId)) +} - call.respond(dashboardAlertService.listAlerts(id, orgId)) - } +private suspend fun io.ktor.server.routing.RoutingContext.handleCreateAlert( + dashboardAlertService: DashboardAlertService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val request = call.receive() + try { + val alert = dashboardAlertService.createAlert(id, orgId, userId.toLong(), request) + call.respond(HttpStatusCode.Created, alert) + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) + } +} - post("/{id}/alerts") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - - val request = call.receive() - try { - val alert = dashboardAlertService.createAlert(id, orgId, userId.toLong(), request) - call.respond(HttpStatusCode.Created, alert) - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleUpdateAlert( + dashboardAlertService: DashboardAlertService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val alertId = call.parameters["alertId"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid alert ID")) + val request = call.receive() + try { + val updated = dashboardAlertService.updateAlert(alertId, id, orgId, request) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse("Alert not found")) + call.respond(updated) + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) + } +} - put("/{id}/alerts/{alertId}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@put call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - val alertId = call.parameters["alertId"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid alert ID")) - - val request = call.receive() - try { - val updated = dashboardAlertService.updateAlert(alertId, id, orgId, request) - ?: return@put call.respond(HttpStatusCode.NotFound, ErrorResponse("Alert not found")) - call.respond(updated) - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleDeleteAlert( + dashboardAlertService: DashboardAlertService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DASHBOARD_ID)) + val alertId = call.parameters["alertId"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid alert ID")) + if (dashboardAlertService.deleteAlert(alertId, id, orgId)) { + call.respond(HttpStatusCode.NoContent, "") + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse("Alert not found")) + } +} - delete("/{id}/alerts/{alertId}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@delete call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@delete call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid dashboard ID")) - val alertId = call.parameters["alertId"]?.toLongOrNull() - ?: return@delete call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid alert ID")) - - if (dashboardAlertService.deleteAlert(alertId, id, orgId)) { - call.respond(HttpStatusCode.NoContent, "") - } else { - call.respond(HttpStatusCode.NotFound, ErrorResponse("Alert not found")) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleGetAvailableDataSources( + dataSourceService: CustomDataSourceService, + queryEngine: DashboardQueryEngine, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + if (orgId != null) { + val customSources = dataSourceService.listDataSources(orgId) + call.respond(queryEngine.getDataSources(customSources)) + } else { + call.respond(queryEngine.getDataSources()) + } +} - // List available data sources and fields (built-in + custom) - get("/datasources") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - if (orgId != null) { - val customSources = dataSourceService.listDataSources(orgId) - call.respond(queryEngine.getDataSources(customSources)) - } else { - call.respond(queryEngine.getDataSources()) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleSearch( + dashboardService: CustomDashboardService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val query = call.request.queryParameters["q"]?.trim().orEmpty() + val result = dashboardService.search(orgId, userId, query) + call.respond(result) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleListCustomDataSources( + dataSourceService: CustomDataSourceService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + call.respond(dataSourceService.listDataSources(orgId)) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleCreateCustomDataSource( + dataSourceService: CustomDataSourceService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val request = call.receive() + try { + val source = dataSourceService.createDataSource(orgId, userId.toLong(), request) + call.respond(HttpStatusCode.Created, source) + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) + } +} - // Get default dashboard templates - get("/templates") { - call.respond(dashboardService.getDefaultDashboardTemplates()) +private suspend fun io.ktor.server.routing.RoutingContext.handleTestConnection( + dataSourceExecutor: CustomDataSourceExecutor, +) { + val request = call.receive() + suspendRunCatching { + val result = dataSourceExecutor.testConnection(request) + call.respond(result) + }.getOrElse { e -> + call.respond(TestConnectionResult(false, "Test failed: ${e.message}")) + } +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleGetCustomDataSource( + dataSourceService: CustomDataSourceService, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DATA_SOURCE_ID)) + val source = dataSourceService.getDataSource(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + call.respond(source) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleUpdateCustomDataSource( + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DATA_SOURCE_ID)) + val request = call.receive() + val updated = dataSourceService.updateDataSource(id, orgId, request) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + + // Invalidate any cached connection pool + dataSourceExecutor.closePool(id) + call.respond(updated) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleDeleteCustomDataSource( + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond( + HttpStatusCode.BadRequest, + ErrorResponse(ERR_INVALID_DATA_SOURCE_ID) + ) + if (dataSourceService.deleteDataSource(id, orgId)) { + dataSourceExecutor.closePool(id) + call.respond(HttpStatusCode.NoContent, "") + } else { + call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + } +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleGetDataSourceSchema( + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DATA_SOURCE_ID)) + val source = dataSourceService.getDataSource(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + val creds = dataSourceService.getDecryptedCredentials(id, orgId) + ?: return call.respond( + HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") + ) + val sourceType = CustomDataSourceType.fromString(source.sourceType) + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_UNKNOWN_SOURCE_TYPE)) + val schema = dataSourceExecutor.getSchema( + sourceType, + source.host, + source.port, + source.databaseName, + creds + ) + call.respond(schema) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleCustomDataSourceQuery( + dataSourceService: CustomDataSourceService, + dataSourceExecutor: CustomDataSourceExecutor, +) { + val principal = call.principal() + val userId = principal!!.payload.getClaim("userId").asInt() + val orgId = getOrgIdForUser(userId) + ?: return call.respond(HttpStatusCode.Forbidden, ErrorResponse(ERR_NO_ORGANIZATION)) + val id = call.parameters["id"]?.toLongOrNull() + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_INVALID_DATA_SOURCE_ID)) + val request = call.receive() + val source = dataSourceService.getDataSource(id, orgId) + ?: return call.respond(HttpStatusCode.NotFound, ErrorResponse(ERR_DATA_SOURCE_NOT_FOUND)) + val creds = dataSourceService.getDecryptedCredentials(id, orgId) + ?: return call.respond( + HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") + ) + val sourceType = CustomDataSourceType.fromString(source.sourceType) + ?: return call.respond(HttpStatusCode.BadRequest, ErrorResponse(ERR_UNKNOWN_SOURCE_TYPE)) + try { + val results = dataSourceExecutor.executeQuery( + id, sourceType, source.host, source.port, + source.databaseName, creds, request.query, request.limit, request.timeRange + ) + call.respond(results) + } catch (e: IllegalArgumentException) { + call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid query")) + } +} + +data class DashboardTranslators( + val dataDog: DataDogTranslator = DataDogTranslator(), + val grafana: GrafanaTranslator = GrafanaTranslator(), +) + +fun Route.customDashboardRoutes( + dashboardService: CustomDashboardService = GlobalContext.get().get(), + queryEngine: DashboardQueryEngine = GlobalContext.get().get(), + retentionPolicyService: RetentionPolicyService = GlobalContext.get().get(), + translators: DashboardTranslators = DashboardTranslators(), + dataSourceService: CustomDataSourceService = GlobalContext.get().get(), + dataSourceExecutor: CustomDataSourceExecutor = GlobalContext.get().get(), + dashboardAlertService: DashboardAlertService = GlobalContext.get().get(), +) { + route("/v1/dashboards") { + authenticate(AUTH_JWT) { + get { handleListDashboards(dashboardService) } + post { handleCreateDashboard(dashboardService) } + route("/folders") { + get { handleListFolders(dashboardService) } + post { handleCreateFolder(dashboardService) } + put("/{folderId}") { handleUpdateFolder(dashboardService) } + delete("/{folderId}") { handleDeleteFolder(dashboardService) } } + get("/{id}") { handleGetDashboard(dashboardService) } + post("/{id}/favorite") { handleToggleFavorite(dashboardService) } + put("/{id}/folder") { handleMoveDashboardToFolder(dashboardService) } + put("/{id}") { handleUpdateDashboard(dashboardService) } + delete("/{id}") { handleDeleteDashboard(dashboardService) } + post("/{id}/query") { + handleDashboardQuery(queryEngine, retentionPolicyService, dataSourceService, dataSourceExecutor) + } + post("/{id}/query/batch") { + handleBatchDashboardQuery(queryEngine, retentionPolicyService, dataSourceService, dataSourceExecutor) + } + post("/{id}/variables/resolve") { + handleVariablesResolve(dashboardService, dataSourceService, dataSourceExecutor) + } + post("/import") { handleImportDashboard(dashboardService, translators.dataDog, translators.grafana) } + get("/{id}/export/{format}") { + handleExportDashboard(dashboardService, translators.dataDog, translators.grafana) + } + get("/{id}/alerts") { handleListAlerts(dashboardAlertService) } + post("/{id}/alerts") { handleCreateAlert(dashboardAlertService) } + put("/{id}/alerts/{alertId}") { handleUpdateAlert(dashboardAlertService) } + delete("/{id}/alerts/{alertId}") { handleDeleteAlert(dashboardAlertService) } + get("/datasources") { handleGetAvailableDataSources(dataSourceService, queryEngine) } + get("/templates") { call.respond(dashboardService.getDefaultDashboardTemplates()) } } } // Global search (dashboards, projects) route("/v1/search") { - authenticate("auth-jwt") { - get { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - val query = call.request.queryParameters["q"]?.trim().orEmpty() - val result = dashboardService.search(orgId, userId, query) - call.respond(result) - } + authenticate(AUTH_JWT) { + get { handleSearch(dashboardService) } } } // Custom data source management routes route("/v1") { route("/datasources") { - authenticate("auth-jwt") { - // List custom data sources for org - get { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - call.respond(dataSourceService.listDataSources(orgId)) - } - - // Create custom data source - post { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val request = call.receive() - try { - val source = dataSourceService.createDataSource(orgId, userId.toLong(), request) - call.respond(HttpStatusCode.Created, source) - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid request")) - } - } - + authenticate(AUTH_JWT) { + get { handleListCustomDataSources(dataSourceService) } + post { handleCreateCustomDataSource(dataSourceService) } // Test connection (without saving) — must be before /{id} routes - post("/test") { - val request = call.receive() - suspendRunCatching { - val result = dataSourceExecutor.testConnection(request) - call.respond(result) - }.getOrElse { e -> - call.respond(TestConnectionResult(false, "Test failed: ${e.message}")) - } - } - - // Get custom data source - get("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid data source ID")) - - val source = dataSourceService.getDataSource(id, orgId) - ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - - call.respond(source) - } - - // Update custom data source - put("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@put call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid data source ID")) - - val request = call.receive() - val updated = dataSourceService.updateDataSource(id, orgId, request) - ?: return@put call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - - // Invalidate any cached connection pool - dataSourceExecutor.closePool(id) - call.respond(updated) - } - - // Delete custom data source - delete("/{id}") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@delete call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@delete call.respond( - HttpStatusCode.BadRequest, - ErrorResponse("Invalid data source ID") - ) - - if (dataSourceService.deleteDataSource(id, orgId)) { - dataSourceExecutor.closePool(id) - call.respond(HttpStatusCode.NoContent, "") - } else { - call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - } - } - - // Get schema for a custom data source - get("/{id}/schema") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@get call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid data source ID")) - - val source = dataSourceService.getDataSource(id, orgId) - ?: return@get call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - - val creds = dataSourceService.getDecryptedCredentials(id, orgId) - ?: return@get call.respond( - HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") - ) - - val sourceType = CustomDataSourceType.fromString(source.sourceType) - ?: return@get call.respond(HttpStatusCode.BadRequest, ErrorResponse("Unknown source type")) - - val schema = dataSourceExecutor.getSchema( - sourceType, - source.host, - source.port, - source.databaseName, - creds - ) - call.respond(schema) - } - - // Execute query against custom data source - post("/{id}/query") { - val principal = call.principal() - val userId = principal!!.payload.getClaim("userId").asInt() - val orgId = getOrgIdForUser(userId) - ?: return@post call.respond(HttpStatusCode.Forbidden, ErrorResponse("No organization found")) - - val id = call.parameters["id"]?.toLongOrNull() - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid data source ID")) - - val request = call.receive() - val source = dataSourceService.getDataSource(id, orgId) - ?: return@post call.respond(HttpStatusCode.NotFound, ErrorResponse("Data source not found")) - - val creds = dataSourceService.getDecryptedCredentials(id, orgId) - ?: return@post call.respond( - HttpStatusCode.InternalServerError, ErrorResponse("Failed to decrypt credentials") - ) - - val sourceType = CustomDataSourceType.fromString(source.sourceType) - ?: return@post call.respond(HttpStatusCode.BadRequest, ErrorResponse("Unknown source type")) - - try { - val results = dataSourceExecutor.executeQuery( - id, sourceType, source.host, source.port, - source.databaseName, creds, request.query, request.limit, request.timeRange - ) - call.respond(results) - } catch (e: IllegalArgumentException) { - call.respond(HttpStatusCode.BadRequest, ErrorResponse(e.message ?: "Invalid query")) - } - } + post("/test") { handleTestConnection(dataSourceExecutor) } + get("/{id}") { handleGetCustomDataSource(dataSourceService) } + put("/{id}") { handleUpdateCustomDataSource(dataSourceService, dataSourceExecutor) } + delete("/{id}") { handleDeleteCustomDataSource(dataSourceService, dataSourceExecutor) } + get("/{id}/schema") { handleGetDataSourceSchema(dataSourceService, dataSourceExecutor) } + post("/{id}/query") { handleCustomDataSourceQuery(dataSourceService, dataSourceExecutor) } } } } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardAlertService.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardAlertService.kt index b75e3ade1..0ebbea5c8 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardAlertService.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardAlertService.kt @@ -85,6 +85,8 @@ class DashboardAlertService( companion object { const val EVALUATION_INTERVAL_SECONDS = 60 const val MIN_ALERT_INTERVAL_MINUTES = 15 + private const val DEFAULT_RETENTION_DAYS = 90 + private const val MILLIS_PER_SECOND = 1000 } fun start(scope: CoroutineScope) { @@ -288,7 +290,7 @@ class DashboardAlertService( val queryDsl = queryConfigs.getOrNull(queryIndex) ?: return val projectId = alert.projectId ?: return - val retentionDays = retentionPolicyService.getRetentionDaysForProject(projectId) ?: 90 + val retentionDays = retentionPolicyService.getRetentionDaysForProject(projectId) ?: DEFAULT_RETENTION_DAYS val results = suspendRunCatching { queryEngine.executeQuery(queryDsl, projectId, null, retentionDays) @@ -430,8 +432,8 @@ class DashboardAlertService( suspendRunCatching { val redis = RedisConfig.sync() val existing = redis.get(pendingKey)?.toLongOrNull() - if (existing != null) return Instant.fromEpochMilliseconds(existing * 1000) - redis.set(pendingKey, (now.toEpochMilliseconds() / 1000).toString()) + if (existing != null) return Instant.fromEpochMilliseconds(existing * MILLIS_PER_SECOND) + redis.set(pendingKey, (now.toEpochMilliseconds() / MILLIS_PER_SECOND).toString()) return now } } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardQueryEngine.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardQueryEngine.kt index d0763da27..66342adc4 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardQueryEngine.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/DashboardQueryEngine.kt @@ -73,16 +73,33 @@ class DashboardQueryEngine { private val TIME_RANGE_REGEX = Regex("""^now-(\d+)([smhdwMy])$""") + private const val MILLIS_PER_SECOND = 1000L + private const val MILLIS_PER_MINUTE = 60_000L + private const val MILLIS_PER_HOUR = 3_600_000L + private const val MILLIS_PER_DAY = 86_400_000L + private const val MILLIS_PER_WEEK = 604_800_000L + private const val MILLIS_PER_MONTH = 2_592_000_000L + private const val MILLIS_PER_YEAR = 31_536_000_000L + private const val MINUTES_IN_1_HOUR = 60L + private const val MINUTES_IN_6_HOURS = 360L + private const val MINUTES_IN_1_DAY = 1440L + private const val MINUTES_IN_1_WEEK = 10080L + private const val MINUTES_IN_30_DAYS = 43200L + private const val MINUTES_IN_90_DAYS = 129600L + private const val MAX_QUERY_RESULT_LIMIT = 10000 + private const val EPOCH_MS_TO_SECONDS_DIVISOR = 1000.0 + private const val DATETIME64_MILLIS_PRECISION = 3 + fun resolveTimeInterval(from: String, to: String): String { val rangeMs = parseRelativeTime(to) - parseRelativeTime(from) - val rangeMinutes = rangeMs / 60_000 + val rangeMinutes = rangeMs / MILLIS_PER_MINUTE return when { - rangeMinutes <= 60 -> "1 MINUTE" - rangeMinutes <= 360 -> "5 MINUTE" - rangeMinutes <= 1440 -> "15 MINUTE" - rangeMinutes <= 10080 -> "1 HOUR" - rangeMinutes <= 43200 -> "4 HOUR" - rangeMinutes <= 129600 -> "1 DAY" + rangeMinutes <= MINUTES_IN_1_HOUR -> "1 MINUTE" + rangeMinutes <= MINUTES_IN_6_HOURS -> "5 MINUTE" + rangeMinutes <= MINUTES_IN_1_DAY -> "15 MINUTE" + rangeMinutes <= MINUTES_IN_1_WEEK -> "1 HOUR" + rangeMinutes <= MINUTES_IN_30_DAYS -> "4 HOUR" + rangeMinutes <= MINUTES_IN_90_DAYS -> "1 DAY" else -> "1 WEEK" } } @@ -93,13 +110,13 @@ class DashboardQueryEngine { val amount = match.groupValues[1].toLong() val unit = match.groupValues[2] val ms = when (unit) { - "s" -> amount * 1000 - "m" -> amount * 60_000 - "h" -> amount * 3_600_000 - "d" -> amount * 86_400_000 - "w" -> amount * 604_800_000 - "M" -> amount * 2_592_000_000 - "y" -> amount * 31_536_000_000 + "s" -> amount * MILLIS_PER_SECOND + "m" -> amount * MILLIS_PER_MINUTE + "h" -> amount * MILLIS_PER_HOUR + "d" -> amount * MILLIS_PER_DAY + "w" -> amount * MILLIS_PER_WEEK + "M" -> amount * MILLIS_PER_MONTH + "y" -> amount * MILLIS_PER_YEAR else -> 0 } return System.currentTimeMillis() - ms @@ -180,7 +197,7 @@ class DashboardQueryEngine { if (orderByClause.isNotEmpty()) { append(" ORDER BY $orderByClause") } - append(" LIMIT ${dsl.limit.coerceIn(1, 10000)}") + append(" LIMIT ${dsl.limit.coerceIn(1, MAX_QUERY_RESULT_LIMIT)}") append(" FORMAT JSONEachRow") } } @@ -252,7 +269,7 @@ class DashboardQueryEngine { ): List { val clauses = mutableListOf() val nowExpr = if (demoEpochMs != null) { - "toDateTime64(${demoEpochMs / 1000.0}, 3)" + "toDateTime64(${demoEpochMs / EPOCH_MS_TO_SECONDS_DIVISOR}, $DATETIME64_MILLIS_PRECISION)" } else { "now()" } @@ -344,14 +361,17 @@ class DashboardQueryEngine { } val sql = buildQuery(dsl, projectId, demoEpochMs, retentionDays) - logger.debug { "Executing dashboard query: ${sql.take(500)}" } + logger.debug { + "Executing dashboard query dataSource=${dsl.dataSource} " + + "metrics=${dsl.metrics.size} filters=${dsl.filters.size} groupBy=${dsl.groupBy.size}" + } return suspendRunCatching { val response = ClickHouseClient.execute(sql) val body = response.bodyAsText() if (response.isClickHouseError(body)) { - logger.error { "ClickHouse error: ${body.take(400)}" } + logger.error { "ClickHouse returned an error body (length=${body.length})" } return emptyList() } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/BigQueryHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/BigQueryHandler.kt index 8bb09b40e..ab024fe71 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/BigQueryHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/BigQueryHandler.kt @@ -40,6 +40,10 @@ private val logger = KotlinLogging.logger {} */ class BigQueryHandler : DataSourceHandler { + companion object { + private const val BIGQUERY_MAX_RESULTS = 10_000L + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { val projectId = request.projectId ?: request.host.ifBlank { null } ?: return TestConnectionResult(false, "Project ID is required") @@ -77,7 +81,7 @@ class BigQueryHandler : DataSourceHandler { return suspendRunCatching { val bigQuery = createBigQueryClient(projectId, serviceAccountJson) val config = QueryJobConfiguration.newBuilder(query) - .setMaxResults(limit.toLong().coerceIn(1, 10000)) + .setMaxResults(limit.toLong().coerceIn(1, BIGQUERY_MAX_RESULTS)) .build() val results = bigQuery.query(config) val schema = results.schema ?: return emptyList() diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/CloudWatchHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/CloudWatchHandler.kt index 57fc5d0cd..366d2f28d 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/CloudWatchHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/CloudWatchHandler.kt @@ -53,6 +53,15 @@ private val json = Json { ignoreUnknownKeys = true } */ class CloudWatchHandler : DataSourceHandler { + companion object { + private const val CLOUDWATCH_DEFAULT_PERIOD_SECONDS = 300 + private const val CLOUDWATCH_MAX_DATAPOINTS = 100_800 + private const val MILLIS_PER_SECOND = 1_000L + private const val DAYS_PER_WEEK = 7L + private const val DAYS_PER_MONTH = 30L + private const val DAYS_PER_YEAR = 365L + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { val region = request.region ?: "us-east-1" val accessKey = request.accessKeyId @@ -77,7 +86,7 @@ class CloudWatchHandler : DataSourceHandler { namespace = "AWS/EC2" metricName = "CPUUtilization" } - period = 300 + period = CLOUDWATCH_DEFAULT_PERIOD_SECONDS stat = "Average" } returnData = true @@ -136,7 +145,7 @@ class CloudWatchHandler : DataSourceHandler { val endTime = resolveTime(timeRange?.to ?: "now", now) return suspendRunCatching { - val normalizedLimit = limit.coerceIn(1, 100800) + val normalizedLimit = limit.coerceIn(1, CLOUDWATCH_MAX_DATAPOINTS) createClient(region, accessKey, secretKey).use { client -> val startJava = java.time.Instant.ofEpochMilli(startTime.toEpochMilliseconds()) val endJava = java.time.Instant.ofEpochMilli(endTime.toEpochMilliseconds()) @@ -153,7 +162,7 @@ class CloudWatchHandler : DataSourceHandler { this.metricName = metricName this.dimensions = dimensions } - period = 300 + period = CLOUDWATCH_DEFAULT_PERIOD_SECONDS stat = "Average" } returnData = true @@ -169,7 +178,7 @@ class CloudWatchHandler : DataSourceHandler { val values = result.values ?: emptyList() for ((i, ts) in timestamps.withIndex()) { val value = values.getOrNull(i) ?: continue - val tsMs = ts.epochSeconds * 1000 + val tsMs = ts.epochSeconds * MILLIS_PER_SECOND rows.add( mapOf( "time_bucket" to JsonPrimitive(tsMs), @@ -210,15 +219,17 @@ class CloudWatchHandler : DataSourceHandler { private fun resolveTime(expr: String, now: Instant): Instant { if (expr == "now") return now val match = Regex("""^now-(\d+)([smhdwMy])$""").matchEntire(expr) ?: return now - val amount = match.groupValues[1].toLong() + val amount = match.groupValues[1].toLongOrNull() ?: return now + fun safeDays(multiplier: Long): Long? = + if (amount <= Long.MAX_VALUE / multiplier) amount * multiplier else null val offset = when (match.groupValues[2]) { "s" -> Duration.parse("${amount}s") "m" -> Duration.parse("${amount}m") "h" -> Duration.parse("${amount}h") "d" -> Duration.parse("${amount}d") - "w" -> Duration.parse("${amount * 7}d") - "M" -> Duration.parse("${amount * 30}d") - "y" -> Duration.parse("${amount * 365}d") + "w" -> Duration.parse("${safeDays(DAYS_PER_WEEK) ?: return now}d") + "M" -> Duration.parse("${safeDays(DAYS_PER_MONTH) ?: return now}d") + "y" -> Duration.parse("${safeDays(DAYS_PER_YEAR) ?: return now}d") else -> Duration.ZERO } return now - offset diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/ElasticsearchHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/ElasticsearchHandler.kt index 2cc480d86..a739dfbbb 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/ElasticsearchHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/ElasticsearchHandler.kt @@ -49,9 +49,15 @@ private val logger = KotlinLogging.logger {} */ class ElasticsearchHandler : HttpApiHandler() { + companion object { + private const val ELASTICSEARCH_DEFAULT_PORT = 9200 + private const val ELASTICSEARCH_MAX_SIZE = 10_000 + private const val ELASTICSEARCH_SCHEMA_LIMIT = 100 + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { return suspendRunCatching { - val baseUrl = buildUrl(request.host, request.port ?: 9200) + val baseUrl = buildUrl(request.host, request.port ?: ELASTICSEARCH_DEFAULT_PORT) val response = httpClient.get("$baseUrl/_cluster/health") { applyAuth(request.apiKey, request.username, request.password) } @@ -76,7 +82,7 @@ class ElasticsearchHandler : HttpApiHandler() { limit: Int, timeRange: TimeRangeDef?, ): List> { - val baseUrl = buildUrl(host, port ?: 9200) + val baseUrl = buildUrl(host, port ?: ELASTICSEARCH_DEFAULT_PORT) val index = databaseName?.ifBlank { null } ?: "_all" val path = if (index == "_all") "/_search" else "/$index/_search" @@ -85,7 +91,7 @@ class ElasticsearchHandler : HttpApiHandler() { "query" to JsonObject( mapOf("query_string" to JsonObject(mapOf("query" to JsonPrimitive(query)))) ), - "size" to JsonPrimitive(limit.coerceIn(1, 10000)) + "size" to JsonPrimitive(limit.coerceIn(1, ELASTICSEARCH_MAX_SIZE)) ) ) val queryBody = suspendRunCatching { @@ -93,12 +99,12 @@ class ElasticsearchHandler : HttpApiHandler() { if (parsed is JsonObject) { parsed } else { - JsonObject(mapOf("query" to parsed, "size" to JsonPrimitive(limit.coerceIn(1, 10000)))) + JsonObject(mapOf("query" to parsed, "size" to JsonPrimitive(limit.coerceIn(1, ELASTICSEARCH_MAX_SIZE)))) } }.getOrDefault(fallbackQueryBody) val bodyObj = queryBody.toMutableMap() - if (!bodyObj.containsKey("size")) bodyObj["size"] = JsonPrimitive(limit.coerceIn(1, 10000)) + if (!bodyObj.containsKey("size")) bodyObj["size"] = JsonPrimitive(limit.coerceIn(1, ELASTICSEARCH_MAX_SIZE)) return suspendRunCatching { val response = httpClient.post("$baseUrl$path") { @@ -123,7 +129,7 @@ class ElasticsearchHandler : HttpApiHandler() { databaseName: String?, credentials: DataSourceCredentials, ): List { - val baseUrl = buildUrl(host, port ?: 9200) + val baseUrl = buildUrl(host, port ?: ELASTICSEARCH_DEFAULT_PORT) return suspendRunCatching { val response = httpClient.get("$baseUrl/_cat/indices?v&format=json") { applyAuth(credentials.apiKey, credentials.username, credentials.password) @@ -134,7 +140,7 @@ class ElasticsearchHandler : HttpApiHandler() { val obj = it.jsonObject val index = obj["index"]?.jsonPrimitive?.content ?: return@mapNotNull null DataSourceField(index, "index", "Elasticsearch index") - }.take(100) + }.take(ELASTICSEARCH_SCHEMA_LIMIT) } else { emptyList() } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/GraphiteHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/GraphiteHandler.kt index b3461b2da..c561e605c 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/GraphiteHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/GraphiteHandler.kt @@ -43,9 +43,15 @@ private val logger = KotlinLogging.logger {} */ class GraphiteHandler : HttpApiHandler() { + companion object { + private const val GRAPHITE_DEFAULT_PORT = 80 + private const val GRAPHITE_SCHEMA_LIMIT = 100 + private const val MILLIS_PER_SECOND = 1_000 + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { return suspendRunCatching { - val baseUrl = buildUrl(request.host, request.port ?: 80) + val baseUrl = buildUrl(request.host, request.port ?: GRAPHITE_DEFAULT_PORT) val response = httpClient.get("$baseUrl/render") { parameter("target", "constantLine(1)") parameter("format", "json") @@ -73,7 +79,7 @@ class GraphiteHandler : HttpApiHandler() { limit: Int, timeRange: TimeRangeDef?, ): List> { - val baseUrl = buildUrl(host, port ?: 80) + val baseUrl = buildUrl(host, port ?: GRAPHITE_DEFAULT_PORT) val from = timeRange?.from ?: "-24h" val until = timeRange?.to ?: "now" val targets = query.split(",").map { it.trim() }.filter { it.isNotBlank() } @@ -104,14 +110,16 @@ class GraphiteHandler : HttpApiHandler() { databaseName: String?, credentials: DataSourceCredentials, ): List { - val baseUrl = buildUrl(host, port ?: 80) + val baseUrl = buildUrl(host, port ?: GRAPHITE_DEFAULT_PORT) return suspendRunCatching { val response = httpClient.get("$baseUrl/metrics/index.json") { credentials.apiKey?.let { header("Authorization", "Bearer $it") } } if (response.status.isSuccess()) { val arr = json.parseToJsonElement(response.bodyAsText()).jsonArray - arr.map { DataSourceField(it.jsonPrimitive.content, "metric", "Graphite metric") }.take(100) + arr.map { + DataSourceField(it.jsonPrimitive.content, "metric", "Graphite metric") + }.take(GRAPHITE_SCHEMA_LIMIT) } else { emptyList() } @@ -134,7 +142,7 @@ class GraphiteHandler : HttpApiHandler() { val value = pair.getOrNull(0)?.jsonPrimitive?.content?.toDoubleOrNull() rows.add( mapOf( - "time_bucket" to JsonPrimitive(ts * 1000), + "time_bucket" to JsonPrimitive(ts * MILLIS_PER_SECOND), target to JsonPrimitive(value ?: 0.0) ) ) diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/InfluxDBHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/InfluxDBHandler.kt index 74d7d12ed..d61f3e955 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/InfluxDBHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/InfluxDBHandler.kt @@ -43,9 +43,14 @@ private val logger = KotlinLogging.logger {} */ class InfluxDBHandler : HttpApiHandler() { + companion object { + private const val INFLUXDB_DEFAULT_PORT = 8086 + private const val FIELDS_PAGE_SIZE = 100 + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { return suspendRunCatching { - val baseUrl = buildUrl(request.host, request.port ?: 8086) + val baseUrl = buildUrl(request.host, request.port ?: INFLUXDB_DEFAULT_PORT) val org = request.databaseName ?: "moneat" val query = "buckets()" val body = "org=$org&query=${java.net.URLEncoder.encode(query, "UTF-8")}" @@ -75,7 +80,7 @@ class InfluxDBHandler : HttpApiHandler() { limit: Int, timeRange: TimeRangeDef?, ): List> { - val baseUrl = buildUrl(host, port ?: 8086) + val baseUrl = buildUrl(host, port ?: INFLUXDB_DEFAULT_PORT) val org = databaseName ?: "moneat" val fluxQuery = if (timeRange != null) { val from = timeRange.from @@ -110,7 +115,7 @@ class InfluxDBHandler : HttpApiHandler() { databaseName: String?, credentials: DataSourceCredentials, ): List { - val baseUrl = buildUrl(host, port ?: 8086) + val baseUrl = buildUrl(host, port ?: INFLUXDB_DEFAULT_PORT) val org = databaseName ?: "moneat" return suspendRunCatching { val bucket = databaseName ?: "moneat" @@ -125,7 +130,7 @@ class InfluxDBHandler : HttpApiHandler() { val csv = response.bodyAsText() val lines = csv.lines().filter { it.isNotBlank() } if (lines.size >= 2) { - lines.drop(1).take(100).mapNotNull { line -> + lines.drop(1).take(FIELDS_PAGE_SIZE).mapNotNull { line -> val vals = line.split(",").map { it.trim() } val name = vals.getOrNull(1) ?: return@mapNotNull null DataSourceField(name, "measurement", "InfluxDB measurement") diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/JdbcHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/JdbcHandler.kt index 39dc99099..0689b4a2d 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/JdbcHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/JdbcHandler.kt @@ -38,6 +38,18 @@ abstract class JdbcHandler( private val pools: ConcurrentHashMap, ) : DataSourceHandler { + companion object { + private const val QUERY_MAX_ROWS = 10_000 + private const val QUERY_TIMEOUT_SECONDS = 30 + private const val SCHEMA_COMMENT_COLUMN = 3 + private const val POOL_MAX_SIZE = 3 + private const val POOL_MIN_IDLE = 0 + private const val POOL_IDLE_TIMEOUT_MS = 60_000L + private const val POOL_MAX_LIFETIME_MS = 300_000L + private const val POOL_CONNECTION_TIMEOUT_MS = 10_000L + private const val TEMP_POOL_MAX_SIZE = 1 + } + protected abstract fun buildJdbcUrl(host: String, port: Int, database: String): String protected abstract fun defaultPort(): Int protected abstract fun schemaIntrospectionQuery(): String @@ -86,8 +98,8 @@ abstract class JdbcHandler( val ds = getOrCreatePool(sourceId, host, p, db, credentials) return ds.connection.use { conn -> conn.createStatement().use { stmt -> - stmt.maxRows = limit.coerceIn(1, 10000) - stmt.queryTimeout = 30 + stmt.maxRows = limit.coerceIn(1, QUERY_MAX_ROWS) + stmt.queryTimeout = QUERY_TIMEOUT_SECONDS stmt.executeQuery(query).use { rs -> resultSetToMaps(rs) } } } @@ -109,7 +121,13 @@ abstract class JdbcHandler( conn.createStatement().use { stmt -> stmt.executeQuery(query).use { rs -> while (rs.next()) { - fields.add(DataSourceField(rs.getString(1), rs.getString(2), rs.getString(3).orEmpty())) + fields.add( + DataSourceField( + rs.getString(1), + rs.getString(2), + rs.getString(SCHEMA_COMMENT_COLUMN).orEmpty(), + ) + ) } } } @@ -144,11 +162,11 @@ abstract class JdbcHandler( jdbcUrl = buildJdbcUrl(host, port, database) this.username = username ?: "" this.password = password ?: "" - maximumPoolSize = 3 + maximumPoolSize = POOL_MAX_SIZE minimumIdle = 0 - idleTimeout = 60_000 - maxLifetime = 300_000 - connectionTimeout = 10_000 + idleTimeout = POOL_IDLE_TIMEOUT_MS + maxLifetime = POOL_MAX_LIFETIME_MS + connectionTimeout = POOL_CONNECTION_TIMEOUT_MS isReadOnly = true addDataSourceProperty("ApplicationName", "moneat-custom-datasource") } @@ -168,7 +186,7 @@ abstract class JdbcHandler( this.username = username ?: "" this.password = password ?: "" maximumPoolSize = 1 - connectionTimeout = 10_000 + connectionTimeout = POOL_CONNECTION_TIMEOUT_MS isReadOnly = true } return HikariDataSource(config) diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/LokiHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/LokiHandler.kt index 35c2950e9..834a3ba30 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/LokiHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/LokiHandler.kt @@ -44,9 +44,22 @@ private val logger = KotlinLogging.logger {} */ class LokiHandler : HttpApiHandler() { + companion object { + private const val LOKI_DEFAULT_PORT = 3100 + private const val MILLIS_PER_SECOND = 1_000 + private const val LOKI_MAX_QUERY_LIMIT = 5_000 + private const val LOKI_TIMESTAMP_MS_LENGTH = 13 + private const val SECONDS_PER_HOUR = 3_600L + private const val SECONDS_PER_MINUTE = 60L + private const val SECONDS_PER_DAY = 86_400L + private const val SECONDS_PER_WEEK = 604_800L + private const val SECONDS_PER_MONTH_30 = 2_592_000L + private const val SECONDS_PER_YEAR_365 = 31_536_000L + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { return suspendRunCatching { - val baseUrl = buildUrl(request.host, request.port ?: 3100) + val baseUrl = buildUrl(request.host, request.port ?: LOKI_DEFAULT_PORT) val response = httpClient.get("$baseUrl/ready") { request.apiKey?.let { header("X-Scope-OrgID", it) } } @@ -71,13 +84,13 @@ class LokiHandler : HttpApiHandler() { limit: Int, timeRange: TimeRangeDef?, ): List> { - val baseUrl = buildUrl(host, port ?: 3100) - val nowSec = System.currentTimeMillis() / 1000 - val fromSec = timeRange?.from?.let { resolveRelativeTimeSec(it, nowSec) } ?: nowSec - 3600 + val baseUrl = buildUrl(host, port ?: LOKI_DEFAULT_PORT) + val nowSec = System.currentTimeMillis() / MILLIS_PER_SECOND + val fromSec = timeRange?.from?.let { resolveRelativeTimeSec(it, nowSec) } ?: nowSec - SECONDS_PER_HOUR val toSec = timeRange?.to?.let { resolveRelativeTimeSec(it, nowSec) } ?: nowSec return suspendRunCatching { - val boundedLimit = limit.coerceIn(1, 5000) + val boundedLimit = limit.coerceIn(1, LOKI_MAX_QUERY_LIMIT) val response = httpClient.get("$baseUrl/loki/api/v1/query_range") { parameter("query", query) parameter("start", fromSec.toString() + "000000000") @@ -102,7 +115,7 @@ class LokiHandler : HttpApiHandler() { databaseName: String?, credentials: DataSourceCredentials, ): List { - val baseUrl = buildUrl(host, port ?: 3100) + val baseUrl = buildUrl(host, port ?: LOKI_DEFAULT_PORT) return suspendRunCatching { val response = httpClient.get("$baseUrl/loki/api/v1/labels") { credentials.apiKey?.let { header("X-Scope-OrgID", it) } @@ -131,7 +144,7 @@ class LokiHandler : HttpApiHandler() { ?: return emptyList() val matcher = labelMatch.groupValues[1].trim() val labelName = labelMatch.groupValues[2].trim() - val baseUrl = buildUrl(host, port ?: 3100) + val baseUrl = buildUrl(host, port ?: LOKI_DEFAULT_PORT) return suspendRunCatching { val response = httpClient.get("$baseUrl/loki/api/v1/label/$labelName/values") { if (matcher.isNotBlank()) parameter("query", matcher) @@ -163,7 +176,7 @@ class LokiHandler : HttpApiHandler() { val tsNs = arr.getOrNull(0)?.jsonPrimitive?.contentOrNull ?: "0" val log = arr.getOrNull(1)?.jsonPrimitive?.content ?: "" val row = mutableMapOf() - row["time_bucket"] = JsonPrimitive(tsNs.take(13).toLongOrNull() ?: 0L) + row["time_bucket"] = JsonPrimitive(tsNs.take(LOKI_TIMESTAMP_MS_LENGTH).toLongOrNull() ?: 0L) row["log"] = JsonPrimitive(log) for ((k, v) in metric) row[k] = v rows.add(row) @@ -178,12 +191,12 @@ class LokiHandler : HttpApiHandler() { val amount = match.groupValues[1].toLong() val offsetSec = when (match.groupValues[2]) { "s" -> amount - "m" -> amount * 60 - "h" -> amount * 3600 - "d" -> amount * 86400 - "w" -> amount * 604800 - "M" -> amount * 2592000 - "y" -> amount * 31536000 + "m" -> amount * SECONDS_PER_MINUTE + "h" -> amount * SECONDS_PER_HOUR + "d" -> amount * SECONDS_PER_DAY + "w" -> amount * SECONDS_PER_WEEK + "M" -> amount * SECONDS_PER_MONTH_30 + "y" -> amount * SECONDS_PER_YEAR_365 else -> 0 } return nowSec - offsetSec diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/MongoDBHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/MongoDBHandler.kt index c1b15b11e..78c569eaf 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/MongoDBHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/MongoDBHandler.kt @@ -40,15 +40,24 @@ private val logger = KotlinLogging.logger {} */ class MongoDBHandler : DataSourceHandler { + companion object { + private const val MONGODB_DEFAULT_PORT = 27017 + private const val MONGODB_DB_SAMPLE_LIMIT = 20 + } + override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { val connStr = request.connectionString ?: buildConnectionString( - request.host, request.port ?: 27017, request.databaseName, request.username, request.password + request.host, request.port ?: MONGODB_DEFAULT_PORT, request.databaseName, request.username, request.password ) return suspendRunCatching { MongoClients.create(connStr).use { client -> val databases = client.listDatabaseNames().toList() - TestConnectionResult(true, "Connected successfully", databases = databases.take(20)) + TestConnectionResult( + true, + "Connected successfully", + databases = databases.take(MONGODB_DB_SAMPLE_LIMIT), + ) } }.getOrElse { e -> logger.warn(e) { "MongoDB connection test failed" } @@ -67,7 +76,13 @@ class MongoDBHandler : DataSourceHandler { timeRange: TimeRangeDef?, ): List> { val connStr = credentials.connectionString - ?: buildConnectionString(host, port ?: 27017, databaseName, credentials.username, credentials.password) + ?: buildConnectionString( + host, + port ?: MONGODB_DEFAULT_PORT, + databaseName, + credentials.username, + credentials.password, + ) val dbName = databaseName ?: "test" return suspendRunCatching { @@ -98,7 +113,13 @@ class MongoDBHandler : DataSourceHandler { credentials: DataSourceCredentials, ): List { val connStr = credentials.connectionString - ?: buildConnectionString(host, port ?: 27017, databaseName, credentials.username, credentials.password) + ?: buildConnectionString( + host, + port ?: MONGODB_DEFAULT_PORT, + databaseName, + credentials.username, + credentials.password, + ) val dbName = databaseName ?: "test" return suspendRunCatching { diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/PrometheusHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/PrometheusHandler.kt index 880fecd31..369540aac 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/PrometheusHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/PrometheusHandler.kt @@ -41,6 +41,19 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} class PrometheusHandler : HttpApiHandler() { + + companion object { + private const val PROMETHEUS_LABEL_LIMIT = 20 + private const val MILLIS_PER_SECOND = 1_000 + private const val SECONDS_PER_MINUTE = 60L + private const val SECONDS_PER_HOUR = 3_600L + private const val SECONDS_SIX_HOURS = 21_600L + private const val SECONDS_PER_DAY = 86_400L + private const val SECONDS_PER_WEEK = 604_800L + private const val SECONDS_PER_MONTH_30 = 2_592_000L + private const val SECONDS_PER_YEAR_365 = 31_536_000L + } + override val defaultPort: Int = 9090 override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { @@ -48,12 +61,12 @@ class PrometheusHandler : HttpApiHandler() { val baseUrl = buildUrl(request.host, request.port) val response = httpClient.get("$baseUrl/api/v1/label/__name__/values") { request.apiKey?.let { header(HttpHeaders.Authorization, "Bearer $it") } - parameter("limit", 20) + parameter("limit", PROMETHEUS_LABEL_LIMIT) } if (response.status.isSuccess()) { val body = json.parseToJsonElement(response.bodyAsText()).jsonObject val metrics = body["data"]?.jsonArray?.map { it.jsonPrimitive.content } ?: emptyList() - TestConnectionResult(true, "Connected successfully", metrics = metrics.take(20)) + TestConnectionResult(true, "Connected successfully", metrics = metrics.take(PROMETHEUS_LABEL_LIMIT)) } else { TestConnectionResult(false, "Prometheus returned ${response.status}") } @@ -122,7 +135,7 @@ class PrometheusHandler : HttpApiHandler() { return suspendRunCatching { val response = if (timeRange != null) { - val nowSec = System.currentTimeMillis() / 1000 + val nowSec = System.currentTimeMillis() / MILLIS_PER_SECOND val fromSec = resolveRelativeTimeSec(timeRange.from, nowSec) val toSec = resolveRelativeTimeSec(timeRange.to, nowSec) val step = resolvePrometheusStep(toSec - fromSec) @@ -224,7 +237,7 @@ class PrometheusHandler : HttpApiHandler() { private fun promTimestampToMs(element: JsonElement): JsonElement { val sec = element.jsonPrimitive.doubleOrNull ?: return element - return JsonPrimitive((sec * 1000).toLong()) + return JsonPrimitive((sec * MILLIS_PER_SECOND).toLong()) } private fun promValueToNumber(element: JsonElement): JsonElement { @@ -239,22 +252,22 @@ class PrometheusHandler : HttpApiHandler() { val amount = match.groupValues[1].toLong() val offsetSec = when (match.groupValues[2]) { "s" -> amount - "m" -> amount * 60 - "h" -> amount * 3600 - "d" -> amount * 86400 - "w" -> amount * 604800 - "M" -> amount * 2592000 - "y" -> amount * 31536000 + "m" -> amount * SECONDS_PER_MINUTE + "h" -> amount * SECONDS_PER_HOUR + "d" -> amount * SECONDS_PER_DAY + "w" -> amount * SECONDS_PER_WEEK + "M" -> amount * SECONDS_PER_MONTH_30 + "y" -> amount * SECONDS_PER_YEAR_365 else -> 0 } return nowSec - offsetSec } internal fun resolvePrometheusStep(rangeSec: Long): String = when { - rangeSec <= 3600 -> "15s" - rangeSec <= 21600 -> "1m" - rangeSec <= 86400 -> "5m" - rangeSec <= 604800 -> "1h" + rangeSec <= SECONDS_PER_HOUR -> "15s" + rangeSec <= SECONDS_SIX_HOURS -> "1m" + rangeSec <= SECONDS_PER_DAY -> "5m" + rangeSec <= SECONDS_PER_WEEK -> "1h" else -> "1d" } } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/RedisHandler.kt b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/RedisHandler.kt index 6ccafee11..5638ba782 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/RedisHandler.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/services/handlers/RedisHandler.kt @@ -40,13 +40,13 @@ private val logger = KotlinLogging.logger {} class RedisHandler : DataSourceHandler { override suspend fun testConnection(request: TestConnectionRequest): TestConnectionResult { - val uri = buildRedisUri(request.host, request.port ?: 6379, request.password ?: request.apiKey) + val uri = buildRedisUri(request.host, request.port ?: REDIS_DEFAULT_PORT, request.password ?: request.apiKey) return suspendRunCatching { RedisClient.create(uri).use { client -> client.connect().use { conn -> conn.sync().ping() - val keys = scanKeys(conn.sync(), "*", 20) + val keys = scanKeys(conn.sync(), "*", REDIS_SAMPLE_KEY_LIMIT) TestConnectionResult(true, "Connected successfully", keys = keys) } } @@ -66,7 +66,7 @@ class RedisHandler : DataSourceHandler { limit: Int, timeRange: TimeRangeDef?, ): List> { - val uri = buildRedisUri(host, port ?: 6379, credentials.password ?: credentials.apiKey) + val uri = buildRedisUri(host, port ?: REDIS_DEFAULT_PORT, credentials.password ?: credentials.apiKey) val db = databaseName?.toIntOrNull() ?: 0 return suspendRunCatching { @@ -141,7 +141,7 @@ class RedisHandler : DataSourceHandler { "LRANGE" -> { val key = parts.getOrNull(1) ?: return emptyList() val start = parts.getOrNull(2)?.toLongOrNull() ?: 0L - val stop = parts.getOrNull(3)?.toLongOrNull() ?: -1L + val stop = parts.getOrNull(LRANGE_STOP_ARG_INDEX)?.toLongOrNull() ?: -1L cmd.lrange(key, start, stop).take(limit).mapIndexed { i, v -> mapOf( "index" to JsonPrimitive(i), @@ -197,13 +197,13 @@ class RedisHandler : DataSourceHandler { databaseName: String?, credentials: DataSourceCredentials, ): List { - val uri = buildRedisUri(host, port ?: 6379, credentials.password ?: credentials.apiKey) + val uri = buildRedisUri(host, port ?: REDIS_DEFAULT_PORT, credentials.password ?: credentials.apiKey) val dbIndex = databaseName?.toIntOrNull() ?: 0 return suspendRunCatching { RedisClient.create(uri).use { client -> client.connect().use { conn -> if (dbIndex != 0) conn.sync().select(dbIndex) - val keys = scanKeys(conn.sync(), "*", 100) + val keys = scanKeys(conn.sync(), "*", REDIS_SCHEMA_KEY_LIMIT) keys.map { DataSourceField(it, "key", "Redis key") } } } @@ -213,13 +213,6 @@ class RedisHandler : DataSourceHandler { } } - private fun buildRedisUri(host: String, port: Int, password: String?): String { - val scheme = if (host.startsWith("rediss://")) "rediss://" else "redis://" - val cleanHost = host.removePrefix("rediss://").removePrefix("redis://") - val auth = if (!password.isNullOrBlank()) ":$password@" else "" - return "$scheme$auth$cleanHost:$port" - } - /** * Scan Redis keys using the non-blocking SCAN command with cursor iteration. * Avoids [io.lettuce.core.api.sync.RedisKeyCommands.keys] which can block the server. @@ -241,6 +234,52 @@ class RedisHandler : DataSourceHandler { } companion object { + /** + * Build a Lettuce-compatible Redis URI from connection parameters. + * + * Bare IPv6 literals (e.g. `2001:db8::5`) are bracketed per RFC 2732 before + * being passed to [java.net.URI]; otherwise the URI parser returns null for + * the host and the fallback truncates the address at the first colon. + */ + internal fun buildRedisUri(host: String, port: Int, password: String?): String { + val scheme = if (host.startsWith(REDISS_SCHEME)) REDISS_SCHEME else REDIS_SCHEME + val hostPart = host.removePrefix(REDISS_SCHEME).removePrefix(REDIS_SCHEME) + // RFC 2732: IPv6 literals require brackets in the authority component. + // Detect bare IPv6 by multiple colons and no leading '['. + val bracketed = if (!hostPart.startsWith('[') && hostPart.count { it == ':' } > 1) { + "[$hostPart]" + } else { + hostPart + } + val normalized = "$scheme$bracketed" + val parsed = runCatching { java.net.URI(normalized) }.getOrNull() + // java.net.URI.getHost() may return IPv6 with or without brackets depending on JVM. + // Normalise: strip any brackets, then re-bracket only when the host contains a colon. + val rawHost = parsed?.host ?: run { + val stripped = bracketed.removePrefix("[") + if (stripped.contains(']')) stripped.substringBefore(']') else stripped.substringBefore(':') + } + val bareHost = rawHost.removePrefix("[").removeSuffix("]") + val cleanHost = if (bareHost.contains(':')) "[$bareHost]" else bareHost + val resolvedPort = if (parsed != null && parsed.port != -1) parsed.port else port + val auth = if (!password.isNullOrBlank()) ":$password@" else "" + return "$scheme$auth$cleanHost:$resolvedPort" + } + + private const val REDIS_SCHEME = "redis://" + private const val REDISS_SCHEME = "rediss://" + private const val REDIS_DEFAULT_PORT = 6379 + private const val REDIS_SAMPLE_KEY_LIMIT = 20 + private const val REDIS_SCHEMA_KEY_LIMIT = 100 + private const val LRANGE_STOP_ARG_INDEX = 3 + private const val SLOWLOG_ARGS_INDEX = 3 + private const val CLUSTER_NODE_MASTER_IDX = 3 + private const val CLUSTER_NODE_PING_IDX = 4 + private const val CLUSTER_NODE_PONG_IDX = 5 + private const val CLUSTER_NODE_EPOCH_IDX = 6 + private const val CLUSTER_NODE_LINK_IDX = 7 + private const val CLUSTER_NODE_SLOT_IDX = 8 + /** * Parse Redis INFO output (key:value lines grouped by sections) * into a single flat map row. @@ -318,7 +357,7 @@ class RedisHandler : DataSourceHandler { val id = (fields.getOrNull(0) as? Number)?.toLong() ?: 0L val ts = (fields.getOrNull(1) as? Number)?.toLong() ?: 0L val duration = (fields.getOrNull(2) as? Number)?.toLong() ?: 0L - val args = (fields.getOrNull(3) as? List<*>) + val args = (fields.getOrNull(SLOWLOG_ARGS_INDEX) as? List<*>) ?.joinToString(" ") ?: "" mapOf( "Id" to JsonPrimitive(id), @@ -343,12 +382,12 @@ class RedisHandler : DataSourceHandler { "Id" to JsonPrimitive(parts.getOrElse(0) { "" }), "Address" to JsonPrimitive(parts.getOrElse(1) { "" }), "Flags" to JsonPrimitive(parts.getOrElse(2) { "" }), - "Master" to JsonPrimitive(parts.getOrElse(3) { "" }), - "Ping" to JsonPrimitive(parts.getOrElse(4) { "" }), - "Pong" to JsonPrimitive(parts.getOrElse(5) { "" }), - "Epoch" to JsonPrimitive(parts.getOrElse(6) { "" }), - "Link" to JsonPrimitive(parts.getOrElse(7) { "" }), - "Slot" to JsonPrimitive(parts.getOrElse(8) { "" }) + "Master" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_MASTER_IDX) { "" }), + "Ping" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_PING_IDX) { "" }), + "Pong" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_PONG_IDX) { "" }), + "Epoch" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_EPOCH_IDX) { "" }), + "Link" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_LINK_IDX) { "" }), + "Slot" to JsonPrimitive(parts.getOrElse(CLUSTER_NODE_SLOT_IDX) { "" }) ) } } diff --git a/backend/src/main/kotlin/com/moneat/dashboards/translation/DataDogTranslator.kt b/backend/src/main/kotlin/com/moneat/dashboards/translation/DataDogTranslator.kt index 28b8e97a8..1fa1ec411 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/translation/DataDogTranslator.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/translation/DataDogTranslator.kt @@ -40,6 +40,11 @@ import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.put import com.moneat.utils.suspendRunCatching +private const val DD_DEFAULT_WIDGET_W = 6 +private const val DD_DEFAULT_WIDGET_H = 4 +private const val DD_MAX_GRID_COL = 11 +private const val DD_GRID_COLS = 12 + class DataDogTranslator : DashboardTranslator { private val widgetTypeMap = mapOf( @@ -128,8 +133,8 @@ class DataDogTranslator : DashboardTranslator { // Parse grid position from DD layout (DD uses 12-col grid) val gridX = layout?.get("x")?.jsonPrimitive?.intOrNull ?: 0 val gridY = layout?.get("y")?.jsonPrimitive?.intOrNull ?: 0 - val gridW = layout?.get("width")?.jsonPrimitive?.intOrNull ?: 6 - val gridH = layout?.get("height")?.jsonPrimitive?.intOrNull ?: 4 + val gridW = layout?.get("width")?.jsonPrimitive?.intOrNull ?: DD_DEFAULT_WIDGET_W + val gridH = layout?.get("height")?.jsonPrimitive?.intOrNull ?: DD_DEFAULT_WIDGET_H val queryConfig = parseDataDogQuery(definition, warnings, index) @@ -138,10 +143,10 @@ class DataDogTranslator : DashboardTranslator { dashboardId = 0, title = widgetTitle, widgetType = moneatType ?: "text", - gridX = gridX.coerceIn(0, 11), + gridX = gridX.coerceIn(0, DD_MAX_GRID_COL), gridY = gridY, - gridW = gridW.coerceIn(1, 12), - gridH = gridH.coerceIn(1, 12), + gridW = gridW.coerceIn(1, DD_GRID_COLS), + gridH = gridH.coerceIn(1, DD_GRID_COLS), queryConfigs = listOf(queryConfig), sortOrder = index ) diff --git a/backend/src/main/kotlin/com/moneat/dashboards/translation/GrafanaTranslator.kt b/backend/src/main/kotlin/com/moneat/dashboards/translation/GrafanaTranslator.kt index 622c902ef..186d88e79 100644 --- a/backend/src/main/kotlin/com/moneat/dashboards/translation/GrafanaTranslator.kt +++ b/backend/src/main/kotlin/com/moneat/dashboards/translation/GrafanaTranslator.kt @@ -50,6 +50,13 @@ private const val GRAFANA_COLS = 24 private const val MONEAT_COLS = 12 private const val GRAFANA_ROW_PX = 30.0 private const val MONEAT_ROW_PX = 30.0 +private const val GRAFANA_DEFAULT_PANEL_W = 12 +private const val GRAFANA_DEFAULT_PANEL_H = 4 +private const val MONEAT_MAX_COL_IDX = MONEAT_COLS - 1 +private const val MIN_WIDGET_HEIGHT = 3 +private const val FILL_OPACITY_SCALE = 100.0 +private const val REGEX_THIRD_GROUP_IDX = 3 +private const val GRAFANA_SCHEMA_VERSION = 39 class GrafanaTranslator : DashboardTranslator { @@ -174,13 +181,13 @@ class GrafanaTranslator : DashboardTranslator { val gridPos = panelJson["gridPos"]?.jsonObject val grafanaX = gridPos?.get("x")?.jsonPrimitive?.intOrNull ?: 0 val grafanaY = gridPos?.get("y")?.jsonPrimitive?.intOrNull ?: 0 - val grafanaW = gridPos?.get("w")?.jsonPrimitive?.intOrNull ?: 12 - val grafanaH = gridPos?.get("h")?.jsonPrimitive?.intOrNull ?: 4 + val grafanaW = gridPos?.get("w")?.jsonPrimitive?.intOrNull ?: GRAFANA_DEFAULT_PANEL_W + val grafanaH = gridPos?.get("h")?.jsonPrimitive?.intOrNull ?: GRAFANA_DEFAULT_PANEL_H // Grafana uses a 24-col grid, Moneat uses 12-col. // Use floor-aligned scaling: x = floor(gx*12/24), w = floor((gx+gw)*12/24) - x // This guarantees adjacent panels share exact column boundaries with no gaps or overflow. - val gridX = (grafanaX * MONEAT_COLS / GRAFANA_COLS).coerceIn(0, 11) + val gridX = (grafanaX * MONEAT_COLS / GRAFANA_COLS).coerceIn(0, MONEAT_MAX_COL_IDX) val gridY = scaleGridValue(grafanaY) val gridXEnd = ((grafanaX + grafanaW) * MONEAT_COLS / GRAFANA_COLS).coerceAtMost(MONEAT_COLS) val gridW = (gridXEnd - gridX).coerceAtLeast(1) @@ -189,7 +196,7 @@ class GrafanaTranslator : DashboardTranslator { val queryConfigs = parseGrafanaTargets(panelJson, warnings, index, inputsMap) val displayConfig = extractDisplayConfig(panelJson) - val minH = if (moneatType == "stat" || moneatType == "gauge") 1 else 3 + val minH = if (moneatType == "stat" || moneatType == "gauge") 1 else MIN_WIDGET_HEIGHT return WidgetResponse( id = 0, dashboardId = 0, @@ -198,7 +205,7 @@ class GrafanaTranslator : DashboardTranslator { gridX = gridX, gridY = gridY, gridW = gridW, - gridH = gridH.coerceIn(minH, 12), + gridH = gridH.coerceIn(minH, MONEAT_COLS), queryConfigs = queryConfigs, displayConfig = displayConfig, sortOrder = index @@ -286,7 +293,7 @@ class GrafanaTranslator : DashboardTranslator { // fillOpacity: Grafana uses 0-100 scale, Moneat uses 0-1 defaults?.get("fillOpacity")?.jsonPrimitive?.intOrNull?.let { - config["fillOpacity"] = (it / 100.0).toString() + config["fillOpacity"] = (it / FILL_OPACITY_SCALE).toString() } // Stacking → stackMode (frontend key) @@ -809,7 +816,7 @@ class GrafanaTranslator : DashboardTranslator { val (metricName, labelStr, aggFunction) = when { aggByMatch != null -> { // Parse inner expression recursively for metric name - val innerExpr = aggByMatch.groupValues[3].trim() + val innerExpr = aggByMatch.groupValues[REGEX_THIRD_GROUP_IDX].trim() val innerFunc = Regex("""(\w+)\(([^{(]+?)(?:\{[^}]*\})?(?:\[[^]]*])?.*\)""").find(innerExpr) val metric = innerFunc?.groupValues?.getOrNull(2)?.trim() ?: "unknown" val innerLabels = Regex("""\{([^}]*)\}""").find(innerExpr)?.groupValues?.get(1) ?: "" @@ -818,7 +825,7 @@ class GrafanaTranslator : DashboardTranslator { funcMatch != null -> { Triple( funcMatch.groupValues[2].trim(), - funcMatch.groupValues[3], + funcMatch.groupValues[REGEX_THIRD_GROUP_IDX], mapPromFunction(funcMatch.groupValues[1]) ) } @@ -851,7 +858,7 @@ class GrafanaTranslator : DashboardTranslator { "!=" -> FilterOp.NEQ else -> FilterOp.EQ } - filters.add(FilterDef(key, op, m.groupValues[3])) + filters.add(FilterDef(key, op, m.groupValues[REGEX_THIRD_GROUP_IDX])) } } @@ -1047,7 +1054,7 @@ class GrafanaTranslator : DashboardTranslator { } ) } - put("schemaVersion", 39) + put("schemaVersion", GRAFANA_SCHEMA_VERSION) put("version", 1) put("timezone", "browser") } diff --git a/backend/src/main/kotlin/com/moneat/datadog/decompression/DecompressionService.kt b/backend/src/main/kotlin/com/moneat/datadog/decompression/DecompressionService.kt index bab42f587..e01a3327c 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/decompression/DecompressionService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/decompression/DecompressionService.kt @@ -24,6 +24,8 @@ import java.util.zip.InflaterInputStream private const val BUFFER_SIZE = 8192 private const val MAX_DECOMPRESSED_SIZE = 50 * 1024 * 1024 // 50 MB +private const val ZSTD_MAGIC_BYTE_COUNT = 4 +private const val ZSTD_MAGIC_LAST_BYTE_IDX = 3 object DecompressionService { @@ -40,8 +42,8 @@ object DecompressionService { if (data.size >= 2 && data[0] == 0x1F.toByte() && data[1] == 0x8B.toByte()) { return decompressGzip(data) } - if (data.size >= 4 && data[0] == 0x28.toByte() && data[1] == 0xB5.toByte() && - data[2] == 0x2F.toByte() && data[3] == 0xFD.toByte() + if (data.size >= ZSTD_MAGIC_BYTE_COUNT && data[0] == 0x28.toByte() && data[1] == 0xB5.toByte() && + data[2] == 0x2F.toByte() && data[ZSTD_MAGIC_LAST_BYTE_IDX] == 0xFD.toByte() ) { return decompressZstd(data) } diff --git a/backend/src/main/kotlin/com/moneat/datadog/decompression/MetricPayloadDecoder.kt b/backend/src/main/kotlin/com/moneat/datadog/decompression/MetricPayloadDecoder.kt index 7353f01f7..02ed45624 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/decompression/MetricPayloadDecoder.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/decompression/MetricPayloadDecoder.kt @@ -41,6 +41,13 @@ import com.moneat.datadog.models.DatadogMetricV1 */ object MetricPayloadDecoder { + private const val PROTO_FIELD_SHIFT = 3 + private const val PROTO_FIELD_3 = 3 + private const val PROTO_FIELD_4 = 4 + private const val PROTO_FIELD_5 = 5 + private const val PROTO_FIELD_6 = 6 + private const val PROTO_FIELD_7 = 7 + private val METRIC_TYPES = mapOf(0 to "gauge", 1 to "count", 2 to "rate", 3 to "gauge") /** Decodes a raw (already decompressed) protobuf MetricPayload into DatadogMetricSeriesV1. */ @@ -52,14 +59,13 @@ object MetricPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1 (LEN): repeated MetricSeries series - (1 shl 3) or 2 -> series += decodeSeries(input.readByteArray()) + (1 shl PROTO_FIELD_SHIFT) or 2 -> series += decodeSeries(input.readByteArray()) else -> input.skipField(tag) } } return DatadogMetricSeriesV1(series = series) } - @Suppress("MagicNumber") private fun decodeSeries(bytes: ByteArray): DatadogMetricV1 { val input = CodedInputStream.newInstance(bytes) var metric = "" @@ -74,22 +80,22 @@ object MetricPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1 (LEN): repeated Resource - (1 shl 3) or 2 -> { + (1 shl PROTO_FIELD_SHIFT) or 2 -> { val resource = decodeResource(input.readByteArray()) if (resource.first == "host") host = resource.second } // field 2 (LEN): string metric - (2 shl 3) or 2 -> metric = input.readString() + (2 shl PROTO_FIELD_SHIFT) or 2 -> metric = input.readString() // field 3 (LEN): repeated string tags - (3 shl 3) or 2 -> tags += input.readString() + (PROTO_FIELD_3 shl PROTO_FIELD_SHIFT) or 2 -> tags += input.readString() // field 4 (LEN): repeated MetricPoint - (4 shl 3) or 2 -> points += decodeMetricPoint(input.readByteArray()) + (PROTO_FIELD_4 shl PROTO_FIELD_SHIFT) or 2 -> points += decodeMetricPoint(input.readByteArray()) // field 5 (VARINT): MetricType - (5 shl 3) or 0 -> typeInt = input.readEnum() + (PROTO_FIELD_5 shl PROTO_FIELD_SHIFT) or 0 -> typeInt = input.readEnum() // field 6 (LEN): string unit - (6 shl 3) or 2 -> unit = input.readString() + (PROTO_FIELD_6 shl PROTO_FIELD_SHIFT) or 2 -> unit = input.readString() // field 7 (LEN): string source_type_name - (7 shl 3) or 2 -> sourceTypeName = input.readString() + (PROTO_FIELD_7 shl PROTO_FIELD_SHIFT) or 2 -> sourceTypeName = input.readString() else -> input.skipField(tag) } } @@ -113,8 +119,8 @@ object MetricPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 2 -> type = input.readString() - (2 shl 3) or 2 -> name = input.readString() + (1 shl PROTO_FIELD_SHIFT) or 2 -> type = input.readString() + (2 shl PROTO_FIELD_SHIFT) or 2 -> name = input.readString() else -> input.skipField(tag) } } @@ -122,7 +128,6 @@ object MetricPayloadDecoder { } // MetricPoint: field 1 (double/fixed64) value, field 2 (varint) timestamp - @Suppress("MagicNumber") private fun decodeMetricPoint(bytes: ByteArray): List { val input = CodedInputStream.newInstance(bytes) var value = 0.0 @@ -130,8 +135,8 @@ object MetricPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 1 -> value = input.readDouble() // wire type 1 = 64-bit - (2 shl 3) or 0 -> timestamp = input.readInt64() // wire type 0 = varint + (1 shl PROTO_FIELD_SHIFT) or 1 -> value = input.readDouble() // wire type 1 = 64-bit + (2 shl PROTO_FIELD_SHIFT) or 0 -> timestamp = input.readInt64() // wire type 0 = varint else -> input.skipField(tag) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/decompression/ProcessAgentPayloadDecoder.kt b/backend/src/main/kotlin/com/moneat/datadog/decompression/ProcessAgentPayloadDecoder.kt index 36e77fb39..961fda948 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/decompression/ProcessAgentPayloadDecoder.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/decompression/ProcessAgentPayloadDecoder.kt @@ -45,6 +45,25 @@ object ProcessAgentPayloadDecoder { private const val ENC_ZSTD_1X_PB: Int = 4 private const val ENC_ZSTD_PB_NO_CGO: Int = 5 + // Protobuf wire format constants + private const val PROTO_FIELD_SHIFT = 3 + private const val PROTO_PAYLOAD_ITEMS_FIELD = 3 + private const val PROTO_WIRE_FLOAT = 5 // 32-bit float wire type + private const val PROTO_FIELD_3 = 3 + private const val PROTO_FIELD_4 = 4 + private const val PROTO_FIELD_5 = 5 + private const val PROTO_FIELD_6 = 6 + private const val PROTO_FIELD_7 = 7 + private const val PROTO_FIELD_8 = 8 + private const val PROTO_FIELD_11 = 11 + private const val PROTO_FIELD_12 = 12 + private const val PROTO_FIELD_16 = 16 + private const val PROTO_FIELD_17 = 17 + private const val PROTO_FIELD_20 = 20 + private const val PROTO_FIELD_21 = 21 + private const val PROTO_FIELD_23 = 23 + private const val PROTO_FIELD_26 = 26 + // Type constants const val TYPE_COLLECTOR_PROC: Int = 12 const val TYPE_COLLECTOR_CONTAINER: Int = 39 @@ -108,9 +127,10 @@ object ProcessAgentPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1, LEN: hostName - (1 shl 3) or 2 -> hostName = input.readString() + (1 shl PROTO_FIELD_SHIFT) or 2 -> hostName = input.readString() // field 3, LEN: repeated Container - (3 shl 3) or 2 -> containers += decodeContainer(input.readByteArray()) + (PROTO_PAYLOAD_ITEMS_FIELD shl PROTO_FIELD_SHIFT) or 2 -> + containers += decodeContainer(input.readByteArray()) else -> input.skipField(tag) } } @@ -128,7 +148,6 @@ object ProcessAgentPayloadDecoder { // 20 (float) : totalPct → cpuPercent // 21 (uint64): memRss → memUsage // 26 (string, repeated): tags - @Suppress("MagicNumber") private fun decodeContainer(bytes: ByteArray): DatadogContainer { val input = CodedInputStream.newInstance(bytes) var id = "" @@ -145,16 +164,16 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (2 shl 3) or 2 -> id = input.readString() - (3 shl 3) or 2 -> name = input.readString() - (4 shl 3) or 2 -> image = input.readString() - (6 shl 3) or 0 -> memLimit = input.readUInt64() - (8 shl 3) or 0 -> state = input.readEnum() - (16 shl 3) or 5 -> netRcvdBps = input.readFloat() - (17 shl 3) or 5 -> netSentBps = input.readFloat() - (20 shl 3) or 5 -> totalPct = input.readFloat() - (21 shl 3) or 0 -> memRss = input.readUInt64() - (26 shl 3) or 2 -> tags += input.readString() + (2 shl PROTO_FIELD_SHIFT) or 2 -> id = input.readString() + (PROTO_FIELD_3 shl PROTO_FIELD_SHIFT) or 2 -> name = input.readString() + (PROTO_FIELD_4 shl PROTO_FIELD_SHIFT) or 2 -> image = input.readString() + (PROTO_FIELD_6 shl PROTO_FIELD_SHIFT) or 0 -> memLimit = input.readUInt64() + (PROTO_FIELD_8 shl PROTO_FIELD_SHIFT) or 0 -> state = input.readEnum() + (PROTO_FIELD_16 shl PROTO_FIELD_SHIFT) or PROTO_WIRE_FLOAT -> netRcvdBps = input.readFloat() + (PROTO_FIELD_17 shl PROTO_FIELD_SHIFT) or PROTO_WIRE_FLOAT -> netSentBps = input.readFloat() + (PROTO_FIELD_20 shl PROTO_FIELD_SHIFT) or PROTO_WIRE_FLOAT -> totalPct = input.readFloat() + (PROTO_FIELD_21 shl PROTO_FIELD_SHIFT) or 0 -> memRss = input.readUInt64() + (PROTO_FIELD_26 shl PROTO_FIELD_SHIFT) or 2 -> tags += input.readString() else -> input.skipField(tag) } } @@ -185,8 +204,9 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (2 shl 3) or 2 -> hostName = input.readString() - (3 shl 3) or 2 -> processes += decodeProcess(input.readByteArray()) + (2 shl PROTO_FIELD_SHIFT) or 2 -> hostName = input.readString() + (PROTO_PAYLOAD_ITEMS_FIELD shl PROTO_FIELD_SHIFT) or 2 -> + processes += decodeProcess(input.readByteArray()) else -> input.skipField(tag) } } @@ -202,7 +222,6 @@ object ProcessAgentPayloadDecoder { // 11 (int32) : openFdCount // 12 (enum) : state // 23 (repeated string): tags - @Suppress("MagicNumber") private fun decodeProcess(bytes: ByteArray): DatadogProcess { val input = CodedInputStream.newInstance(bytes) var pid = 0 @@ -220,32 +239,32 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (2 shl 3) or 0 -> pid = input.readInt32() - (4 shl 3) or 2 -> { + (2 shl PROTO_FIELD_SHIFT) or 0 -> pid = input.readInt32() + (PROTO_FIELD_4 shl PROTO_FIELD_SHIFT) or 2 -> { val r = decodeCommand( input.readByteArray() ) cmdName = r.first cmdFull = r.second } - (5 shl 3) or 2 -> userName = decodeProcessUser(input.readByteArray()) - (7 shl 3) or 2 -> { + (PROTO_FIELD_5 shl PROTO_FIELD_SHIFT) or 2 -> userName = decodeProcessUser(input.readByteArray()) + (PROTO_FIELD_7 shl PROTO_FIELD_SHIFT) or 2 -> { val r = decodeMemoryStat( input.readByteArray() ) memRss = r.first memVms = r.second } - (8 shl 3) or 2 -> { + (PROTO_FIELD_8 shl PROTO_FIELD_SHIFT) or 2 -> { val r = decodeCpuStat( input.readByteArray() ) cpuTotalPct = r.first numThreads = r.second } - (11 shl 3) or 0 -> openFdCount = input.readInt32() - (12 shl 3) or 0 -> state = input.readEnum() - (23 shl 3) or 2 -> tags += input.readString() + (PROTO_FIELD_11 shl PROTO_FIELD_SHIFT) or 0 -> openFdCount = input.readInt32() + (PROTO_FIELD_12 shl PROTO_FIELD_SHIFT) or 0 -> state = input.readEnum() + (PROTO_FIELD_23 shl PROTO_FIELD_SHIFT) or 2 -> tags += input.readString() else -> input.skipField(tag) } } @@ -265,7 +284,6 @@ object ProcessAgentPayloadDecoder { } // Command: field 1 (repeated string) args, field 8 (string) exe - @Suppress("MagicNumber") private fun decodeCommand(bytes: ByteArray): Pair { val input = CodedInputStream.newInstance(bytes) val args = mutableListOf() @@ -273,8 +291,8 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 2 -> args += input.readString() - (8 shl 3) or 2 -> exe = input.readString() + (1 shl PROTO_FIELD_SHIFT) or 2 -> args += input.readString() + (PROTO_FIELD_8 shl PROTO_FIELD_SHIFT) or 2 -> exe = input.readString() else -> input.skipField(tag) } } @@ -288,7 +306,7 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 2 -> return input.readString() + (1 shl PROTO_FIELD_SHIFT) or 2 -> return input.readString() else -> input.skipField(tag) } } @@ -303,8 +321,8 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> rss = input.readUInt64() - (2 shl 3) or 0 -> vms = input.readUInt64() + (1 shl PROTO_FIELD_SHIFT) or 0 -> rss = input.readUInt64() + (2 shl PROTO_FIELD_SHIFT) or 0 -> vms = input.readUInt64() else -> input.skipField(tag) } } @@ -312,7 +330,6 @@ object ProcessAgentPayloadDecoder { } // CPUStat: field 2 (float) totalPct, field 5 (int32) numThreads - @Suppress("MagicNumber") private fun decodeCpuStat(bytes: ByteArray): Pair { val input = CodedInputStream.newInstance(bytes) var totalPct = 0f @@ -320,8 +337,8 @@ object ProcessAgentPayloadDecoder { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (2 shl 3) or 5 -> totalPct = input.readFloat() - (5 shl 3) or 0 -> numThreads = input.readInt32() + (2 shl PROTO_FIELD_SHIFT) or PROTO_WIRE_FLOAT -> totalPct = input.readFloat() + (PROTO_FIELD_5 shl PROTO_FIELD_SHIFT) or 0 -> numThreads = input.readInt32() else -> input.skipField(tag) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/decompression/SketchPayloadDecoder.kt b/backend/src/main/kotlin/com/moneat/datadog/decompression/SketchPayloadDecoder.kt index c5eef9556..883addd93 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/decompression/SketchPayloadDecoder.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/decompression/SketchPayloadDecoder.kt @@ -45,9 +45,16 @@ import com.moneat.datadog.models.DatadogSketchPoint * repeated uint32 n = 8; // DDSketch bucket counts (packed) * } */ -@Suppress("MagicNumber") object SketchPayloadDecoder { + private const val PROTO_FIELD_SHIFT = 3 + private const val PROTO_FIELD_3 = 3 + private const val PROTO_FIELD_4 = 4 + private const val PROTO_FIELD_5 = 5 + private const val PROTO_FIELD_6 = 6 + private const val PROTO_FIELD_7 = 7 + private const val PROTO_FIELD_8 = 8 + fun decode(proto: ByteArray): DatadogSketchPayload { val input = CodedInputStream.newInstance(proto) val sketches = mutableListOf() @@ -56,7 +63,7 @@ object SketchPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1 (LEN): repeated Sketch sketches - (1 shl 3) or 2 -> sketches += decodeSketch(input.readByteArray()) + (1 shl PROTO_FIELD_SHIFT) or 2 -> sketches += decodeSketch(input.readByteArray()) else -> input.skipField(tag) } } @@ -74,13 +81,13 @@ object SketchPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1 (LEN): string metric - (1 shl 3) or 2 -> metric = input.readString() + (1 shl PROTO_FIELD_SHIFT) or 2 -> metric = input.readString() // field 2 (LEN): string host - (2 shl 3) or 2 -> host = input.readString() + (2 shl PROTO_FIELD_SHIFT) or 2 -> host = input.readString() // field 3 (LEN): repeated string tags - (3 shl 3) or 2 -> tags += input.readString() + (PROTO_FIELD_3 shl PROTO_FIELD_SHIFT) or 2 -> tags += input.readString() // field 4 (LEN): repeated Distribution - (4 shl 3) or 2 -> distributions += decodeDistribution(input.readByteArray()) + (PROTO_FIELD_4 shl PROTO_FIELD_SHIFT) or 2 -> distributions += decodeDistribution(input.readByteArray()) else -> input.skipField(tag) } } @@ -102,25 +109,25 @@ object SketchPayloadDecoder { when (val tag = input.readTag()) { 0 -> break // field 1 (VARINT): int64 ts - (1 shl 3) or 0 -> ts = input.readInt64() + (1 shl PROTO_FIELD_SHIFT) or 0 -> ts = input.readInt64() // field 2 (VARINT): int64 cnt - (2 shl 3) or 0 -> cnt = input.readInt64() + (2 shl PROTO_FIELD_SHIFT) or 0 -> cnt = input.readInt64() // field 3 (64-bit): double min - (3 shl 3) or 1 -> min = input.readDouble() + (PROTO_FIELD_3 shl PROTO_FIELD_SHIFT) or 1 -> min = input.readDouble() // field 4 (64-bit): double max - (4 shl 3) or 1 -> max = input.readDouble() + (PROTO_FIELD_4 shl PROTO_FIELD_SHIFT) or 1 -> max = input.readDouble() // field 5 (64-bit): double avg - (5 shl 3) or 1 -> avg = input.readDouble() + (PROTO_FIELD_5 shl PROTO_FIELD_SHIFT) or 1 -> avg = input.readDouble() // field 6 (64-bit): double sum - (6 shl 3) or 1 -> sum = input.readDouble() + (PROTO_FIELD_6 shl PROTO_FIELD_SHIFT) or 1 -> sum = input.readDouble() // field 7 (LEN): packed repeated sint32 k (zigzag) - (7 shl 3) or 2 -> k += decodePackedSint32(input.readByteArray()) + (PROTO_FIELD_7 shl PROTO_FIELD_SHIFT) or 2 -> k += decodePackedSint32(input.readByteArray()) // field 7 (VARINT): non-packed sint32 k (single value) - (7 shl 3) or 0 -> k += input.readSInt32() + (PROTO_FIELD_7 shl PROTO_FIELD_SHIFT) or 0 -> k += input.readSInt32() // field 8 (LEN): packed repeated uint32 n - (8 shl 3) or 2 -> n += decodePackedUint32(input.readByteArray()) + (PROTO_FIELD_8 shl PROTO_FIELD_SHIFT) or 2 -> n += decodePackedUint32(input.readByteArray()) // field 8 (VARINT): non-packed uint32 n (single value) - (8 shl 3) or 0 -> n += input.readUInt32() + (PROTO_FIELD_8 shl PROTO_FIELD_SHIFT) or 0 -> n += input.readUInt32() else -> input.skipField(tag) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogDogStatsDRoutes.kt b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogDogStatsDRoutes.kt index d5a13536f..9fc0d18b3 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogDogStatsDRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogDogStatsDRoutes.kt @@ -33,6 +33,7 @@ private val logger = KotlinLogging.logger {} private const val DOGSTATSD_METRIC_SEPARATOR = '|' private const val DOGSTATSD_TAG_SEPARATOR = '#' +private const val MILLIS_TO_SECONDS = 1000.0 fun Route.datadogDogStatsDRoutes() { route("/dd") { @@ -119,7 +120,7 @@ internal fun parseDogStatsDLine( } } - val now = System.currentTimeMillis().toDouble() / 1000.0 + val now = System.currentTimeMillis().toDouble() / MILLIS_TO_SECONDS return DatadogMetricV1( metric = metricName, diff --git a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogHostRoutes.kt b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogHostRoutes.kt index 2e62b2d4d..bab2cf281 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogHostRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogHostRoutes.kt @@ -48,6 +48,8 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val PERCENTAGE_SCALE = 100.0 + private val json = Json { ignoreUnknownKeys = true isLenient = true @@ -326,19 +328,19 @@ fun Route.datadogHostQueryRoutes() { // cpu: sum user+system, capped at 100 val cpuUser = m["system.cpu.user"] ?: 0.0 val cpuSys = m["system.cpu.system"] ?: 0.0 - put("cpu_percent", minOf(cpuUser + cpuSys, 100.0)) + put("cpu_percent", minOf(cpuUser + cpuSys, PERCENTAGE_SCALE)) // mem: prefer pct_usable → derive used%, else skip val pctUsable = m["system.mem.pct_usable"] if (pctUsable != null) { - put("mem_percent", (1.0 - pctUsable) * 100.0) + put("mem_percent", (1.0 - pctUsable) * PERCENTAGE_SCALE) } else { val used = m["system.mem.used"] val total = m["system.mem.total"] if (used != null && total != null && total > 0) { - put("mem_percent", (used / total) * 100.0) + put("mem_percent", (used / total) * PERCENTAGE_SCALE) } } - m["system.disk.in_use"]?.let { put("disk_percent", it * 100.0) } + m["system.disk.in_use"]?.let { put("disk_percent", it * PERCENTAGE_SCALE) } m["system.net.bytes_rcvd"]?.let { put("net_recv_bytes", it) } m["system.net.bytes_sent"]?.let { put("net_sent_bytes", it) } m["system.load.1"]?.let { put("load_1", it) } diff --git a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogInfraQueryRoutes.kt b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogInfraQueryRoutes.kt index 284a55e1a..91edf4fa5 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogInfraQueryRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/routes/DatadogInfraQueryRoutes.kt @@ -43,6 +43,7 @@ private val json = Json { ignoreUnknownKeys = true } private const val DEFAULT_LIMIT = 50 private const val MAX_LIMIT = 200 +private const val LOG_BODY_MAX_LEN = 200 fun Route.datadogInfraQueryRoutes() { route("/v1/infra") { @@ -304,7 +305,7 @@ private suspend fun executeCount(sql: String): Long { val resp = ClickHouseClient.execute(sql) val body = resp.bodyAsText() if (resp.isClickHouseError(body)) { - logger.warn { "ClickHouse query failed (${resp.status.value}): ${body.take(200)}" } + logger.warn { "ClickHouse query failed (${resp.status.value}): ${body.take(LOG_BODY_MAX_LEN)}" } return 0L } return body.trim().lines().firstOrNull()?.let { @@ -322,7 +323,7 @@ private suspend fun executeRows( val resp = ClickHouseClient.execute(sql) val body = resp.bodyAsText() if (resp.isClickHouseError(body)) { - logger.warn { "ClickHouse query failed (${resp.status.value}): ${body.take(200)}" } + logger.warn { "ClickHouse query failed (${resp.status.value}): ${body.take(LOG_BODY_MAX_LEN)}" } return emptyList() } return body.trim() diff --git a/backend/src/main/kotlin/com/moneat/datadog/security/SecurityQueryRoutes.kt b/backend/src/main/kotlin/com/moneat/datadog/security/SecurityQueryRoutes.kt index 9ead7cb3e..a1813d88e 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/security/SecurityQueryRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/security/SecurityQueryRoutes.kt @@ -43,6 +43,8 @@ private val json = Json { ignoreUnknownKeys = true } private const val DEFAULT_LIMIT = 50 private const val MAX_LIMIT = 200 +private const val LOG_SQL_MAX_LEN = 200 +private const val LOG_BODY_MAX_LEN = 300 fun Route.securityQueryRoutes() { route("/v1/security") { @@ -240,8 +242,10 @@ private suspend fun executeCount(sql: String): Long { val resp = ClickHouseClient.execute(sql) val body = resp.bodyAsText() if (resp.isClickHouseError(body)) { - logger.error { "ClickHouse error in executeCount. SQL: ${sql.take(200)} Body: ${body.take(300)}" } - throw IllegalStateException("ClickHouse query error: ${body.take(300)}") + logger.error { + "ClickHouse error in executeCount. SQL: ${sql.take(LOG_SQL_MAX_LEN)} Body: ${body.take(LOG_BODY_MAX_LEN)}" + } + throw IllegalStateException("ClickHouse query error: ${body.take(LOG_BODY_MAX_LEN)}") } return body.trim().lines().firstOrNull()?.let { json.parseToJsonElement(it).jsonObject["cnt"] @@ -256,8 +260,10 @@ private suspend fun executeRows( val resp = ClickHouseClient.execute(sql) val body = resp.bodyAsText() if (resp.isClickHouseError(body)) { - logger.error { "ClickHouse error in executeRows. SQL: ${sql.take(200)} Body: ${body.take(300)}" } - throw IllegalStateException("ClickHouse query error: ${body.take(300)}") + logger.error { + "ClickHouse error in executeRows. SQL: ${sql.take(LOG_SQL_MAX_LEN)} Body: ${body.take(LOG_BODY_MAX_LEN)}" + } + throw IllegalStateException("ClickHouse query error: ${body.take(LOG_BODY_MAX_LEN)}") } return body.trim().lines().filter { it.isNotBlank() }.map { line -> mapper(json.parseToJsonElement(line).jsonObject) diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogEventService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogEventService.kt index 23bdbe3e5..2e85dc0a8 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogEventService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogEventService.kt @@ -30,6 +30,8 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} private const val EVENT_QUEUE_KEY = "moneat:infra_events:queue" +private const val ERROR_BODY_MAX_LEN = 600 +private const val MILLIS_PER_SECOND = 1000L @Serializable data class QueuedEventBatch( @@ -74,7 +76,6 @@ object DatadogEventService { encodeDefaults = true } - @Suppress("MagicNumber") fun mapEvents( organizationId: Long, events: List @@ -83,7 +84,7 @@ object DatadogEventService { val entries = events.map { event -> val tags = DatadogMetricService.parseDdTagList(event.tags) val timestampMs = if (event.dateHappened != null) { - event.dateHappened * 1000 + event.dateHappened * MILLIS_PER_SECOND } else { now } @@ -113,7 +114,6 @@ object DatadogEventService { ) } - @Suppress("MagicNumber") fun mapServiceChecks( organizationId: Long, checks: List @@ -122,7 +122,7 @@ object DatadogEventService { val entries = checks.map { sc -> val tags = DatadogMetricService.parseDdTagList(sc.tags) val timestampMs = if (sc.timestamp != null) { - sc.timestamp * 1000 + sc.timestamp * MILLIS_PER_SECOND } else { now } @@ -191,7 +191,7 @@ object DatadogEventService { if (!response.status.isSuccess()) { val errorBody = response.bodyAsText() throw IllegalStateException( - "Failed to insert DD events: ${errorBody.take(600)}" + "Failed to insert DD events: ${errorBody.take(ERROR_BODY_MAX_LEN)}" ) } } @@ -235,7 +235,7 @@ object DatadogEventService { val errorBody = response.bodyAsText() throw IllegalStateException( "Failed to insert DD service checks: " + - errorBody.take(600) + errorBody.take(ERROR_BODY_MAX_LEN) ) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogInfraService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogInfraService.kt index 8a552416c..28cdc327b 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogInfraService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogInfraService.kt @@ -31,6 +31,7 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} private const val INFRA_QUEUE_KEY = "moneat:infra:queue" +private const val ERROR_BODY_MAX_LEN = 600 @Serializable data class QueuedInfraBatch( @@ -343,7 +344,7 @@ object DatadogInfraService { if (!response.status.isSuccess()) { val errorBody = response.bodyAsText() throw IllegalStateException( - "Failed to insert $label: ${errorBody.take(600)}" + "Failed to insert $label: ${errorBody.take(ERROR_BODY_MAX_LEN)}" ) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogJfrFlamegraphService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogJfrFlamegraphService.kt index 158c9811f..8edc78b17 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogJfrFlamegraphService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogJfrFlamegraphService.kt @@ -32,6 +32,8 @@ private val logger = KotlinLogging.logger {} object DatadogJfrFlamegraphService { + private const val JFR_MAGIC_LAST_BYTE_INDEX = 3 + private data class MutableFrame( val name: String, var value: Long = 0L, @@ -119,7 +121,7 @@ object DatadogJfrFlamegraphService { data[0] == JFR_MAGIC[0] && data[1] == JFR_MAGIC[1] && data[2] == JFR_MAGIC[2] && - data[3] == JFR_MAGIC[3] + data[JFR_MAGIC_LAST_BYTE_INDEX] == JFR_MAGIC[JFR_MAGIC_LAST_BYTE_INDEX] } private fun toJson(frame: MutableFrame): JsonObject = buildJsonObject { diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogMetricService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogMetricService.kt index 4ffef29ef..3e4945eb9 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogMetricService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogMetricService.kt @@ -31,6 +31,8 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} private const val METRIC_QUEUE_KEY = "moneat:metrics:queue" +private const val ERROR_BODY_MAX_LEN = 600 +private const val MILLIS_PER_SECOND = 1000L @Serializable data class QueuedMetricBatch( @@ -100,8 +102,7 @@ object DatadogMetricService { return series.points.mapNotNull { point -> if (point.size < 2) return@mapNotNull null - @Suppress("MagicNumber") - val timestampMs = (point[0] * 1000).toLong() + val timestampMs = (point[0] * MILLIS_PER_SECOND).toLong() val value = point[1] QueuedMetricEntry( name = series.metric, @@ -123,10 +124,9 @@ object DatadogMetricService { val sketches = payload.sketches.flatMap { sketch -> val tags = parseDdTagList(sketch.tags) sketch.distributions.map { dist -> - @Suppress("MagicNumber") QueuedSketchEntry( name = sketch.metric, - timestampMs = dist.ts * 1000, + timestampMs = dist.ts * MILLIS_PER_SECOND, host = sketch.host, tags = tags, count = dist.cnt, @@ -195,7 +195,7 @@ object DatadogMetricService { if (!response.status.isSuccess()) { val errorBody = response.bodyAsText() throw IllegalStateException( - "Failed to insert DD metrics: ${errorBody.take(600)}" + "Failed to insert DD metrics: ${errorBody.take(ERROR_BODY_MAX_LEN)}" ) } } @@ -238,7 +238,7 @@ object DatadogMetricService { if (!response.status.isSuccess()) { val errorBody = response.bodyAsText() throw IllegalStateException( - "Failed to insert DD sketches: ${errorBody.take(600)}" + "Failed to insert DD sketches: ${errorBody.take(ERROR_BODY_MAX_LEN)}" ) } } diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogPprofFlamegraphService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogPprofFlamegraphService.kt index e48c253a9..74ef1cf25 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/DatadogPprofFlamegraphService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/DatadogPprofFlamegraphService.kt @@ -27,6 +27,15 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} +private const val HEX_RADIX = 16 +private const val PROTO_TAG_SHIFT = 3 +private const val PROTO_FIELD_3 = 3 +private const val PROTO_FIELD_4 = 4 +private const val PROTO_FIELD_5 = 5 +private const val PROTO_FIELD_6 = 6 +private const val PROTO_FIELD_14 = 14 +private const val JFR_MAGIC_LAST_INDEX = 3 + object DatadogPprofFlamegraphService { data class MutableFrame( @@ -209,7 +218,7 @@ object DatadogPprofFlamegraphService { val mapping = profile.mappingsById[location.mappingId] val file = mapping?.let { stringAt(profile.strings, it.filenameIdx) }.orEmpty() val buildId = mapping?.let { stringAt(profile.strings, it.buildIdIdx) }.orEmpty() - val address = "0x${location.address.toString(16)}" + val address = "0x${location.address.toString(HEX_RADIX)}" return when { file.isNotBlank() && buildId.isNotBlank() -> "$file@$buildId:$address" file.isNotBlank() -> "$file:$address" @@ -249,25 +258,25 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 2 -> sampleTypes += decodeValueType(input.readByteArray()) - (2 shl 3) or 2 -> samples += decodeSample(input.readByteArray()) - (3 shl 3) or 2 -> { + (1 shl PROTO_TAG_SHIFT) or 2 -> sampleTypes += decodeValueType(input.readByteArray()) + (2 shl PROTO_TAG_SHIFT) or 2 -> samples += decodeSample(input.readByteArray()) + (PROTO_FIELD_3 shl PROTO_TAG_SHIFT) or 2 -> { val mapping = decodeMapping(input.readByteArray()) mappings[mapping.id] = mapping } - (4 shl 3) or 2 -> { + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 2 -> { val location = decodeLocation(input.readByteArray()) if (location.id != 0L) { locations[location.id] = location } locationList += location } - (5 shl 3) or 2 -> { + (PROTO_FIELD_5 shl PROTO_TAG_SHIFT) or 2 -> { val fn = decodeFunction(input.readByteArray()) functions[fn.id] = fn } - (6 shl 3) or 2 -> strings += input.readString() - (14 shl 3) or 0 -> defaultSampleType = input.readInt64() + (PROTO_FIELD_6 shl PROTO_TAG_SHIFT) or 2 -> strings += input.readString() + (PROTO_FIELD_14 shl PROTO_TAG_SHIFT) or 0 -> defaultSampleType = input.readInt64() else -> input.skipField(tag) } } @@ -291,8 +300,8 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> typeIdx = input.readInt64() - (2 shl 3) or 0 -> unitIdx = input.readInt64() + (1 shl PROTO_TAG_SHIFT) or 0 -> typeIdx = input.readInt64() + (2 shl PROTO_TAG_SHIFT) or 0 -> unitIdx = input.readInt64() else -> input.skipField(tag) } } @@ -308,12 +317,12 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> locationIds += input.readUInt64() - (1 shl 3) or 2 -> readPackedUInt64(input.readByteArray(), locationIds) - (2 shl 3) or 0 -> values += input.readInt64() - (2 shl 3) or 2 -> readPackedInt64(input.readByteArray(), values) - (4 shl 3) or 0 -> locationIdx += input.readUInt64() - (4 shl 3) or 2 -> readPackedUInt64(input.readByteArray(), locationIdx) + (1 shl PROTO_TAG_SHIFT) or 0 -> locationIds += input.readUInt64() + (1 shl PROTO_TAG_SHIFT) or 2 -> readPackedUInt64(input.readByteArray(), locationIds) + (2 shl PROTO_TAG_SHIFT) or 0 -> values += input.readInt64() + (2 shl PROTO_TAG_SHIFT) or 2 -> readPackedInt64(input.readByteArray(), values) + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 0 -> locationIdx += input.readUInt64() + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 2 -> readPackedUInt64(input.readByteArray(), locationIdx) else -> input.skipField(tag) } } @@ -333,9 +342,9 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> id = input.readUInt64() - (5 shl 3) or 0 -> filenameIdx = input.readInt64() - (6 shl 3) or 0 -> buildIdIdx = input.readInt64() + (1 shl PROTO_TAG_SHIFT) or 0 -> id = input.readUInt64() + (PROTO_FIELD_5 shl PROTO_TAG_SHIFT) or 0 -> filenameIdx = input.readInt64() + (PROTO_FIELD_6 shl PROTO_TAG_SHIFT) or 0 -> buildIdIdx = input.readInt64() else -> input.skipField(tag) } } @@ -352,10 +361,10 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> id = input.readUInt64() - (2 shl 3) or 0 -> mappingId = input.readUInt64() - (3 shl 3) or 0 -> address = input.readUInt64() - (4 shl 3) or 2 -> lines += decodeLine(input.readByteArray()) + (1 shl PROTO_TAG_SHIFT) or 0 -> id = input.readUInt64() + (2 shl PROTO_TAG_SHIFT) or 0 -> mappingId = input.readUInt64() + (PROTO_FIELD_3 shl PROTO_TAG_SHIFT) or 0 -> address = input.readUInt64() + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 2 -> lines += decodeLine(input.readByteArray()) else -> input.skipField(tag) } } @@ -374,8 +383,8 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> functionId = input.readUInt64() - (2 shl 3) or 0 -> line = input.readInt64() + (1 shl PROTO_TAG_SHIFT) or 0 -> functionId = input.readUInt64() + (2 shl PROTO_TAG_SHIFT) or 0 -> line = input.readInt64() else -> input.skipField(tag) } } @@ -392,10 +401,10 @@ object DatadogPprofFlamegraphService { while (!input.isAtEnd) { when (val tag = input.readTag()) { 0 -> break - (1 shl 3) or 0 -> id = input.readUInt64() - (2 shl 3) or 0 -> nameIdx = input.readInt64() - (3 shl 3) or 0 -> systemNameIdx = input.readInt64() - (4 shl 3) or 0 -> filenameIdx = input.readInt64() + (1 shl PROTO_TAG_SHIFT) or 0 -> id = input.readUInt64() + (2 shl PROTO_TAG_SHIFT) or 0 -> nameIdx = input.readInt64() + (PROTO_FIELD_3 shl PROTO_TAG_SHIFT) or 0 -> systemNameIdx = input.readInt64() + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 0 -> filenameIdx = input.readInt64() else -> input.skipField(tag) } } @@ -436,7 +445,7 @@ object DatadogPprofFlamegraphService { data[0] == JFR_MAGIC[0] && data[1] == JFR_MAGIC[1] && data[2] == JFR_MAGIC[2] && - data[3] == JFR_MAGIC[3] + data[JFR_MAGIC_LAST_INDEX] == JFR_MAGIC[JFR_MAGIC_LAST_INDEX] } private fun toJson(frame: MutableFrame): JsonObject = buildJsonObject { diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/ProfileIngestionService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/ProfileIngestionService.kt index 66d7e9e05..fce0d6bf3 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/ProfileIngestionService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/ProfileIngestionService.kt @@ -45,6 +45,9 @@ object ProfileIngestionService { private val json = Json { ignoreUnknownKeys = true } private val usageTracking = UsageTrackingService.instance + private const val NANOS_PER_MILLI = 1_000_000L + private const val MIN_PROFILE_PARTS = 3 + private data class LockEntry(val mutex: Mutex = Mutex(), val refs: AtomicInteger = AtomicInteger(0)) private val sessionLocks = ConcurrentHashMap() @@ -163,7 +166,7 @@ object ProfileIngestionService { runtime.substringBefore(" ").trim() } - val durationNs = (endMs - startMs) * 1_000_000 + val durationNs = (endMs - startMs) * NANOS_PER_MILLI val insert = """ INSERT INTO `$clickhouseDb`.profiles ( @@ -359,7 +362,7 @@ object ProfileIngestionService { val result = ClickHouseClient.executeWithFormat(query, "") val line = result.trim().takeIf { it.isNotBlank() } ?: return null val parts = line.split("\t") - return if (parts.size >= 3) { + return if (parts.size >= MIN_PROFILE_PARTS) { ProfileMeta(parts[0], parts[1], parts[2]) } else if (parts.size >= 2) { ProfileMeta(parts[0], parts[1], "datadog") diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/ProfileStorageService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/ProfileStorageService.kt index acb200934..41a2c26b2 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/ProfileStorageService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/ProfileStorageService.kt @@ -184,7 +184,7 @@ object ProfileStorageService { private fun sanitizeFilename(name: String): String { val sanitized = name.replace(Regex("[^a-zA-Z0-9._-]"), "_") - if (sanitized.isBlank()) return "profile_${UUID.randomUUID().toString().take(8)}" + if (sanitized.isBlank()) return "profile_${UUID.randomUUID().toString().take(PROFILE_FALLBACK_ID_LENGTH)}" return sanitized } @@ -220,4 +220,5 @@ object ProfileStorageService { private const val GZIP_MAGIC_0: Byte = 0x1f private const val GZIP_MAGIC_1: Byte = 0x8b.toByte() + private const val PROFILE_FALLBACK_ID_LENGTH = 8 } diff --git a/backend/src/main/kotlin/com/moneat/datadog/services/TraceIngestionService.kt b/backend/src/main/kotlin/com/moneat/datadog/services/TraceIngestionService.kt index 673ef44b3..e520245ca 100644 --- a/backend/src/main/kotlin/com/moneat/datadog/services/TraceIngestionService.kt +++ b/backend/src/main/kotlin/com/moneat/datadog/services/TraceIngestionService.kt @@ -57,6 +57,20 @@ private const val CANONICAL_TRACE_ID_SQL = private const val MAX_META_VALUE_LENGTH = 5000 private const val DEFAULT_QUERY_LIMIT = 50 private const val MAX_QUERY_LIMIT = 200 +private const val HEX_RADIX = 16 +private const val NANOSECONDS_PER_MILLISECOND = 1_000_000L +private const val PROTO_TAG_SHIFT = 3 +private const val PROTO_FIELD_3 = 3 +private const val PROTO_FIELD_4 = 4 +private const val PROTO_FIELD_5 = 5 +private const val PROTO_FIELD_6 = 6 +private const val PROTO_FIELD_7 = 7 +private const val PROTO_FIELD_8 = 8 +private const val PROTO_FIELD_9 = 9 +private const val PROTO_FIELD_10 = 10 +private const val PROTO_FIELD_11 = 11 +private const val PROTO_FIELD_12 = 12 +private const val PROTO_FIELD_14 = 14 object TraceIngestionService { private val clickhouseDb by lazy { ClickHouseClient.getDatabase() } @@ -131,6 +145,11 @@ object TraceIngestionService { ?: hostname val spanEnv = span.meta["env"] ?: env val ver = span.meta["version"] ?: appVersion + val parentIdHex = if (span.parentId != 0UL) { + java.lang.Long.toUnsignedString(span.parentId.toLong(), HEX_RADIX) + } else { + "" + } """( ${span.spanId}, @@ -152,9 +171,9 @@ object TraceIngestionService { '${escapeSql(host)}', '${escapeSql(spanEnv)}', '${escapeSql(ver)}', - '${java.lang.Long.toUnsignedString(span.traceId.toLong(), 16)}', - '${java.lang.Long.toUnsignedString(span.spanId.toLong(), 16)}', - '${if (span.parentId != 0UL) java.lang.Long.toUnsignedString(span.parentId.toLong(), 16) else ""}', + '${java.lang.Long.toUnsignedString(span.traceId.toLong(), HEX_RADIX)}', + '${java.lang.Long.toUnsignedString(span.spanId.toLong(), HEX_RADIX)}', + '$parentIdHex', 'datadog' )""" } @@ -199,7 +218,7 @@ object TraceIngestionService { if (entries.isEmpty()) return val rows = entries.joinToString(",\n") { (bucket, entry) -> - val startMs = bucket.start / 1_000_000 // ns to ms + val startMs = bucket.start / NANOSECONDS_PER_MILLISECOND // ns to ms """( $organizationId, '${escapeSql(entry.service)}', @@ -745,13 +764,9 @@ object TraceIngestionService { fun parseProtobufAgentPayload(bytes: ByteArray): List> { val input = CodedInputStream.newInstance(bytes) val traces = mutableListOf>() - var hostname = "" - var env = "" while (!input.isAtEnd) { when (val tag = input.readTag()) { - (1 shl 3) or 2 -> hostname = input.readString() - (2 shl 3) or 2 -> env = input.readString() - (5 shl 3) or 2 -> { + (PROTO_FIELD_5 shl PROTO_TAG_SHIFT) or 2 -> { val payloadBytes = input.readBytes().toByteArray() traces.addAll(decodeTracerPayload(payloadBytes)) } @@ -767,7 +782,7 @@ object TraceIngestionService { val traces = mutableListOf>() while (!input.isAtEnd) { when (val tag = input.readTag()) { - (6 shl 3) or 2 -> { + (PROTO_FIELD_6 shl PROTO_TAG_SHIFT) or 2 -> { val chunkBytes = input.readBytes().toByteArray() val spans = decodeTraceChunk(chunkBytes) if (spans.isNotEmpty()) traces.add(spans) @@ -784,7 +799,7 @@ object TraceIngestionService { val spans = mutableListOf() while (!input.isAtEnd) { when (val tag = input.readTag()) { - (3 shl 3) or 2 -> { + (PROTO_FIELD_3 shl PROTO_TAG_SHIFT) or 2 -> { val spanBytes = input.readBytes().toByteArray() spans.add(decodeProtobufSpan(spanBytes)) } @@ -814,24 +829,24 @@ object TraceIngestionService { val metrics = mutableMapOf() while (!input.isAtEnd) { when (val tag = input.readTag()) { - (1 shl 3) or 2 -> service = input.readString() - (2 shl 3) or 2 -> name = input.readString() - (3 shl 3) or 2 -> resource = input.readString() - (4 shl 3) or 0 -> traceId = input.readUInt64().toULong() - (5 shl 3) or 0 -> spanId = input.readUInt64().toULong() - (6 shl 3) or 0 -> parentId = input.readUInt64().toULong() - (7 shl 3) or 0 -> start = input.readInt64() - (8 shl 3) or 0 -> duration = input.readInt64() - (9 shl 3) or 0 -> error = input.readInt32() - (10 shl 3) or 2 -> { + (1 shl PROTO_TAG_SHIFT) or 2 -> service = input.readString() + (2 shl PROTO_TAG_SHIFT) or 2 -> name = input.readString() + (PROTO_FIELD_3 shl PROTO_TAG_SHIFT) or 2 -> resource = input.readString() + (PROTO_FIELD_4 shl PROTO_TAG_SHIFT) or 0 -> traceId = input.readUInt64().toULong() + (PROTO_FIELD_5 shl PROTO_TAG_SHIFT) or 0 -> spanId = input.readUInt64().toULong() + (PROTO_FIELD_6 shl PROTO_TAG_SHIFT) or 0 -> parentId = input.readUInt64().toULong() + (PROTO_FIELD_7 shl PROTO_TAG_SHIFT) or 0 -> start = input.readInt64() + (PROTO_FIELD_8 shl PROTO_TAG_SHIFT) or 0 -> duration = input.readInt64() + (PROTO_FIELD_9 shl PROTO_TAG_SHIFT) or 0 -> error = input.readInt32() + (PROTO_FIELD_10 shl PROTO_TAG_SHIFT) or 2 -> { val (k, v) = decodeStringStringEntry(input.readBytes().toByteArray()) meta[k] = v } - (11 shl 3) or 2 -> { + (PROTO_FIELD_11 shl PROTO_TAG_SHIFT) or 2 -> { val (k, v) = decodeStringDoubleEntry(input.readBytes().toByteArray()) metrics[k] = v } - (12 shl 3) or 2 -> type = input.readString() + (PROTO_FIELD_12 shl PROTO_TAG_SHIFT) or 2 -> type = input.readString() else -> input.skipField(tag) } } @@ -850,8 +865,8 @@ object TraceIngestionService { var value = "" while (!input.isAtEnd) { when (val tag = input.readTag()) { - (1 shl 3) or 2 -> key = input.readString() - (2 shl 3) or 2 -> value = input.readString() + (1 shl PROTO_TAG_SHIFT) or 2 -> key = input.readString() + (2 shl PROTO_TAG_SHIFT) or 2 -> value = input.readString() else -> input.skipField(tag) } } @@ -865,8 +880,8 @@ object TraceIngestionService { var value = 0.0 while (!input.isAtEnd) { when (val tag = input.readTag()) { - (1 shl 3) or 2 -> key = input.readString() - (2 shl 3) or 1 -> value = input.readDouble() + (1 shl PROTO_TAG_SHIFT) or 2 -> key = input.readString() + (2 shl PROTO_TAG_SHIFT) or 1 -> value = input.readDouble() else -> input.skipField(tag) } } diff --git a/backend/src/main/kotlin/com/moneat/enterprise/license/LicenseValidator.kt b/backend/src/main/kotlin/com/moneat/enterprise/license/LicenseValidator.kt index 5dd9df308..da487e3ab 100644 --- a/backend/src/main/kotlin/com/moneat/enterprise/license/LicenseValidator.kt +++ b/backend/src/main/kotlin/com/moneat/enterprise/license/LicenseValidator.kt @@ -112,11 +112,13 @@ class LicenseValidator(publicKeyPem: String = PRODUCTION_PUBLIC_KEY_PEM) { // Base64url strings from openssl may omit padding — add it back for Java's decoder private fun padBase64Url(s: String): String { - val pad = (4 - s.length % 4) % 4 + val pad = (BASE64_PADDING_SIZE - s.length % BASE64_PADDING_SIZE) % BASE64_PADDING_SIZE return s + "=".repeat(pad) } companion object { + private const val BASE64_PADDING_SIZE = 4 + // Public key corresponding to the Moneat license signing private key. // The private key is held only by Moneat and is never distributed. private val PRODUCTION_PUBLIC_KEY_PEM = """ diff --git a/backend/src/main/kotlin/com/moneat/events/models/SentryModels.kt b/backend/src/main/kotlin/com/moneat/events/models/SentryModels.kt index c265dbb72..f77b44b50 100644 --- a/backend/src/main/kotlin/com/moneat/events/models/SentryModels.kt +++ b/backend/src/main/kotlin/com/moneat/events/models/SentryModels.kt @@ -39,6 +39,8 @@ import kotlinx.serialization.json.long import java.time.format.DateTimeParseException import java.util.* +private const val NANOS_PER_SECOND = 1_000_000_000.0 + @Serializable data class SentryEnvelope( val eventId: String, @@ -300,7 +302,7 @@ object FlexibleTimestampSerializer : KSerializer { val isoString = element.contentOrNull ?: return null try { val instant = java.time.Instant.parse(isoString) - instant.epochSecond.toDouble() + instant.nano / 1_000_000_000.0 + instant.epochSecond.toDouble() + instant.nano / NANOS_PER_SECOND } catch (_: DateTimeParseException) { null } diff --git a/backend/src/main/kotlin/com/moneat/events/repositories/ProjectRepositoryImpl.kt b/backend/src/main/kotlin/com/moneat/events/repositories/ProjectRepositoryImpl.kt index 231102b19..f4437f7de 100644 --- a/backend/src/main/kotlin/com/moneat/events/repositories/ProjectRepositoryImpl.kt +++ b/backend/src/main/kotlin/com/moneat/events/repositories/ProjectRepositoryImpl.kt @@ -51,6 +51,10 @@ private val logger = KotlinLogging.logger {} class ProjectRepositoryImpl( private val timestampRetentionClause: (String, Int, Long?) -> String ) : ProjectRepository { + companion object { + private const val HTTP_SUCCESS_MIN = 200 + private const val HTTP_SUCCESS_MAX = 299 + } private val clickhouseDb: String get() = ClickHouseClient.getDatabase() private val json = kotlinx.serialization.json.Json { ignoreUnknownKeys = true } @@ -222,7 +226,11 @@ class ProjectRepositoryImpl( return suspendRunCatching { val response = ClickHouseClient.execute(query) val body = response.bodyAsText() - if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) return 0 + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX || + body.trimStart().startsWith("Code:") + ) { + return 0 + } if (body.isBlank()) return 0 val obj = json.parseToJsonElement(body.lines().first()).jsonObject obj["total"]?.jsonPrimitive?.long ?: 0 diff --git a/backend/src/main/kotlin/com/moneat/events/routes/ApiRoutes.kt b/backend/src/main/kotlin/com/moneat/events/routes/ApiRoutes.kt index a96fd0366..301da637a 100644 --- a/backend/src/main/kotlin/com/moneat/events/routes/ApiRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/events/routes/ApiRoutes.kt @@ -79,6 +79,12 @@ import org.koin.core.context.GlobalContext import kotlin.time.Clock import com.moneat.utils.suspendRunCatching +private const val DEFAULT_PAGE_LIMIT = 25 +private const val DEFAULT_EVENTS_LIMIT = 50 +private const val DEFAULT_TRANSACTIONS_LIMIT = 20 +private const val DEFAULT_REPLAYS_LIMIT = 10 +private const val DEFAULT_ALERT_FREQUENCY_MINUTES = 30 + fun Route.apiRoutes() { val koin = GlobalContext.get() val dashboardService = koin.get() @@ -555,7 +561,7 @@ fun Route.apiRoutes() { } val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 25 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_PAGE_LIMIT val status = call.request.queryParameters["status"] val issues = dashboardService.getIssues(projectId, page, limit, status, demoEpochMs) @@ -603,7 +609,7 @@ fun Route.apiRoutes() { val demoEpochMs = call.getDemoEpochMs() val issueId = call.parameters["issueId"] - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 50 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_EVENTS_LIMIT val projectId = call.request.queryParameters["projectId"]?.toLongOrNull() if (issueId == null) { @@ -634,7 +640,7 @@ fun Route.apiRoutes() { val demoEpochMs = call.getDemoEpochMs() val issueId = call.parameters["issueId"] - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 20 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_TRANSACTIONS_LIMIT val projectId = call.request.queryParameters["projectId"]?.toLongOrNull() if (issueId == null) { @@ -836,7 +842,7 @@ fun Route.apiRoutes() { return@get } - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 20 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_TRANSACTIONS_LIMIT val relatedErrors = dashboardService.getRelatedErrorsForTransaction(eventId, limit) call.respond(relatedErrors) } @@ -912,7 +918,7 @@ fun Route.apiRoutes() { } val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 25 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_PAGE_LIMIT val environment = call.request.queryParameters["environment"] val period = call.request.queryParameters["period"] ?: "7d" @@ -1008,7 +1014,7 @@ fun Route.apiRoutes() { } val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 25 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_PAGE_LIMIT val status = call.request.queryParameters["status"] val feedback = dashboardService.getFeedback(projectId, page, limit, status, demoEpochMs) @@ -1104,7 +1110,7 @@ fun Route.apiRoutes() { return@get } - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 10 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_REPLAYS_LIMIT val replays = dashboardService.getReplaysForIssue(issueId, limit) call.respond(replays) } @@ -1249,7 +1255,8 @@ fun Route.apiRoutes() { ?: existing?.get(NotificationPreferences.weekly_summary) ?: true val alertFrequency = (request["alertFrequencyMinutes"] as? Number)?.toInt() - ?: existing?.get(NotificationPreferences.alert_frequency_minutes) ?: 30 + ?: existing?.get(NotificationPreferences.alert_frequency_minutes) + ?: DEFAULT_ALERT_FREQUENCY_MINUTES // NOSONAR kotlin:S6619 if (existing != null) { NotificationPreferences.update({ @@ -1316,7 +1323,8 @@ fun Route.apiRoutes() { ?: existing?.get(NotificationPreferences.weekly_summary) ?: true val alertFrequency = (request["alertFrequencyMinutes"] as? Number)?.toInt() - ?: existing?.get(NotificationPreferences.alert_frequency_minutes) ?: 30 + ?: existing?.get(NotificationPreferences.alert_frequency_minutes) + ?: DEFAULT_ALERT_FREQUENCY_MINUTES // NOSONAR kotlin:S6619 if (existing != null) { NotificationPreferences.update({ diff --git a/backend/src/main/kotlin/com/moneat/events/routes/IngestRoutes.kt b/backend/src/main/kotlin/com/moneat/events/routes/IngestRoutes.kt index 6d036a774..10ba686fb 100644 --- a/backend/src/main/kotlin/com/moneat/events/routes/IngestRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/events/routes/IngestRoutes.kt @@ -41,9 +41,16 @@ import kotlinx.serialization.json.Json import mu.KotlinLogging import org.koin.core.context.GlobalContext import com.moneat.utils.suspendRunCatching +import java.security.MessageDigest private val logger = KotlinLogging.logger {} private val json = Json { ignoreUnknownKeys = true } +private const val SHA256_HEX_PREFIX_CHARS = 16 + +private fun sha256HexPrefix(bytes: ByteArray, maxHexChars: Int): String { + val digest = MessageDigest.getInstance("SHA-256").digest(bytes) + return digest.joinToString("") { b -> "%02x".format(b) }.take(maxHexChars) +} fun Route.ingestRoutes( eventService: EventService = GlobalContext.get().get(), @@ -98,7 +105,10 @@ fun Route.ingestRoutes( val decompressedBytes = DecompressionService.decompress(bodyBytes, contentEncoding) logger.debug { "Received envelope for project $projectId" } - logger.debug { "Envelope body:\n${decompressedBytes.decodeToString().take(500)}" } + logger.debug { + "Envelope payload (redacted): bytes=${decompressedBytes.size}, " + + "sha256_prefix=${sha256HexPrefix(decompressedBytes, SHA256_HEX_PREFIX_CHARS)}" + } val envelope = SentryEnvelope.parse(decompressedBytes) logger.debug { "Envelope parsed successfully, items: ${envelope.items.size}" } diff --git a/backend/src/main/kotlin/com/moneat/events/services/AccessService.kt b/backend/src/main/kotlin/com/moneat/events/services/AccessService.kt index 64e78a6f0..deeabe0c4 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/AccessService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/AccessService.kt @@ -31,6 +31,9 @@ import org.jetbrains.exposed.v1.jdbc.selectAll import org.jetbrains.exposed.v1.jdbc.transactions.transaction import com.moneat.utils.suspendRunCatching +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 + private val logger = KotlinLogging.logger {} class AccessService( @@ -98,7 +101,11 @@ class AccessService( return suspendRunCatching { val response = ClickHouseClient.execute(query) val body = response.bodyAsText() - if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) return null + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX || + body.trimStart().startsWith("Code:") + ) { + return null + } if (body.isBlank()) return null val obj = json.parseToJsonElement(body.lines().first()).jsonObject obj["project_id"]?.jsonPrimitive?.longOrNull?.takeIf { it != 0L } @@ -121,7 +128,11 @@ class AccessService( return suspendRunCatching { val response = ClickHouseClient.execute(query) val body = response.bodyAsText() - if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) return null + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX || + body.trimStart().startsWith("Code:") + ) { + return null + } if (body.isBlank()) return null val obj = json.parseToJsonElement(body.lines().first()).jsonObject obj["issue_id"]?.jsonPrimitive?.contentOrNull?.takeIf { it.isNotBlank() } diff --git a/backend/src/main/kotlin/com/moneat/events/services/DashboardQueryHelper.kt b/backend/src/main/kotlin/com/moneat/events/services/DashboardQueryHelper.kt index 281b29b68..510f5fc4f 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/DashboardQueryHelper.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/DashboardQueryHelper.kt @@ -102,7 +102,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "$context failed for $entityId: ${response.status} ${body.take(400)}" } + logger.error { "$context failed for $entityId: ${response.status} ${body.take(LOG_BODY_CHARS)}" } return null } if (body.isBlank()) return null @@ -126,7 +126,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "$errorContext failed: ${response.status} ${body.take(400)}" } + logger.error { "$errorContext failed: ${response.status} ${body.take(LOG_BODY_CHARS)}" } return null } body @@ -152,7 +152,7 @@ class DashboardQueryHelper( val body = response.bodyAsText() // ClickHouse returns error messages as plain text starting with "Code:" if (body.startsWith("Code:") && body.contains("DB::Exception")) { - logger.warn { "ClickHouse error: ${body.take(200)}" } + logger.warn { "ClickHouse error: ${body.take(LOG_SNIPPET_CHARS)}" } return null } return body @@ -165,8 +165,11 @@ class DashboardQueryHelper( val hexRegex = Regex("^[0-9a-f]{32}$") if (hexRegex.matches(trimmed)) { - return "${trimmed.substring(0, 8)}-${trimmed.substring(8, 12)}-" + - "${trimmed.substring(12, 16)}-${trimmed.substring(16, 20)}-${trimmed.substring(20)}" + return "${trimmed.substring(0, UUID_HEX_BLOCK1_END)}-" + + "${trimmed.substring(UUID_HEX_BLOCK1_END, UUID_HEX_BLOCK2_END)}-" + + "${trimmed.substring(UUID_HEX_BLOCK2_END, UUID_HEX_BLOCK3_END)}-" + + "${trimmed.substring(UUID_HEX_BLOCK3_END, UUID_HEX_BLOCK4_END)}-" + + trimmed.substring(UUID_HEX_BLOCK4_END) } return null @@ -236,7 +239,7 @@ class DashboardQueryHelper( fun demoNowClause(demoEpochMs: Long? = null): String { return if (demoEpochMs != null) { - "toDateTime64(${demoEpochMs / 1000.0}, 3)" + "toDateTime64(${demoEpochMs / MILLIS_PER_SECOND}, 3)" } else { "now()" } @@ -286,14 +289,16 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "Failed to execute scalar query: ${response.status} ${body.take(400)}" } + logger.error { "Failed to execute scalar query: ${response.status} ${body.take(LOG_BODY_CHARS)}" } return 0 } if (body.isBlank()) return 0 val obj = json.parseToJsonElement(body.lines().first()).jsonObject obj["total"]?.jsonPrimitive?.long ?: 0 }.getOrElse { e -> - logger.error(e) { "Scalar query failed (transport/parse): query=${query.take(200)}, returning 0" } + logger.error(e) { + "Scalar query failed (transport/parse): query=${query.take(LOG_SNIPPET_CHARS)}, returning 0" + } 0 } } @@ -306,7 +311,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "ClickHouse query failed: ${body.take(400)}" } + logger.error { "ClickHouse query failed: ${body.take(LOG_BODY_CHARS)}" } return emptyList() } body @@ -327,7 +332,10 @@ class DashboardQueryHelper( }.getOrElse { e -> logger.error( e - ) { "executeTimelineQuery failed (transport/parse): query=${query.take(200)}, returning emptyList" } + ) { + "executeTimelineQuery failed (transport/parse): " + + "query=${query.take(LOG_SNIPPET_CHARS)}, returning emptyList" + } emptyList() } } @@ -340,7 +348,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "ClickHouse query failed: ${body.take(400)}" } + logger.error { "ClickHouse query failed: ${body.take(LOG_BODY_CHARS)}" } return emptyList() } body @@ -367,7 +375,7 @@ class DashboardQueryHelper( }.getOrElse { e -> logger.error(e) { "executeSlowestTransactionsQuery failed (transport/parse): query=${query.take( - 200 + LOG_SNIPPET_CHARS )}, returning emptyList" } emptyList() @@ -383,7 +391,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "Failed to execute map query: ${response.status} ${body.take(400)}" } + logger.error { "Failed to execute map query: ${response.status} ${body.take(LOG_BODY_CHARS)}" } return emptyMap() } body @@ -396,13 +404,15 @@ class DashboardQueryHelper( val count = obj["count"]?.jsonPrimitive?.long ?: 0 key to count }.getOrElse { e -> - logger.error(e) { "Failed to parse map query line: ${line.take(200)}" } + logger.error(e) { "Failed to parse map query line: ${line.take(LOG_SNIPPET_CHARS)}" } null } } .toMap() }.getOrElse { e -> - logger.error(e) { "executeMapQuery failed (transport/parse): query=${query.take(200)}, returning emptyMap" } + logger.error(e) { + "executeMapQuery failed (transport/parse): query=${query.take(LOG_SNIPPET_CHARS)}, returning emptyMap" + } emptyMap() } } @@ -424,7 +434,7 @@ class DashboardQueryHelper( body.trimStart().startsWith("Code:") ) { if (response.status.value !in 200..299) { - logger.error { "$errorContext failed: ${response.status} ${body.take(400)}" } + logger.error { "$errorContext failed: ${response.status} ${body.take(LOG_BODY_CHARS)}" } } return emptyMap() } @@ -459,11 +469,11 @@ class DashboardQueryHelper( val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { logger.error { - "$errorContext failed: query=${query.take(200)}, status=${response.status}, " + - "body=${body.take(400)}" + "$errorContext failed: query=${query.take(LOG_SNIPPET_CHARS)}, status=${response.status}, " + + "body=${body.take(LOG_BODY_CHARS)}" } throw IllegalStateException( - "ClickHouse mutation failed: ${response.status} ${body.take(200)}" + "ClickHouse mutation failed: ${response.status} ${body.take(LOG_SNIPPET_CHARS)}" ) } } @@ -476,7 +486,7 @@ class DashboardQueryHelper( val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "Failed to execute top issues query: ${response.status} ${body.take(400)}" } + logger.error { "Failed to execute top issues query: ${response.status} ${body.take(LOG_BODY_CHARS)}" } return emptyList() } body @@ -491,15 +501,28 @@ class DashboardQueryHelper( count = obj["count"]?.jsonPrimitive?.long ?: 0 ) }.getOrElse { e -> - logger.error(e) { "Failed to parse top issues line: ${line.take(200)}" } + logger.error(e) { "Failed to parse top issues line: ${line.take(LOG_SNIPPET_CHARS)}" } null } } }.getOrElse { e -> logger.error( e - ) { "executeTopIssuesQuery failed (transport/parse): query=${query.take(200)}, returning emptyList" } + ) { + "executeTopIssuesQuery failed (transport/parse): " + + "query=${query.take(LOG_SNIPPET_CHARS)}, returning emptyList" + } emptyList() } } + + companion object { + private const val LOG_BODY_CHARS = 400 + private const val LOG_SNIPPET_CHARS = 200 + private const val MILLIS_PER_SECOND = 1000.0 + private const val UUID_HEX_BLOCK1_END = 8 + private const val UUID_HEX_BLOCK2_END = 12 + private const val UUID_HEX_BLOCK3_END = 16 + private const val UUID_HEX_BLOCK4_END = 20 + } } diff --git a/backend/src/main/kotlin/com/moneat/events/services/EventService.kt b/backend/src/main/kotlin/com/moneat/events/services/EventService.kt index a06f0c5c2..050836460 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/EventService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/EventService.kt @@ -75,6 +75,17 @@ class EventService( companion object { private const val DEFAULT_PROFILE_MAX_PAYLOAD_BYTES = 10 * 1024 * 1024 // 10 MiB + private const val LOG_PAYLOAD_PREVIEW_CHARS = 500 + private const val MAX_REPLAY_SEGMENT_COUNTERS = 10_000 + private const val MAX_REPLAY_URLS = 100 + private const val FINGERPRINT_HASH_LENGTH = 16 + private const val MS_PER_SECOND = 1000.0 + private const val NS_PER_SECOND = 1_000_000_000.0 + private const val UUID_SEG1 = 8 + private const val UUID_SEG2 = 12 + private const val UUID_SEG3 = 16 + private const val UUID_SEG4 = 20 + /** Keys set by the server for apm_spans; must not be overwritten by SDK tag maps. */ private val SENTRY_APM_META_RESERVED_KEYS = setOf("sentry.transaction_id", "sentry.project_id") } @@ -140,7 +151,7 @@ class EventService( logger.debug { "Processing envelope item type: ${item.type}" } when (item.type) { "event" -> { - logger.debug { "Event payload: ${item.payload.take(500)}" } + logger.debug { "Event payload: ${item.payload.take(LOG_PAYLOAD_PREVIEW_CHARS)}" } val event = json.decodeFromString(item.payload) if (storeEvent(projectId, event)) { recordUsage(projectId, "error", item) @@ -148,7 +159,7 @@ class EventService( } "transaction" -> { - logger.debug { "Transaction payload: ${item.payload.take(500)}" } + logger.debug { "Transaction payload: ${item.payload.take(LOG_PAYLOAD_PREVIEW_CHARS)}" } val transaction = parseTransactionPayload(item.payload) if (storeTransaction(projectId, transaction)) { recordUsage(projectId, "transaction", item) @@ -178,51 +189,20 @@ class EventService( } "replay_video" -> { - // Mobile replay uses replay_video instead of replay_recording. - // The SDK may send a preceding replay_event item (with replay_id & segment_id) - // or just a standalone replay_video. Handle both cases. - val rid = lastReplayId ?: envelope.eventId - - val segmentId = - if (lastReplayId != null) { - // A replay_event was parsed in this envelope - use its segment_id - lastSegmentId - } else { - // No replay_event - derive segment_id from an in-memory counter. - // Evict stale entries if the map grows too large. - if (replaySegmentCounters.size > 10_000) { - replaySegmentCounters.clear() - } - replaySegmentCounters - .computeIfAbsent(rid) { AtomicInteger(0) } - .getAndIncrement() - } - - // Only create a synthetic replay event for the first segment of a session - if (lastReplayId == null && segmentId == 0) { - storeSyntheticReplayEvent(projectId, rid, segmentId, envelope) - } - - storeReplayRecording(projectId, rid, segmentId, item.payload) - recordUsage(projectId, "replay", item) - lastReplayId = null - lastSegmentId = 0 + val (newReplayId, newSegmentId) = handleReplayVideoItem( + projectId, + item, + envelope, + lastReplayId, + lastSegmentId, + ) + lastReplayId = newReplayId + lastSegmentId = newSegmentId } - "profile" -> { - logger.debug { "Profile payload: ${item.payload.take(500)}" } - if (storeProfile(projectId, item.payload)) { - recordUsage(projectId, "profile", item) - } - } + "profile" -> handleProfileItem(projectId, item) - "feedback" -> { - logger.debug { "Feedback payload: ${item.payload.take(500)}" } - val feedback = json.decodeFromString(item.payload) - if (storeFeedback(projectId, feedback)) { - recordUsage(projectId, "feedback", item) - } - } + "feedback" -> handleFeedbackItem(projectId, item) else -> { logger.debug { "Unknown item type: ${item.type}" } @@ -231,6 +211,56 @@ class EventService( } } + private suspend fun handleReplayVideoItem( + projectId: Long, + item: EnvelopeItem, + envelope: SentryEnvelope, + lastReplayId: String?, + lastSegmentId: Int, + ): Pair { + // Mobile replay uses replay_video instead of replay_recording. + // The SDK may send a preceding replay_event item (with replay_id & segment_id) + // or just a standalone replay_video. Handle both cases. + val rid = lastReplayId ?: envelope.eventId + + val segmentId = + if (lastReplayId != null) { + // A replay_event was parsed in this envelope - use its segment_id + lastSegmentId + } else { + // No replay_event - derive segment_id from an in-memory counter. + // Evict stale entries if the map grows too large. + if (replaySegmentCounters.size > MAX_REPLAY_SEGMENT_COUNTERS) { + replaySegmentCounters.clear() + } + replaySegmentCounters.computeIfAbsent(rid) { AtomicInteger(0) }.getAndIncrement() + } + + // Only create a synthetic replay event for the first segment of a session + if (lastReplayId == null && segmentId == 0) { + storeSyntheticReplayEvent(projectId, rid, segmentId, envelope) + } + + storeReplayRecording(projectId, rid, segmentId, item.payload) + recordUsage(projectId, "replay", item) + return null to 0 + } + + private suspend fun handleProfileItem(projectId: Long, item: EnvelopeItem) { + logger.debug { "Profile payload: ${item.payload.take(LOG_PAYLOAD_PREVIEW_CHARS)}" } + if (storeProfile(projectId, item.payload)) { + recordUsage(projectId, "profile", item) + } + } + + private suspend fun handleFeedbackItem(projectId: Long, item: EnvelopeItem) { + logger.debug { "Feedback payload: ${item.payload.take(LOG_PAYLOAD_PREVIEW_CHARS)}" } + val feedback = json.decodeFromString(item.payload) + if (storeFeedback(projectId, feedback)) { + recordUsage(projectId, "feedback", item) + } + } + suspend fun processStoreEvent( projectId: Long, body: String @@ -262,7 +292,7 @@ class EventService( val traceStatus = traceContext?.get("status")?.jsonPrimitive?.contentOrNull val transactionLevel = if (traceStatus == null || traceStatus == "ok") "info" else "error" - val endTimestampMs = unixSecondsToMillis(transaction.timestamp ?: (System.currentTimeMillis() / 1000.0)) + val endTimestampMs = unixSecondsToMillis(transaction.timestamp ?: (System.currentTimeMillis() / MS_PER_SECOND)) val durationMs = durationMs(transaction.start_timestamp, transaction.timestamp) val contexts = transaction.contexts?.toString() ?: "{}" @@ -533,7 +563,7 @@ class EventService( val ts = replayEvent.timestamp?.let { unixSecondsToMillis(it) } ?: System.currentTimeMillis() val startTs = replayEvent.replay_start_timestamp?.let { unixSecondsToMillis(it) } ?: ts - val urls = replayEvent.urls?.take(100) ?: emptyList() + val urls = replayEvent.urls?.take(MAX_REPLAY_URLS) ?: emptyList() val errorIds = replayEvent.error_ids ?: emptyList() val traceIds = replayEvent.trace_ids ?: emptyList() val tags = replayEvent.tags?.let { JsonObject(it.mapValues { (_, v) -> JsonPrimitive(v) }).toString() } ?: "{}" @@ -769,7 +799,7 @@ class EventService( private fun parseTimestampString(raw: String): Double? { raw.toDoubleOrNull()?.let { return it } val instant = runCatching { Instant.parse(raw) }.getOrNull() ?: return null - return instant.epochSecond.toDouble() + instant.nano / 1_000_000_000.0 + return instant.epochSecond.toDouble() + instant.nano / NS_PER_SECOND } private fun generateFingerprint(event: SentryEvent): List { @@ -806,7 +836,7 @@ class EventService( val combined = fingerprint.joinToString("::") val digest = MessageDigest.getInstance("SHA-256") val hash = digest.digest(combined.toByteArray()) - return hash.joinToString("") { "%02x".format(it) }.take(16) + return hash.joinToString("") { "%02x".format(it) }.take(FINGERPRINT_HASH_LENGTH) } private fun normalizeUuid(value: String): String { @@ -816,13 +846,9 @@ class EventService( val hexRegex = Regex("^[0-9a-f]{32}$") if (hexRegex.matches(trimmed)) { - return "${trimmed.substring( - 0, - 8 - )}-${trimmed.substring( - 8, - 12 - )}-${trimmed.substring(12, 16)}-${trimmed.substring(16, 20)}-${trimmed.substring(20)}" + return "${trimmed.substring(0, UUID_SEG1)}-${trimmed.substring(UUID_SEG1, UUID_SEG2)}" + + "-${trimmed.substring(UUID_SEG2, UUID_SEG3)}-${trimmed.substring(UUID_SEG3, UUID_SEG4)}" + + "-${trimmed.substring(UUID_SEG4)}" } return UUID.randomUUID().toString() @@ -901,7 +927,7 @@ class EventService( } private fun unixSecondsToMillis(value: Double): Long { - return (value * 1000).toLong() + return (value * MS_PER_SECOND).toLong() } private fun unixSecondsToMillis(value: Double?): Long? { @@ -913,15 +939,15 @@ class EventService( end: Double? ): Double { if (start == null || end == null) return 0.0 - return ((end - start) * 1000.0).coerceAtLeast(0.0) + return ((end - start) * MS_PER_SECOND).coerceAtLeast(0.0) } private fun unixSecondsToNanos(value: Double): Long = - (value * 1_000_000_000.0).toLong() + (value * NS_PER_SECOND).toLong() private fun durationNanos(start: Double?, end: Double?): Long { if (start == null || end == null) return 0L - return ((end - start) * 1_000_000_000.0).toLong().coerceAtLeast(0L) + return ((end - start) * NS_PER_SECOND).toLong().coerceAtLeast(0L) } private fun sentryStatusToError(status: String?): Int = @@ -999,7 +1025,7 @@ class EventService( ?.get("span_id")?.jsonPrimitive?.contentOrNull ?: "" if (rootSpanId.isBlank()) return null - val startTs = transaction.start_timestamp ?: (System.currentTimeMillis() / 1000.0) + val startTs = transaction.start_timestamp ?: (System.currentTimeMillis() / MS_PER_SECOND) val (traceIdHigh, traceIdLow) = hexToULongPair(traceId) val (spanIdHigh, spanIdLow) = hexToULongPair(rootSpanId) val rootMetrics = extractMeasurementMetrics(transaction) diff --git a/backend/src/main/kotlin/com/moneat/events/services/IngestionWorker.kt b/backend/src/main/kotlin/com/moneat/events/services/IngestionWorker.kt index 212970ff4..38a401355 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/IngestionWorker.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/IngestionWorker.kt @@ -37,6 +37,7 @@ import kotlinx.coroutines.launch import kotlinx.serialization.SerializationException import mu.KotlinLogging import java.io.IOException +import java.nio.ByteBuffer import java.util.* private val logger = KotlinLogging.logger {} @@ -149,22 +150,16 @@ class IngestionWorker( } companion object { + private const val PROJECT_ID_BYTE_LENGTH = 8 + /** * Decode a queue message: Base64(8 bytes projectId big-endian + envelope bytes). */ fun decodeMessage(encoded: String): Pair { val bytes = Base64.getDecoder().decode(encoded) - if (bytes.size < 8) throw IllegalArgumentException("Message too short") - val projectId = - ((bytes[0].toLong() and 0xFF) shl 56) or - ((bytes[1].toLong() and 0xFF) shl 48) or - ((bytes[2].toLong() and 0xFF) shl 40) or - ((bytes[3].toLong() and 0xFF) shl 32) or - ((bytes[4].toLong() and 0xFF) shl 24) or - ((bytes[5].toLong() and 0xFF) shl 16) or - ((bytes[6].toLong() and 0xFF) shl 8) or - (bytes[7].toLong() and 0xFF) - val envelopeBytes = bytes.copyOfRange(8, bytes.size) + require(bytes.size >= PROJECT_ID_BYTE_LENGTH) { "Message too short" } + val projectId = ByteBuffer.wrap(bytes, 0, PROJECT_ID_BYTE_LENGTH).long + val envelopeBytes = bytes.copyOfRange(PROJECT_ID_BYTE_LENGTH, bytes.size) return projectId to envelopeBytes } @@ -175,16 +170,9 @@ class IngestionWorker( projectId: Long, envelopeBytes: ByteArray ): String { - val bytes = ByteArray(8 + envelopeBytes.size) - bytes[0] = (projectId shr 56).toByte() - bytes[1] = (projectId shr 48).toByte() - bytes[2] = (projectId shr 40).toByte() - bytes[3] = (projectId shr 32).toByte() - bytes[4] = (projectId shr 24).toByte() - bytes[5] = (projectId shr 16).toByte() - bytes[6] = (projectId shr 8).toByte() - bytes[7] = projectId.toByte() - envelopeBytes.copyInto(bytes, 8) + val bytes = ByteArray(PROJECT_ID_BYTE_LENGTH + envelopeBytes.size) + ByteBuffer.wrap(bytes).putLong(projectId) + envelopeBytes.copyInto(bytes, PROJECT_ID_BYTE_LENGTH) return Base64.getEncoder().encodeToString(bytes) } } diff --git a/backend/src/main/kotlin/com/moneat/events/services/IssueService.kt b/backend/src/main/kotlin/com/moneat/events/services/IssueService.kt index 40b43074e..39da38f85 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/IssueService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/IssueService.kt @@ -34,6 +34,10 @@ class IssueService( private val queryHelper: DashboardQueryHelper ) { + companion object { + private const val ISSUE_OVERFETCH_MULTIPLIER = 5 + } + suspend fun getProjectIdForIssue(issueId: String): Long? = issueRepository.getProjectIdForIssue(issueId) @@ -52,7 +56,7 @@ class IssueService( val pgOverrides = issueRepository.getIssueStatusOverrides(projectId) - val overfetch = if (status != null) (limit + offset) * 5 else limit + offset + val overfetch = if (status != null) (limit + offset) * ISSUE_OVERFETCH_MULTIPLIER else limit + offset val rows = issueRepository.getIssuesRaw( projectId = projectId, offset = offset, diff --git a/backend/src/main/kotlin/com/moneat/events/services/ProjectService.kt b/backend/src/main/kotlin/com/moneat/events/services/ProjectService.kt index 78311e90a..754cdc311 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/ProjectService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/ProjectService.kt @@ -33,6 +33,11 @@ class ProjectService( private val billingQuotaService: BillingQuotaService = BillingQuotaService(), ) { + companion object { + private const val KEY_BYTE_LENGTH = 32 + private const val PUBLIC_KEY_LENGTH = 40 + } + suspend fun getProjects( userId: Int, demoEpochMs: Long? = null @@ -154,13 +159,13 @@ class ProjectService( } private fun generatePublicKey(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(KEY_BYTE_LENGTH) SecureRandom().nextBytes(bytes) - return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes).take(40) + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes).take(PUBLIC_KEY_LENGTH) } private fun generateSecretKey(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(KEY_BYTE_LENGTH) SecureRandom().nextBytes(bytes) return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) } diff --git a/backend/src/main/kotlin/com/moneat/events/services/ProjectStatsService.kt b/backend/src/main/kotlin/com/moneat/events/services/ProjectStatsService.kt index 107ba6da0..fcdda19e2 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/ProjectStatsService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/ProjectStatsService.kt @@ -38,6 +38,19 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val STATS_CACHE_TTL_SECONDS = 60L +private const val HOURS_IN_24H = 24 +private const val HOURS_IN_7D = 168 +private const val HOURS_IN_30D = 720 +private const val HOURS_IN_90D = 2160 +private const val INTERVAL_MINUTES_24H = 60 +private const val INTERVAL_MINUTES_7D = 360 +private const val INTERVAL_MINUTES_30D = 1440 +private const val INTERVAL_MINUTES_90D = 4320 +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 +private const val ERROR_BODY_PREVIEW_CHARS = 400 + class ProjectStatsService(private val queryHelper: DashboardQueryHelper) { private val clickhouseDb: String get() = queryHelper.clickhouseDb private val json get() = queryHelper.json @@ -48,24 +61,28 @@ class ProjectStatsService(private val queryHelper: DashboardQueryHelper) { parentSpan: ISpan? = null, demoEpochMs: Long? = null ): ProjectStatsResponse = - CacheService.cached("cache:project_stats:$projectId:$period:${demoEpochMs ?: ""}", 60, parentSpan) { + CacheService.cached( + "cache:project_stats:$projectId:$period:${demoEpochMs ?: ""}", + STATS_CACHE_TTL_SECONDS, + parentSpan, + ) { val retentionDays = queryHelper.getProjectRetentionDays(projectId) val hoursBack = when (period) { - "24h" -> 24 - "7d" -> 168 - "30d" -> 720 - "90d" -> 2160 - else -> 168 + "24h" -> HOURS_IN_24H + "7d" -> HOURS_IN_7D + "30d" -> HOURS_IN_30D + "90d" -> HOURS_IN_90D + else -> HOURS_IN_7D } val intervalMinutes = when (period) { - "24h" -> 60 - "7d" -> 360 - "30d" -> 1440 - "90d" -> 4320 - else -> 360 + "24h" -> INTERVAL_MINUTES_24H + "7d" -> INTERVAL_MINUTES_7D + "30d" -> INTERVAL_MINUTES_30D + "90d" -> INTERVAL_MINUTES_90D + else -> INTERVAL_MINUTES_7D } val nowSql = queryHelper.demoNowClause(demoEpochMs) @@ -318,8 +335,10 @@ class ProjectStatsService(private val queryHelper: DashboardQueryHelper) { return suspendRunCatching { val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() - if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { - logger.error { "Failed to execute release markers query: ${response.status} ${body.take(400)}" } + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX || body.trimStart().startsWith("Code:")) { + logger.error { + "Failed to execute release markers query: ${response.status} ${body.take(ERROR_BODY_PREVIEW_CHARS)}" + } return emptyList() } body @@ -369,7 +388,7 @@ class ProjectStatsService(private val queryHelper: DashboardQueryHelper) { """.trimIndent() val response = ClickHouseClient.execute(query, parentSpan) val body = response.bodyAsText() - if (response.status.value !in 200..299 || body.trimStart().startsWith("Code:")) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX || body.trimStart().startsWith("Code:")) { return pgOverrides } val retainedIds = body.lines() diff --git a/backend/src/main/kotlin/com/moneat/events/services/ReleaseService.kt b/backend/src/main/kotlin/com/moneat/events/services/ReleaseService.kt index 1a729ae45..7461a6781 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/ReleaseService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/ReleaseService.kt @@ -51,6 +51,10 @@ class ReleaseService { private val dateFormatter = DateTimeFormatter.ISO_INSTANT private val logger = KotlinLogging.logger {} + companion object { + private const val IO_BUFFER_SIZE = 8192 + } + /** * Create a new release for a project */ @@ -467,7 +471,7 @@ class ReleaseService { // Verify checksum val digest = MessageDigest.getInstance("SHA-1") storage.openInputStream(storageKey)?.use { input -> - val buffer = ByteArray(8192) + val buffer = ByteArray(IO_BUFFER_SIZE) var read: Int while (input.read(buffer).also { read = it } != -1) { digest.update(buffer, 0, read) diff --git a/backend/src/main/kotlin/com/moneat/events/services/ReleaseStatsService.kt b/backend/src/main/kotlin/com/moneat/events/services/ReleaseStatsService.kt index eb5625372..8186bd441 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/ReleaseStatsService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/ReleaseStatsService.kt @@ -45,11 +45,16 @@ class ReleaseStatsService(private val queryHelper: DashboardQueryHelper) { private val clickhouseDb: String get() = queryHelper.clickhouseDb private val json get() = queryHelper.json + companion object { + private const val RELEASES_CACHE_TTL_SECONDS = 120L + private const val RELEASE_STATS_INTERVAL_MINUTES = 360 + } + suspend fun getReleases( projectId: Long, parentSpan: ISpan? = null ): List = - CacheService.cached("cache:releases:$projectId", 120, parentSpan) { + CacheService.cached("cache:releases:$projectId", RELEASES_CACHE_TTL_SECONDS, parentSpan) { val retentionDays = queryHelper.getProjectRetentionDays(projectId) val projectIdClause = ClickHouseQueryUtils.projectIdClause(projectId) val releasesQuery = @@ -126,7 +131,7 @@ class ReleaseStatsService(private val queryHelper: DashboardQueryHelper) { // Crash-free user rate stays null until a user-based ClickHouse query is implemented. val crashFreeUserRate: Double? = null - val intervalMinutes = 360 + val intervalMinutes = RELEASE_STATS_INTERVAL_MINUTES val eventsTimelineQuery = """ SELECT diff --git a/backend/src/main/kotlin/com/moneat/events/services/ReplayService.kt b/backend/src/main/kotlin/com/moneat/events/services/ReplayService.kt index a30bff92c..f2dc42a66 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/ReplayService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/ReplayService.kt @@ -117,9 +117,9 @@ class ReplayService( private fun isClickHouseError(statusCode: Int, body: String, context: String): Boolean { if (statusCode !in 200..299 || body.trimStart().startsWith("Code:")) { if (statusCode !in 200..299) { - logger.error { "$context failed: $statusCode ${body.take(400)}" } + logger.error { "$context failed: $statusCode ${body.take(LOG_BODY_PREVIEW_LENGTH)}" } } else { - logger.error { "$context (ClickHouse): ${body.take(400)}" } + logger.error { "$context (ClickHouse): ${body.take(LOG_BODY_PREVIEW_LENGTH)}" } } return true } @@ -347,8 +347,8 @@ class ReplayService( } private fun isLikelyMp4(payloadBytes: ByteArray): Boolean { - if (payloadBytes.size < 8) return false - val boxType = String(payloadBytes.copyOfRange(4, 8), Charsets.US_ASCII) + if (payloadBytes.size < MP4_HEADER_MIN_BYTES) return false + val boxType = String(payloadBytes.copyOfRange(MP4_BOX_TYPE_OFFSET, MP4_HEADER_MIN_BYTES), Charsets.US_ASCII) return boxType == "ftyp" } @@ -455,13 +455,13 @@ class ReplayService( val nowMs = demoEpochMs ?: System.currentTimeMillis() val periodMs = when (period) { - "24h" -> 24 * 60 * 60 * 1000L - "30d" -> 30 * 24 * 60 * 60 * 1000L - "90d" -> 90 * 24 * 60 * 60 * 1000L - else -> 7 * 24 * 60 * 60 * 1000L + "24h" -> PERIOD_24H_MS + "30d" -> PERIOD_30D_MS + "90d" -> PERIOD_90D_MS + else -> PERIOD_7D_MS } val periodStartMs = nowMs - periodMs - val retentionStartMs = nowMs - (retentionDays * 24 * 60 * 60 * 1000L) + val retentionStartMs = nowMs - (retentionDays * MILLIS_PER_DAY) val envClause = if (environment != null && environment.isNotBlank()) { @@ -778,7 +778,7 @@ class ReplayService( suspendRunCatching { Instant.parse(replay.finishedAt).toEpochMilli() }.getOrElse { _ -> - replayStartMs + 86400_000L + replayStartMs + MILLIS_PER_DAY } val projectId = replay.projectId val retentionDays = queryHelper.getProjectRetentionDays(projectId) @@ -1063,4 +1063,15 @@ class ReplayService( emptyList() } } + + companion object { + private const val LOG_BODY_PREVIEW_LENGTH = 400 + private const val MP4_HEADER_MIN_BYTES = 8 + private const val MP4_BOX_TYPE_OFFSET = 4 + private const val MILLIS_PER_DAY = 24L * 60 * 60 * 1000 + private const val PERIOD_24H_MS = MILLIS_PER_DAY + private const val PERIOD_7D_MS = 7 * MILLIS_PER_DAY + private const val PERIOD_30D_MS = 30 * MILLIS_PER_DAY + private const val PERIOD_90D_MS = 90 * MILLIS_PER_DAY + } } diff --git a/backend/src/main/kotlin/com/moneat/events/services/SentrySpanBackfill.kt b/backend/src/main/kotlin/com/moneat/events/services/SentrySpanBackfill.kt index 85af4edef..9540efb57 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/SentrySpanBackfill.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/SentrySpanBackfill.kt @@ -33,6 +33,8 @@ private val logger = KotlinLogging.logger {} */ object SentrySpanBackfill { + private const val ERROR_BODY_MAX_LENGTH = 500 + suspend fun run() { val mapping = loadProjectOrgMapping() if (mapping.isEmpty()) { @@ -95,9 +97,9 @@ object SentrySpanBackfill { } else { val bodySnippet = suspendRunCatching { - response.bodyAsText().take(500) + response.bodyAsText().take(ERROR_BODY_MAX_LENGTH) }.getOrElse { e -> - (e.message ?: "").take(500) + (e.message ?: "").take(ERROR_BODY_MAX_LENGTH) } logger.error { "Spans backfill failed: HTTP ${response.status}, body: $bodySnippet" } } diff --git a/backend/src/main/kotlin/com/moneat/events/services/TransactionService.kt b/backend/src/main/kotlin/com/moneat/events/services/TransactionService.kt index 315adbdeb..877adb52b 100644 --- a/backend/src/main/kotlin/com/moneat/events/services/TransactionService.kt +++ b/backend/src/main/kotlin/com/moneat/events/services/TransactionService.kt @@ -49,6 +49,8 @@ class TransactionService(private val queryHelper: DashboardQueryHelper) { companion object { private const val APDEX_THRESHOLD_MS = 500 private const val NANOS_PER_MILLI = 1_000_000.0 + private const val APDEX_FRUSTRATED_MULTIPLIER = 4 + private const val APDEX_TOLERATED_WEIGHT = 0.5 } private fun getOrganizationIdForProject(projectId: Long): Int? = @@ -162,10 +164,14 @@ class TransactionService(private val queryHelper: DashboardQueryHelper) { val nowClause = queryHelper.demoNowClause(demoEpochMs) val filterClause = queryHelper.buildTransactionFilterClause(environment, operation) + val apdexToleratedUpper = APDEX_THRESHOLD_MS * APDEX_FRUSTRATED_MULTIPLIER val totalQuery = """ SELECT count() as total, avg(duration_ms) as avg_duration, countIf(duration_ms <= $APDEX_THRESHOLD_MS) as satisfied, - countIf(duration_ms > $APDEX_THRESHOLD_MS AND duration_ms <= ${APDEX_THRESHOLD_MS * 4}) as tolerated + countIf( + duration_ms > $APDEX_THRESHOLD_MS + AND duration_ms <= $apdexToleratedUpper + ) as tolerated FROM `$clickhouseDb`.events WHERE $projectIdClause AND event_type = 'transaction' @@ -232,7 +238,7 @@ class TransactionService(private val queryHelper: DashboardQueryHelper) { val slowest = queryHelper.executeSlowestTransactionsQuery(slowestQuery) val apdex = if (totalCount > 0) { - ((satisfiedCount + toleratedCount * 0.5) / totalCount).coerceIn(0.0, 1.0) + ((satisfiedCount + toleratedCount * APDEX_TOLERATED_WEIGHT) / totalCount).coerceIn(0.0, 1.0) } else { 0.0 } diff --git a/backend/src/main/kotlin/com/moneat/incident/routes/IncidentProviderRoutes.kt b/backend/src/main/kotlin/com/moneat/incident/routes/IncidentProviderRoutes.kt index b3ab573b4..38e9e47fb 100644 --- a/backend/src/main/kotlin/com/moneat/incident/routes/IncidentProviderRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/incident/routes/IncidentProviderRoutes.kt @@ -29,6 +29,7 @@ import io.ktor.server.auth.authenticate import io.ktor.server.auth.jwt.JWTPrincipal import io.ktor.server.auth.principal import io.ktor.server.request.receive +import io.ktor.server.plugins.BadRequestException import io.ktor.server.response.respond import io.ktor.server.routing.Route import io.ktor.server.routing.delete @@ -56,6 +57,10 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import kotlin.time.Clock import com.moneat.utils.suspendRunCatching +private const val PROVIDER_TYPE_MAX_LENGTH = 50 +private const val PROVIDER_NAME_MAX_LENGTH = 255 +private const val DEFAULT_INCIDENT_PAGE_LIMIT = 50 + fun Route.incidentProviderRoutes() { val json = Json { @@ -110,168 +115,13 @@ fun Route.incidentProviderRoutes() { } // Create provider config - post { - val principal = call.principal()!! - val userId = principal.payload.getClaim("userId").asInt() - - val organizationId = - transaction { - Memberships - .selectAll() - .where { Memberships.user_id eq userId } - .firstOrNull() - ?.get(Memberships.organization_id) - } ?: return@post call.respond(HttpStatusCode.Forbidden) - - val request = call.receive() - - val configId = - transaction { - // Custom SQL for JSONB insertion - val jsonString = request.configJson.toString() - val sql = - """ - INSERT INTO incident_provider_configs - (organization_id, provider_type, name, api_key, config_json, enabled, created_at, updated_at) - VALUES (?, ?, ?, ?, CAST(? AS JSONB), ?, CAST(? AS TIMESTAMP), CAST(? AS TIMESTAMP)) - RETURNING id - """.trimIndent() - - val now = Clock.System.now() - val nowStr = now.toString() - - var resultId: Int? = null - TransactionManager.current().exec( - sql, - listOf( - IntegerColumnType() to organizationId, - VarCharColumnType(50) to request.providerType, - VarCharColumnType(255) to request.name, - TextColumnType() to request.apiKey, - TextColumnType() to jsonString, - BooleanColumnType() to true, - TextColumnType() to nowStr, - TextColumnType() to nowStr - ) - ) { rs -> - if (rs.next()) { - resultId = rs.getInt(1) - } - } - - resultId ?: throw Exception("Failed to insert provider config") - } - - call.respond(HttpStatusCode.Created, mapOf("id" to configId)) - } + post { handleCreateProviderConfig() } // Update provider config - put("/{id}") { - val principal = call.principal()!! - val userId = principal.payload.getClaim("userId").asInt() - val configId = - call.parameters["id"]?.toIntOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest) - - val organizationId = - transaction { - Memberships - .selectAll() - .where { Memberships.user_id eq userId } - .firstOrNull() - ?.get(Memberships.organization_id) - } ?: return@put call.respond(HttpStatusCode.Forbidden) - - val request = call.receive() - - val updated = - transaction { - val exists = - IncidentProviderConfigs - .selectAll() - .where { - (IncidentProviderConfigs.id eq configId) and - (IncidentProviderConfigs.organizationId eq organizationId) - }.count() > 0 - - if (!exists) return@transaction false - - // Build update SQL dynamically based on what's provided - val setClauses = mutableListOf() - val params = mutableListOf, Any?>>() - - request.name?.let { - setClauses.add("name = ?") - params.add(VarCharColumnType(255) to it) - } - request.apiKey?.let { - setClauses.add("api_key = ?") - params.add(TextColumnType() to it) - } - request.configJson?.let { - setClauses.add("config_json = CAST(? AS JSONB)") - params.add(TextColumnType() to it.toString()) - } - request.enabled?.let { - setClauses.add("enabled = ?") - params.add(BooleanColumnType() to it) - } - - if (setClauses.isNotEmpty()) { - setClauses.add("updated_at = CAST(? AS TIMESTAMP)") - params.add(TextColumnType() to Clock.System.now().toString()) - params.add(IntegerColumnType() to configId) - - val updateSql = - """ - UPDATE incident_provider_configs - SET ${setClauses.joinToString(", ")} - WHERE id = ? - """.trimIndent() - - TransactionManager.current().exec(updateSql, params) {} - } - - true - } - - if (updated) { - call.respond(HttpStatusCode.OK) - } else { - call.respond(HttpStatusCode.NotFound) - } - } + put("/{id}") { handleUpdateProviderConfig() } // Delete provider config - delete("/{id}") { - val principal = call.principal()!! - val userId = principal.payload.getClaim("userId").asInt() - val configId = - call.parameters["id"]?.toIntOrNull() - ?: return@delete call.respond(HttpStatusCode.BadRequest) - - val organizationId = - transaction { - Memberships - .selectAll() - .where { Memberships.user_id eq userId } - .firstOrNull() - ?.get(Memberships.organization_id) - } ?: return@delete call.respond(HttpStatusCode.Forbidden) - - val deleted = - transaction { - IncidentProviderConfigs.deleteWhere { - (id eq configId) and (IncidentProviderConfigs.organizationId eq organizationId) - } > 0 - } - - if (deleted) { - call.respond(HttpStatusCode.OK) - } else { - call.respond(HttpStatusCode.NotFound) - } - } + delete("/{id}") { handleDeleteProviderConfig() } // Test connection post("/{id}/test") { @@ -382,116 +232,284 @@ fun Route.incidentProviderRoutes() { } // Bulk upsert routing rules - put("/{id}/rules") { - val principal = call.principal()!! - val userId = principal.payload.getClaim("userId").asInt() - val configId = - call.parameters["id"]?.toIntOrNull() - ?: return@put call.respond(HttpStatusCode.BadRequest) + put("/{id}/rules") { handleUpsertRoutingRules() } - val organizationId = - transaction { - Memberships - .selectAll() - .where { Memberships.user_id eq userId } - .firstOrNull() - ?.get(Memberships.organization_id) - } ?: return@put call.respond(HttpStatusCode.Forbidden) + // Get event log + get("/{id}/events") { handleGetEventLog() } + } + } +} - val hasAccess = - transaction { - IncidentProviderConfigs - .selectAll() - .where { - (IncidentProviderConfigs.id eq configId) and - (IncidentProviderConfigs.organizationId eq organizationId) - }.count() > 0 - } +private suspend fun io.ktor.server.routing.RoutingContext.handleCreateProviderConfig() { + val principal = call.principal()!! + val userId = principal.payload.getClaim("userId").asInt() + + val organizationId = + transaction { + Memberships + .selectAll() + .where { Memberships.user_id eq userId } + .firstOrNull() + ?.get(Memberships.organization_id) + } ?: return call.respond(HttpStatusCode.Forbidden) + + val request = call.receive() + + val configId = + transaction { + // Custom SQL for JSONB insertion + val jsonString = request.configJson.toString() + val sql = + """ + INSERT INTO incident_provider_configs + (organization_id, provider_type, name, api_key, config_json, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, CAST(? AS JSONB), ?, CAST(? AS TIMESTAMP), CAST(? AS TIMESTAMP)) + RETURNING id + """.trimIndent() + + val now = Clock.System.now() + val nowStr = now.toString() + + var resultId: Int? = null + TransactionManager.current().exec( + sql, + listOf( + IntegerColumnType() to organizationId, + VarCharColumnType(PROVIDER_TYPE_MAX_LENGTH) to request.providerType, + VarCharColumnType(PROVIDER_NAME_MAX_LENGTH) to request.name, + TextColumnType() to request.apiKey, + TextColumnType() to jsonString, + BooleanColumnType() to true, + TextColumnType() to nowStr, + TextColumnType() to nowStr + ) + ) { rs -> + if (rs.next()) { + resultId = rs.getInt(1) + } + } - if (!hasAccess) return@put call.respond(HttpStatusCode.NotFound) + resultId ?: throw Exception("Failed to insert provider config") + } - val request = call.receive>() + call.respond(HttpStatusCode.Created, mapOf("id" to configId)) +} - transaction { - // Delete existing rules for this provider - IncidentRoutingRules.deleteWhere { - providerConfigId eq configId - } +private suspend fun io.ktor.server.routing.RoutingContext.handleUpdateProviderConfig() { + val principal = call.principal()!! + val userId = principal.payload.getClaim("userId").asInt() + val configId = + call.parameters["id"]?.toIntOrNull() + ?: return call.respond(HttpStatusCode.BadRequest) + + val organizationId = + transaction { + Memberships + .selectAll() + .where { Memberships.user_id eq userId } + .firstOrNull() + ?.get(Memberships.organization_id) + } ?: return call.respond(HttpStatusCode.Forbidden) + + val request = call.receive() + + val updated = + transaction { + val exists = + IncidentProviderConfigs + .selectAll() + .where { + (IncidentProviderConfigs.id eq configId) and + (IncidentProviderConfigs.organizationId eq organizationId) + }.count() > 0 + + if (!exists) return@transaction false + + // Build update SQL dynamically based on what's provided + val setClauses = mutableListOf() + val params = mutableListOf, Any?>>() + + request.name?.let { + setClauses.add("name = ?") + params.add(VarCharColumnType(PROVIDER_NAME_MAX_LENGTH) to it) + } + request.apiKey?.let { + setClauses.add("api_key = ?") + params.add(TextColumnType() to it) + } + request.configJson?.let { + setClauses.add("config_json = CAST(? AS JSONB)") + params.add(TextColumnType() to it.toString()) + } + request.enabled?.let { + setClauses.add("enabled = ?") + params.add(BooleanColumnType() to it) + } - // Insert new rules - request.forEach { rule -> - IncidentRoutingRules.insert { - it[IncidentRoutingRules.providerConfigId] = configId - it[IncidentRoutingRules.alertSource] = rule.alertSource - it[IncidentRoutingRules.alertType] = rule.alertType - it[IncidentRoutingRules.incidentSeverity] = rule.incidentSeverity - it[IncidentRoutingRules.createdAt] = Clock.System.now() - it[IncidentRoutingRules.updatedAt] = Clock.System.now() - } - } - } + if (setClauses.isNotEmpty()) { + setClauses.add("updated_at = CAST(? AS TIMESTAMP)") + params.add(TextColumnType() to Clock.System.now().toString()) + params.add(IntegerColumnType() to configId) - call.respond(HttpStatusCode.OK) + val updateSql = + """ + UPDATE incident_provider_configs + SET ${setClauses.joinToString(", ")} + WHERE id = ? + """.trimIndent() + + TransactionManager.current().exec(updateSql, params) {} } - // Get event log - get("/{id}/events") { - val principal = call.principal()!! - val userId = principal.payload.getClaim("userId").asInt() - val configId = - call.parameters["id"]?.toIntOrNull() - ?: return@get call.respond(HttpStatusCode.BadRequest) - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 50 + true + } - val organizationId = - transaction { - Memberships - .selectAll() - .where { Memberships.user_id eq userId } - .firstOrNull() - ?.get(Memberships.organization_id) - } ?: return@get call.respond(HttpStatusCode.Forbidden) + if (updated) { + call.respond(HttpStatusCode.OK) + } else { + call.respond(HttpStatusCode.NotFound) + } +} - val hasAccess = - transaction { - IncidentProviderConfigs - .selectAll() - .where { - (IncidentProviderConfigs.id eq configId) and - (IncidentProviderConfigs.organizationId eq organizationId) - }.count() > 0 - } +private suspend fun io.ktor.server.routing.RoutingContext.handleDeleteProviderConfig() { + val principal = call.principal()!! + val userId = principal.payload.getClaim("userId").asInt() + val configId = + call.parameters["id"]?.toIntOrNull() + ?: return call.respond(HttpStatusCode.BadRequest) + + val organizationId = + transaction { + Memberships + .selectAll() + .where { Memberships.user_id eq userId } + .firstOrNull() + ?.get(Memberships.organization_id) + } ?: return call.respond(HttpStatusCode.Forbidden) + + val deleted = + transaction { + IncidentProviderConfigs.deleteWhere { + (id eq configId) and (IncidentProviderConfigs.organizationId eq organizationId) + } > 0 + } - if (!hasAccess) return@get call.respond(HttpStatusCode.NotFound) + if (deleted) { + call.respond(HttpStatusCode.OK) + } else { + call.respond(HttpStatusCode.NotFound) + } +} - val events = - transaction { - IncidentEventLog - .selectAll() - .where { IncidentEventLog.providerConfigId eq configId } - .orderBy(IncidentEventLog.createdAt to SortOrder.DESC) - .limit(limit) - .map { row -> - EventLogResponse( - id = row[IncidentEventLog.id].value, - alertSource = row[IncidentEventLog.alertSource], - deduplicationKey = row[IncidentEventLog.deduplicationKey], - incidentSeverity = row[IncidentEventLog.incidentSeverity], - incidentStatus = row[IncidentEventLog.incidentStatus], - title = row[IncidentEventLog.title], - description = row[IncidentEventLog.description], - providerIncidentId = row[IncidentEventLog.providerIncidentId], - success = row[IncidentEventLog.success], - errorMessage = row[IncidentEventLog.errorMessage], - createdAt = row[IncidentEventLog.createdAt].toEpochMilliseconds() - ) - } - } +private suspend fun io.ktor.server.routing.RoutingContext.handleUpsertRoutingRules() { + val principal = call.principal()!! + val userId = principal.payload.getClaim("userId").asInt() + val configId = + call.parameters["id"]?.toIntOrNull() + ?: return call.respond(HttpStatusCode.BadRequest) + + val organizationId = + transaction { + Memberships + .selectAll() + .where { Memberships.user_id eq userId } + .firstOrNull() + ?.get(Memberships.organization_id) + } ?: return call.respond(HttpStatusCode.Forbidden) + + val hasAccess = + transaction { + IncidentProviderConfigs + .selectAll() + .where { + (IncidentProviderConfigs.id eq configId) and + (IncidentProviderConfigs.organizationId eq organizationId) + }.count() > 0 + } + + if (!hasAccess) return call.respond(HttpStatusCode.NotFound) - call.respond(events) + val request = call.receive>() + + transaction { + // Delete existing rules for this provider + IncidentRoutingRules.deleteWhere { + providerConfigId eq configId + } + + // Insert new rules + request.forEach { rule -> + IncidentRoutingRules.insert { + it[IncidentRoutingRules.providerConfigId] = configId + it[IncidentRoutingRules.alertSource] = rule.alertSource + it[IncidentRoutingRules.alertType] = rule.alertType + it[IncidentRoutingRules.incidentSeverity] = rule.incidentSeverity + it[IncidentRoutingRules.createdAt] = Clock.System.now() + it[IncidentRoutingRules.updatedAt] = Clock.System.now() } } } + + call.respond(HttpStatusCode.OK) +} + +private suspend fun io.ktor.server.routing.RoutingContext.handleGetEventLog() { + val principal = call.principal()!! + val userId = principal.payload.getClaim("userId").asInt() + val configId = + call.parameters["id"]?.toIntOrNull() + ?: return call.respond(HttpStatusCode.BadRequest) + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_INCIDENT_PAGE_LIMIT + if (limit <= 0) { + throw BadRequestException("limit must be a positive integer") + } + + val organizationId = + transaction { + Memberships + .selectAll() + .where { Memberships.user_id eq userId } + .firstOrNull() + ?.get(Memberships.organization_id) + } ?: return call.respond(HttpStatusCode.Forbidden) + + val hasAccess = + transaction { + IncidentProviderConfigs + .selectAll() + .where { + (IncidentProviderConfigs.id eq configId) and + (IncidentProviderConfigs.organizationId eq organizationId) + }.count() > 0 + } + + if (!hasAccess) return call.respond(HttpStatusCode.NotFound) + + val events = + transaction { + IncidentEventLog + .selectAll() + .where { IncidentEventLog.providerConfigId eq configId } + .orderBy(IncidentEventLog.createdAt to SortOrder.DESC) + .limit(limit) + .map { row -> + EventLogResponse( + id = row[IncidentEventLog.id].value, + alertSource = row[IncidentEventLog.alertSource], + deduplicationKey = row[IncidentEventLog.deduplicationKey], + incidentSeverity = row[IncidentEventLog.incidentSeverity], + incidentStatus = row[IncidentEventLog.incidentStatus], + title = row[IncidentEventLog.title], + description = row[IncidentEventLog.description], + providerIncidentId = row[IncidentEventLog.providerIncidentId], + success = row[IncidentEventLog.success], + errorMessage = row[IncidentEventLog.errorMessage], + createdAt = row[IncidentEventLog.createdAt].toEpochMilliseconds() + ) + } + } + + call.respond(events) } @Serializable diff --git a/backend/src/main/kotlin/com/moneat/incident/services/IncidentIoProvider.kt b/backend/src/main/kotlin/com/moneat/incident/services/IncidentIoProvider.kt index 9188d34b8..1f9d768ee 100644 --- a/backend/src/main/kotlin/com/moneat/incident/services/IncidentIoProvider.kt +++ b/backend/src/main/kotlin/com/moneat/incident/services/IncidentIoProvider.kt @@ -39,6 +39,9 @@ import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonPrimitive import org.slf4j.LoggerFactory +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 + /** * incident.io provider implementation using Alert Events V2 API. * https://api-docs.incident.io/#tag/Alert-Events-V2 @@ -109,7 +112,7 @@ class IncidentIoProvider : IncidentProvider { setBody(payload) } - if (response.status.value in 200..299) { + if (response.status.value in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { val responseBody = response.body() Result.success(responseBody.deduplication_key) } else { @@ -147,7 +150,7 @@ class IncidentIoProvider : IncidentProvider { setBody(payload) } - if (response.status.value in 200..299) { + if (response.status.value in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { val responseBody = response.body() Result.success(responseBody.deduplication_key) } else { @@ -184,7 +187,7 @@ class IncidentIoProvider : IncidentProvider { setBody(payload) } - if (response.status.value in 200..299) { + if (response.status.value in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { // Immediately resolve the test alert resolveAlert(testDedup, config) Result.success(true) diff --git a/backend/src/main/kotlin/com/moneat/llm/routes/LlmRoutes.kt b/backend/src/main/kotlin/com/moneat/llm/routes/LlmRoutes.kt index 9e34565de..c374ad476 100644 --- a/backend/src/main/kotlin/com/moneat/llm/routes/LlmRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/llm/routes/LlmRoutes.kt @@ -33,6 +33,8 @@ import io.ktor.server.routing.get import io.ktor.server.routing.route import org.koin.core.context.GlobalContext +private const val DEFAULT_PAGE_SIZE = 25 + fun Route.llmRoutes( dashboardService: DashboardService = GlobalContext.get().get(), llmService: LlmDashboardService = GlobalContext.get().get(), @@ -90,7 +92,7 @@ private suspend fun io.ktor.server.routing.RoutingContext.handleLlmGenerations( val type = call.request.queryParameters["type"] val status = call.request.queryParameters["status"] val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val pageSize = call.request.queryParameters["pageSize"]?.toIntOrNull() ?: 25 + val pageSize = call.request.queryParameters["pageSize"]?.toIntOrNull() ?: DEFAULT_PAGE_SIZE val demoEpochMs = call.getDemoEpochMs() call.respond( llmService.getGenerations(projectId, range, model, provider, type, status, page, pageSize, demoEpochMs) diff --git a/backend/src/main/kotlin/com/moneat/llm/services/LlmDashboardService.kt b/backend/src/main/kotlin/com/moneat/llm/services/LlmDashboardService.kt index 005a83294..f1445d1cf 100644 --- a/backend/src/main/kotlin/com/moneat/llm/services/LlmDashboardService.kt +++ b/backend/src/main/kotlin/com/moneat/llm/services/LlmDashboardService.kt @@ -45,6 +45,9 @@ import mu.KotlinLogging private val logger = KotlinLogging.logger {} +private const val MILLIS_PER_SECOND = 1000.0 +private const val DATETIME64_PRECISION_MS = 3 + class LlmDashboardService { private val clickhouseDb: String get() = ClickHouseClient.getDatabase() private val json = Json { ignoreUnknownKeys = true } @@ -68,7 +71,7 @@ class LlmDashboardService { if (response.status != HttpStatusCode.OK) return null val body = response.bodyAsText() if (body.isClickHouseError()) { - logger.warn { "ClickHouse error: ${body.take(200)}" } + logger.warn { "ClickHouse returned an error body (length=${body.length})" } return null } return body @@ -99,7 +102,7 @@ class LlmDashboardService { private fun nowClause(demoEpochMs: Long?): String { return if (demoEpochMs != null) { - "toDateTime64(${demoEpochMs / 1000.0}, 3)" + "toDateTime64(${demoEpochMs / MILLIS_PER_SECOND}, $DATETIME64_PRECISION_MS)" } else { "now()" } diff --git a/backend/src/main/kotlin/com/moneat/llm/services/LlmIngestionWorker.kt b/backend/src/main/kotlin/com/moneat/llm/services/LlmIngestionWorker.kt index eb662fca2..d6903b4c2 100644 --- a/backend/src/main/kotlin/com/moneat/llm/services/LlmIngestionWorker.kt +++ b/backend/src/main/kotlin/com/moneat/llm/services/LlmIngestionWorker.kt @@ -38,6 +38,7 @@ import kotlinx.coroutines.launch import kotlinx.serialization.json.Json import mu.KotlinLogging import java.io.IOException +import java.nio.ByteBuffer import java.time.Instant import java.time.format.DateTimeParseException import java.util.Base64 @@ -46,6 +47,10 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val BRPOP_BACKOFF_DELAY_MS = 1000L +private const val ERROR_BODY_PREVIEW_CHARS = 600 +private const val PROJECT_ID_HEADER_SIZE = 8 + class LlmIngestionWorker( private val queueKey: String, private val dlqKey: String, @@ -83,9 +88,9 @@ class LlmIngestionWorker( } catch (e: CancellationException) { break } catch (e: RedisException) { - brpopLoopBackoff(logger, workerId, "LLM", 1000L, e) + brpopLoopBackoff(logger, workerId, "LLM", BRPOP_BACKOFF_DELAY_MS, e) } catch (e: IOException) { - brpopLoopBackoff(logger, workerId, "LLM", 1000L, e) + brpopLoopBackoff(logger, workerId, "LLM", BRPOP_BACKOFF_DELAY_MS, e) } } } finally { @@ -196,7 +201,7 @@ class LlmIngestionWorker( val response = ClickHouseClient.execute(query) if (!response.status.isSuccess()) { val body = response.bodyAsText() - throw IllegalStateException("Failed to insert LLM generations: ${body.take(600)}") + throw IllegalStateException("Failed to insert LLM generations: ${body.take(ERROR_BODY_PREVIEW_CHARS)}") } logger.info { "Inserted ${rows.size} LLM generations for project $projectId" } } @@ -237,17 +242,9 @@ class LlmIngestionWorker( companion object { fun decodeMessage(encoded: String): Pair { val bytes = Base64.getDecoder().decode(encoded) - if (bytes.size < 8) throw IllegalArgumentException("Message too short") - val projectId = - ((bytes[0].toLong() and 0xFF) shl 56) or - ((bytes[1].toLong() and 0xFF) shl 48) or - ((bytes[2].toLong() and 0xFF) shl 40) or - ((bytes[3].toLong() and 0xFF) shl 32) or - ((bytes[4].toLong() and 0xFF) shl 24) or - ((bytes[5].toLong() and 0xFF) shl 16) or - ((bytes[6].toLong() and 0xFF) shl 8) or - (bytes[7].toLong() and 0xFF) - val payloadBytes = bytes.copyOfRange(8, bytes.size) + require(bytes.size >= PROJECT_ID_HEADER_SIZE) { "Message too short" } + val projectId = ByteBuffer.wrap(bytes, 0, PROJECT_ID_HEADER_SIZE).long + val payloadBytes = bytes.copyOfRange(PROJECT_ID_HEADER_SIZE, bytes.size) return projectId to payloadBytes } @@ -255,16 +252,9 @@ class LlmIngestionWorker( projectId: Long, payloadBytes: ByteArray ): String { - val bytes = ByteArray(8 + payloadBytes.size) - bytes[0] = (projectId shr 56).toByte() - bytes[1] = (projectId shr 48).toByte() - bytes[2] = (projectId shr 40).toByte() - bytes[3] = (projectId shr 32).toByte() - bytes[4] = (projectId shr 24).toByte() - bytes[5] = (projectId shr 16).toByte() - bytes[6] = (projectId shr 8).toByte() - bytes[7] = projectId.toByte() - payloadBytes.copyInto(bytes, 8) + val bytes = ByteArray(PROJECT_ID_HEADER_SIZE + payloadBytes.size) + ByteBuffer.wrap(bytes).putLong(projectId) + payloadBytes.copyInto(bytes, PROJECT_ID_HEADER_SIZE) return Base64.getEncoder().encodeToString(bytes) } } diff --git a/backend/src/main/kotlin/com/moneat/logging/MoneatLogAppender.kt b/backend/src/main/kotlin/com/moneat/logging/MoneatLogAppender.kt index 062930b03..b0f779284 100644 --- a/backend/src/main/kotlin/com/moneat/logging/MoneatLogAppender.kt +++ b/backend/src/main/kotlin/com/moneat/logging/MoneatLogAppender.kt @@ -34,6 +34,10 @@ class MoneatLogAppender : AppenderBase() { companion object { private const val QUEUE_CAPACITY = 10_000 + private const val LOG_HTTP_TIMEOUT_MS = 1000 + private const val HTTP_SUCCESS_MIN = 200 + private const val HTTP_SUCCESS_MAX = 299 + private const val SHUTDOWN_TIMEOUT_SECONDS = 5L } private val executor = @@ -123,12 +127,12 @@ class MoneatLogAppender : AppenderBase() { connection.setRequestProperty("Content-Type", "application/json") connection.setRequestProperty("Authorization", "Bearer $token") connection.doOutput = true - connection.connectTimeout = 1000 - connection.readTimeout = 1000 + connection.connectTimeout = LOG_HTTP_TIMEOUT_MS + connection.readTimeout = LOG_HTTP_TIMEOUT_MS connection.outputStream.use { it.write(payload.toByteArray()) } val responseCode = connection.responseCode - if (responseCode !in 200..299) { + if (responseCode !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { val body = runCatching { connection.errorStream?.bufferedReader()?.readText() }.getOrNull() System.err.println( "[MoneatLogAppender] Non-2xx response: $responseCode${if (body != null) " - $body" else ""}" @@ -151,7 +155,7 @@ class MoneatLogAppender : AppenderBase() { override fun stop() { executor.shutdown() - executor.awaitTermination(5, TimeUnit.SECONDS) + executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS) super.stop() } } diff --git a/backend/src/main/kotlin/com/moneat/logs/routes/LogRoutes.kt b/backend/src/main/kotlin/com/moneat/logs/routes/LogRoutes.kt index c493cafb9..bf99bd299 100644 --- a/backend/src/main/kotlin/com/moneat/logs/routes/LogRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/logs/routes/LogRoutes.kt @@ -68,6 +68,9 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} private val json = Json { ignoreUnknownKeys = true } +private const val MILLIS_IN_24_HOURS = 24L * 60 * 60 * 1000 +private const val SSE_POLL_TIMEOUT_SECONDS = 15L + fun Route.logRoutes( logService: LogService = GlobalContext.get().get(), otlpApiKeyService: OtlpApiKeyService = GlobalContext.get().get(), @@ -214,7 +217,7 @@ fun Route.logRoutes( // For demo mode, if no time range specified, default to last 24 hours from demo epoch val defaultFrom = if (isDemo && demoEpochMs != null && call.request.queryParameters["from"] == null) { - val twentyFourHoursAgo = demoEpochMs - (24 * 60 * 60 * 1000) + val twentyFourHoursAgo = demoEpochMs - MILLIS_IN_24_HOURS Instant.ofEpochMilli(twentyFourHoursAgo).toString() } else { call.request.queryParameters["from"] @@ -278,7 +281,7 @@ fun Route.logRoutes( // For demo mode, if no time range specified, default to last 24 hours from demo epoch val defaultFrom = if (isDemo && demoEpochMs != null && call.request.queryParameters["from"] == null) { - val twentyFourHoursAgo = demoEpochMs - (24 * 60 * 60 * 1000) + val twentyFourHoursAgo = demoEpochMs - MILLIS_IN_24_HOURS Instant.ofEpochMilli(twentyFourHoursAgo).toString() } else { call.request.queryParameters["from"] @@ -309,7 +312,7 @@ fun Route.logRoutes( // For demo mode, if no time range specified, default to last 24 hours from demo epoch val defaultFrom = if (isDemo && demoEpochMs != null && call.request.queryParameters["from"] == null) { - val twentyFourHoursAgo = demoEpochMs - (24 * 60 * 60 * 1000) + val twentyFourHoursAgo = demoEpochMs - MILLIS_IN_24_HOURS Instant.ofEpochMilli(twentyFourHoursAgo).toString() } else { call.request.queryParameters["from"] @@ -362,7 +365,7 @@ fun Route.logRoutes( // For demo mode, if no time range specified, default to last 24 hours from demo epoch val defaultFrom = if (isDemo && demoEpochMs != null && call.request.queryParameters["from"] == null) { - val twentyFourHoursAgo = demoEpochMs - (24 * 60 * 60 * 1000) + val twentyFourHoursAgo = demoEpochMs - MILLIS_IN_24_HOURS Instant.ofEpochMilli(twentyFourHoursAgo).toString() } else { call.request.queryParameters["from"] @@ -476,7 +479,7 @@ fun Route.logRoutes( flush() while (true) { - val next = queue.poll(15, TimeUnit.SECONDS) + val next = queue.poll(SSE_POLL_TIMEOUT_SECONDS, TimeUnit.SECONDS) if (next == null) { write(": heartbeat\n\n") flush() diff --git a/backend/src/main/kotlin/com/moneat/logs/services/LogService.kt b/backend/src/main/kotlin/com/moneat/logs/services/LogService.kt index 1051037be..4927da8a1 100644 --- a/backend/src/main/kotlin/com/moneat/logs/services/LogService.kt +++ b/backend/src/main/kotlin/com/moneat/logs/services/LogService.kt @@ -57,9 +57,11 @@ import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.longOrNull import mu.KotlinLogging +import java.security.MessageDigest import java.time.Instant import com.moneat.utils.suspendRunCatching import java.util.* +import kotlin.text.Charsets private val logger = KotlinLogging.logger {} @@ -75,6 +77,38 @@ private data class LogWithCursor( val timestampMs: Long ) +private const val MAX_LOG_QUERY_LIMIT = 500 +private const val MAX_TOP_VALUES_LIMIT = 100 +private const val MAX_FILTER_VALUES_LIMIT = 200 +private const val MAX_EXPORT_LIMIT = 10_000 +private const val ERROR_BODY_PREVIEW_CHARS = 600 +private const val WARN_BODY_PREVIEW_CHARS = 500 +private const val MILLIS_PER_HOUR = 3_600_000L +private const val MILLIS_PER_6_HOURS = 21_600_000L +private const val MILLIS_PER_DAY = 86_400_000L +private const val MILLIS_PER_WEEK = 604_800_000L +private const val MAX_LOG_MESSAGE_CHARS = 8192 +private const val MAX_LOG_BODY_CHARS = 32768 +private const val MAX_LOG_SERVICE_CHARS = 256 +private const val MAX_LOG_ENVIRONMENT_CHARS = 128 +private const val MAX_LOG_HOST_CHARS = 256 +private const val MAX_LOG_CONTAINER_ID_CHARS = 128 +private const val MAX_LOG_CONTAINER_IMAGE_CHARS = 512 +private const val MAX_LOG_TRACE_ID_CHARS = 128 +private const val MAX_LOG_SPAN_ID_CHARS = 128 +private const val INGEST_KEY_MAX_CHARS = 128 +private const val INGEST_VALUE_MAX_CHARS = 1024 +private const val ERROR_BODY_LONG_CHARS = 1000 +private const val MAX_LOG_CONTAINER_NAME_CHARS = 256 +private const val MS_PER_SECOND = 1000 +private const val UNIX_EPOCH_SECONDS_MAX_DIGITS = 10 +private const val SQL_LOG_FINGERPRINT_HEX_CHARS = 16 + +private fun utf8Fingerprint(text: String, hexChars: Int = SQL_LOG_FINGERPRINT_HEX_CHARS): String { + val digest = MessageDigest.getInstance("SHA-256").digest(text.toByteArray(Charsets.UTF_8)) + return digest.joinToString("") { b -> "%02x".format(b) }.take(hexChars) +} + class LogService(private val logRepository: LogRepository) { private val clickhouseDb: String get() = ClickHouseClient.getDatabase() private val json = Json { ignoreUnknownKeys = true } @@ -260,7 +294,7 @@ class LogService(private val logRepository: LogRepository) { organizationId: Long, request: LogQueryRequest ): LogQueryResponse { - val limit = request.limit.coerceIn(1, 500) + val limit = request.limit.coerceIn(1, MAX_LOG_QUERY_LIMIT) val conditions = mutableListOf() val totalCountFilter = @@ -318,7 +352,10 @@ class LogService(private val logRepository: LogRepository) { } } }.getOrElse { e -> - logger.error(e) { "Failed to parse query '${request.query}', falling back to simple search" } + val q = request.query.orEmpty() + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(q)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(request.query) } @@ -361,8 +398,10 @@ class LogService(private val logRepository: LogRepository) { val whereClause = conditions.joinToString(" AND ") - // Log the complete WHERE clause for debugging (at DEBUG level to avoid logging user data in production) - logger.debug { "Executing log query with WHERE clause: $whereClause" } + logger.debug { + "Executing log query: where_fp=${utf8Fingerprint(whereClause)} " + + "(where_len=${whereClause.length})" + } val query = """ @@ -393,9 +432,11 @@ class LogService(private val logRepository: LogRepository) { val body = logRepository.executeClickHouseQuery(query) if (body.isClickHouseError()) { - logger.error("ClickHouse query failed. WHERE clause: $whereClause") - logger.error("Full query: $query") - throw IllegalStateException("Failed to query logs: ${body.take(1000)}") + logger.error( + "ClickHouse query failed. where_fp=${utf8Fingerprint(whereClause)} " + + "query_fp=${utf8Fingerprint(query)}" + ) + throw IllegalStateException("Failed to query logs: ${body.take(ERROR_BODY_LONG_CHARS)}") } val parsed = parseQueryRows(body) @@ -443,16 +484,16 @@ class LogService(private val logRepository: LogRepository) { if (fromMs == null || toMs == null) return "1h" val rangeMs = toMs - fromMs return when { - rangeMs <= 3_600_000L -> "1m" + rangeMs <= MILLIS_PER_HOUR -> "1m" // ≤1h → 1m - rangeMs <= 21_600_000L -> "5m" + rangeMs <= MILLIS_PER_6_HOURS -> "5m" // ≤6h → 5m - rangeMs <= 86_400_000L -> "15m" + rangeMs <= MILLIS_PER_DAY -> "15m" // ≤24h → 15m - rangeMs <= 604_800_000L -> "1h" + rangeMs <= MILLIS_PER_WEEK -> "1h" // ≤7d → 1h else -> "1d" // >7d → 1d @@ -518,7 +559,9 @@ class LogService(private val logRepository: LogRepository) { } } }.getOrElse { e -> - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } @@ -573,12 +616,12 @@ class LogService(private val logRepository: LogRepository) { logger.debug { "Aggregate logs SQL for org $organizationId (fromMs=$fromMs, toMs=$toMs, interval=$chInterval, " + - "groupBy=$validGroupBy):\n$sql" + "groupBy=$validGroupBy): query_fp=${utf8Fingerprint(sql)}" } val body = logRepository.executeClickHouseQuery(sql) - logger.debug { "Aggregate logs response body (first 500 chars): ${body.take(500)}" } + logger.debug { "Aggregate logs response body (first 500 chars): ${body.take(WARN_BODY_PREVIEW_CHARS)}" } if (body.isClickHouseError()) { - logger.warn { "Failed to aggregate logs: ${body.take(600)}" } + logger.warn { "Failed to aggregate logs: ${body.take(ERROR_BODY_PREVIEW_CHARS)}" } return LogAggregateResponse(buckets = emptyList(), totalCount = 0, interval = resolvedInterval) } @@ -666,7 +709,9 @@ class LogService(private val logRepository: LogRepository) { } } }.getOrElse { e -> - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } @@ -692,7 +737,7 @@ class LogService(private val logRepository: LogRepository) { } val whereClause = conditions.joinToString(" AND ") - val safeLimit = limit.coerceIn(1, 100) + val safeLimit = limit.coerceIn(1, MAX_TOP_VALUES_LIMIT) // Determine the SQL column expression for the field val columnExpr = @@ -721,7 +766,7 @@ class LogService(private val logRepository: LogRepository) { val body = logRepository.executeClickHouseQuery(sql) if (body.isClickHouseError()) { - logger.warn { "Failed to query top values: ${body.take(600)}" } + logger.warn { "Failed to query top values: ${body.take(ERROR_BODY_PREVIEW_CHARS)}" } return LogTopResponse(field = field, values = emptyList(), totalCount = 0) } @@ -808,19 +853,27 @@ class LogService(private val logRepository: LogRepository) { } } } catch (e: SerializationException) { - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } catch (e: IOException) { - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } catch (e: IllegalStateException) { - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } catch (e: IllegalArgumentException) { - logger.error(e) { "Failed to parse query '$query', falling back to simple search" } + logger.error(e) { + "Failed to parse query (query_fp=${utf8Fingerprint(query)}), falling back to simple search" + } // Fallback: treat as simple full-text search conditions += buildSimpleSearchCondition(query) } @@ -846,7 +899,7 @@ class LogService(private val logRepository: LogRepository) { } val whereClause = conditions.joinToString(" AND ") - val safeLimit = limit.coerceIn(1, 10_000) + val safeLimit = limit.coerceIn(1, MAX_EXPORT_LIMIT) val sql = """ @@ -862,11 +915,14 @@ class LogService(private val logRepository: LogRepository) { FORMAT JSONEachRow """.trimIndent() - logger.debug { "Export CSV SQL: $sql" } + logger.debug { "Export CSV SQL: query_fp=${utf8Fingerprint(sql)}" } val body = logRepository.executeClickHouseQuery(sql) if (body.isClickHouseError()) { - logger.error { "ClickHouse export error. SQL: $sql\nError: ${body.take(600)}" } - throw IllegalStateException("Failed to export logs: ${body.take(600)}") + logger.error { + "ClickHouse export error. query_fp=${utf8Fingerprint(sql)}\n" + + "Error: ${body.take(ERROR_BODY_PREVIEW_CHARS)}" + } + throw IllegalStateException("Failed to export logs: ${body.take(ERROR_BODY_PREVIEW_CHARS)}") } val sb = StringBuilder() @@ -966,7 +1022,7 @@ class LogService(private val logRepository: LogRepository) { private suspend fun queryValueCounts(query: String): List { val body = logRepository.executeClickHouseQuery(query) if (body.isClickHouseError()) { - logger.warn { "Failed to query value counts: ${body.take(600)}" } + logger.warn { "Failed to query value counts: ${body.take(ERROR_BODY_PREVIEW_CHARS)}" } return emptyList() } val results = mutableListOf() @@ -1097,7 +1153,7 @@ class LogService(private val logRepository: LogRepository) { FROM `$clickhouseDb`.logs WHERE $whereClause AND $fieldRef != '' ORDER BY tag_value - LIMIT ${limit.coerceIn(1, 200)} + LIMIT ${limit.coerceIn(1, MAX_FILTER_VALUES_LIMIT)} FORMAT TSV """.trimIndent() } else { @@ -1107,7 +1163,7 @@ class LogService(private val logRepository: LogRepository) { FROM `$clickhouseDb`.logs WHERE $whereClause AND tags['$escapedKey'] != '' ORDER BY tag_value - LIMIT ${limit.coerceIn(1, 200)} + LIMIT ${limit.coerceIn(1, MAX_FILTER_VALUES_LIMIT)} FORMAT TSV """.trimIndent() } @@ -1120,7 +1176,7 @@ class LogService(private val logRepository: LogRepository) { private suspend fun queryDistinctLines(query: String): List { val body = logRepository.executeClickHouseQuery(query) if (body.isClickHouseError()) { - logger.warn { "Failed to query log filter values: ${body.take(600)}" } + logger.warn { "Failed to query log filter values: ${body.take(ERROR_BODY_PREVIEW_CHARS)}" } return emptyList() } return body @@ -1240,17 +1296,17 @@ class LogService(private val logRepository: LogRepository) { logId = UUID.randomUUID().toString(), timestampMs = resolveTimestampMs(entry.timestamp, entry.timestampMs), level = normalizeLevel(entry.level), - message = trimTo(entry.message.orEmpty(), 8192), - body = trimTo(entry.body ?: entry.message.orEmpty(), 32768), - service = trimTo(entry.service.orEmpty(), 256), - environment = trimTo(entry.environment.orEmpty(), 128), - host = trimTo(entry.host.orEmpty(), 256), + message = trimTo(entry.message.orEmpty(), MAX_LOG_MESSAGE_CHARS), + body = trimTo(entry.body ?: entry.message.orEmpty(), MAX_LOG_BODY_CHARS), + service = trimTo(entry.service.orEmpty(), MAX_LOG_SERVICE_CHARS), + environment = trimTo(entry.environment.orEmpty(), MAX_LOG_ENVIRONMENT_CHARS), + host = trimTo(entry.host.orEmpty(), MAX_LOG_HOST_CHARS), source = normalizeSource(entry.source ?: "sdk"), - containerName = trimTo(entry.containerName.orEmpty(), 256), - containerId = trimTo(entry.containerId.orEmpty(), 128), - containerImage = trimTo(entry.containerImage.orEmpty(), 512), - traceId = trimTo(entry.traceId.orEmpty(), 128), - spanId = trimTo(entry.spanId.orEmpty(), 128), + containerName = trimTo(entry.containerName.orEmpty(), MAX_LOG_CONTAINER_NAME_CHARS), + containerId = trimTo(entry.containerId.orEmpty(), MAX_LOG_CONTAINER_ID_CHARS), + containerImage = trimTo(entry.containerImage.orEmpty(), MAX_LOG_CONTAINER_IMAGE_CHARS), + traceId = trimTo(entry.traceId.orEmpty(), MAX_LOG_TRACE_ID_CHARS), + spanId = trimTo(entry.spanId.orEmpty(), MAX_LOG_SPAN_ID_CHARS), tags = sanitizeMap(entry.tags), resourceAttributes = sanitizeMap(entry.resourceAttributes) ) @@ -1273,17 +1329,17 @@ class LogService(private val logRepository: LogRepository) { logId = UUID.randomUUID().toString(), timestampMs = resolveTimestampMs(entry.timestamp, entry.timestampMs), level = normalizeLevel(entry.level), - message = trimTo(entry.message.orEmpty(), 8192), - body = trimTo(entry.body ?: entry.message.orEmpty(), 32768), - service = trimTo(entry.service ?: entry.containerName.orEmpty(), 256), - environment = trimTo(entry.environment.orEmpty(), 128), - host = trimTo(entry.host.orEmpty(), 256), + message = trimTo(entry.message.orEmpty(), MAX_LOG_MESSAGE_CHARS), + body = trimTo(entry.body ?: entry.message.orEmpty(), MAX_LOG_BODY_CHARS), + service = trimTo(entry.service ?: entry.containerName.orEmpty(), MAX_LOG_SERVICE_CHARS), + environment = trimTo(entry.environment.orEmpty(), MAX_LOG_ENVIRONMENT_CHARS), + host = trimTo(entry.host.orEmpty(), MAX_LOG_HOST_CHARS), source = source, - containerName = trimTo(entry.containerName.orEmpty(), 256), - containerId = trimTo(entry.containerId.orEmpty(), 128), - containerImage = trimTo(entry.containerImage.orEmpty(), 512), - traceId = trimTo(entry.traceId.orEmpty(), 128), - spanId = trimTo(entry.spanId.orEmpty(), 128), + containerName = trimTo(entry.containerName.orEmpty(), MAX_LOG_CONTAINER_NAME_CHARS), + containerId = trimTo(entry.containerId.orEmpty(), MAX_LOG_CONTAINER_ID_CHARS), + containerImage = trimTo(entry.containerImage.orEmpty(), MAX_LOG_CONTAINER_IMAGE_CHARS), + traceId = trimTo(entry.traceId.orEmpty(), MAX_LOG_TRACE_ID_CHARS), + spanId = trimTo(entry.spanId.orEmpty(), MAX_LOG_SPAN_ID_CHARS), tags = sanitizeMap(entry.tags), resourceAttributes = sanitizeMap(entry.resourceAttributes), systemId = systemId @@ -1327,7 +1383,7 @@ class LogService(private val logRepository: LogRepository) { try { ExportLogsServiceRequest.parseFrom(bytes) } catch (e: InvalidProtocolBufferException) { - logger.warn { "Invalid OTLP protobuf logs payload: ${e.message?.take(500)}" } + logger.warn { "Invalid OTLP protobuf logs payload: ${e.message?.take(WARN_BODY_PREVIEW_CHARS)}" } return emptyList() } @@ -1436,7 +1492,11 @@ class LogService(private val logRepository: LogRepository) { val trimmed = value.trim() trimmed.toLongOrNull()?.let { numeric -> - return if (numeric > 1_000_000_000_000L) numeric else numeric * 1000 + // Use digit count to distinguish seconds (≤10 digits) from milliseconds (13 digits). + // A numeric threshold would misclassify valid pre-2001 millisecond timestamps + // (e.g. 946684800000 for 2000-01-01). + val digits = trimmed.trimStart('-') + return if (digits.length <= UNIX_EPOCH_SECONDS_MAX_DIGITS) numeric * MS_PER_SECOND else numeric } return suspendRunCatching { @@ -1504,9 +1564,9 @@ class LogService(private val logRepository: LogRepository) { if (input == null || input.isEmpty()) return emptyMap() return input .mapNotNull { (rawKey, rawValue) -> - val key = rawKey.trim().take(128) + val key = rawKey.trim().take(INGEST_KEY_MAX_CHARS) if (key.isBlank()) return@mapNotNull null - key to rawValue.trim().take(1024) + key to rawValue.trim().take(INGEST_VALUE_MAX_CHARS) }.toMap() } diff --git a/backend/src/main/kotlin/com/moneat/monitor/routes/InfraRoutes.kt b/backend/src/main/kotlin/com/moneat/monitor/routes/InfraRoutes.kt index 303afa00a..fb3c0d399 100644 --- a/backend/src/main/kotlin/com/moneat/monitor/routes/InfraRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/monitor/routes/InfraRoutes.kt @@ -41,6 +41,8 @@ private val json = Json { ignoreUnknownKeys = true } private const val DEFAULT_LIMIT = 100 private const val MAX_LIMIT = 500 +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 private fun getOrgIdsForUser(userId: Int): List { return transaction { @@ -521,7 +523,7 @@ fun Route.infraRoutes() { private suspend fun executeChQuery(query: String): List? { return runCatching { val response = ClickHouseClient.execute(query) - if (response.status.value !in 200..299) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { logger.warn { "ClickHouse query failed: ${response.status}" } return null } diff --git a/backend/src/main/kotlin/com/moneat/monitor/routes/MonitorRoutes.kt b/backend/src/main/kotlin/com/moneat/monitor/routes/MonitorRoutes.kt index 15398fd11..3da1730f0 100644 --- a/backend/src/main/kotlin/com/moneat/monitor/routes/MonitorRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/monitor/routes/MonitorRoutes.kt @@ -50,6 +50,7 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.koin.core.context.GlobalContext private const val DEFAULT_PROJECT_ID = 0L +private const val DEFAULT_LIMIT = 100 /** * Helper function to get organization IDs for a user from their memberships. @@ -363,7 +364,7 @@ fun Route.monitorRoutes( ensureHostAccessible(call, monitorService.getHostById(hostId), organizationIds) ?: return@get - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 100 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_LIMIT val cursor = call.request.queryParameters["cursor"] val query = call.request.queryParameters["query"] val levels = call.request.queryParameters.getAll("levels") ?: emptyList() diff --git a/backend/src/main/kotlin/com/moneat/monitor/services/MonitorAlertService.kt b/backend/src/main/kotlin/com/moneat/monitor/services/MonitorAlertService.kt index 946127dc8..a900e43f6 100644 --- a/backend/src/main/kotlin/com/moneat/monitor/services/MonitorAlertService.kt +++ b/backend/src/main/kotlin/com/moneat/monitor/services/MonitorAlertService.kt @@ -93,6 +93,7 @@ class MonitorAlertService( const val MIN_ALERT_INTERVAL_MINUTES = 15 // Don't spam alerts const val POLL_INTERVAL_SECONDS = 15 const val MIN_DATA_POINT_RATIO = 0.8 + private const val SECONDS_PER_MINUTE = 60 } /** @@ -874,7 +875,7 @@ class MonitorAlertService( val lastSeenText = run { - val minutesAgo = ((Clock.System.now() - lastSeenAt).inWholeSeconds / 60).toInt() + val minutesAgo = ((Clock.System.now() - lastSeenAt).inWholeSeconds / SECONDS_PER_MINUTE).toInt() "Last seen $minutesAgo minutes ago" } diff --git a/backend/src/main/kotlin/com/moneat/monitor/services/MonitorService.kt b/backend/src/main/kotlin/com/moneat/monitor/services/MonitorService.kt index 59fa9281b..5d9eb7650 100644 --- a/backend/src/main/kotlin/com/moneat/monitor/services/MonitorService.kt +++ b/backend/src/main/kotlin/com/moneat/monitor/services/MonitorService.kt @@ -71,6 +71,100 @@ class MonitorService( const val ALERT_SCOPE_HOST = "host" const val INFRA_LOOKBACK_DAYS = 7 const val MONITOR_HISTORY_CACHE_TTL_SECONDS = 30L + + // Time-range thresholds for historical downsampling (in seconds) + private const val ONE_HOUR_SECONDS = 3600L + private const val SIX_HOURS_SECONDS = 21600L + private const val ONE_DAY_SECONDS = 86400L + private const val ONE_WEEK_SECONDS = 604800L + + // Interval step sizes returned by the downsampling selector (in seconds) + private const val INTERVAL_TEN_SECONDS = 10 + private const val INTERVAL_ONE_MINUTE = 60 + private const val INTERVAL_FIVE_MINUTES = 300 + private const val INTERVAL_THIRTY_MINUTES = 1800 + private const val INTERVAL_ONE_HOUR = 3600 + + // Unit conversion: multiply epoch-seconds by this to get epoch-milliseconds + private const val MILLIS_PER_SECOND = 1000L + + // Container freshness: keep rows within (monitorInterval * multiplier) seconds, + // but never less than the minimum window. + private const val FRESHNESS_MONITOR_MULTIPLIER = 3 + private const val FRESHNESS_MIN_WINDOW_SECONDS = 300 + + // JSONCompact column indices — single-host latest metrics query + // SELECT: cpu_percent(0), mem_total(1), mem_used(2), mem_available(3), + // disk_total(4), disk_used(5), net_recv_bytes(6), net_sent_bytes(7), + // load_1(8), temp_max(9), gpu_percent(10), battery_percent(11) + private const val LATEST_COL_MEM_AVAILABLE = 3 + private const val LATEST_COL_DISK_TOTAL = 4 + private const val LATEST_COL_DISK_USED = 5 + private const val LATEST_COL_NET_RECV_BYTES = 6 + private const val LATEST_COL_NET_SENT_BYTES = 7 + private const val LATEST_COL_LOAD_1 = 8 + private const val LATEST_COL_TEMP_MAX = 9 + private const val LATEST_COL_GPU_PERCENT = 10 + private const val LATEST_COL_BATTERY_PERCENT = 11 + + // JSONCompact column indices — multi-host batch latest metrics query + // SELECT: host_id(0), cpu_percent(1), mem_total(2), mem_used(3), + // mem_available(4), disk_total(5), disk_used(6), net_recv_bytes(7), + // net_sent_bytes(8), load_1(9), temp_max(10), gpu_percent(11), + // battery_percent(12) + private const val BATCH_COL_MEM_USED = 3 + private const val BATCH_COL_MEM_AVAILABLE = 4 + private const val BATCH_COL_DISK_TOTAL = 5 + private const val BATCH_COL_DISK_USED = 6 + private const val BATCH_COL_NET_RECV_BYTES = 7 + private const val BATCH_COL_NET_SENT_BYTES = 8 + private const val BATCH_COL_LOAD_1 = 9 + private const val BATCH_COL_TEMP_MAX = 10 + private const val BATCH_COL_GPU_PERCENT = 11 + private const val BATCH_COL_BATTERY_PERCENT = 12 + + // JSONCompact column indices — historical metrics query + // SELECT: ts(0), cpu(1), mem(2), disk(3), net_recv(4), net_sent(5), + // load1(6), load5(7), load15(8), temp(9), gpu(10), battery(11) + private const val HIST_COL_DISK_PERCENT = 3 + private const val HIST_COL_NET_RECV_BYTES = 4 + private const val HIST_COL_NET_SENT_BYTES = 5 + private const val HIST_COL_LOAD_1 = 6 + private const val HIST_COL_LOAD_5 = 7 + private const val HIST_COL_LOAD_15 = 8 + private const val HIST_COL_TEMP_MAX = 9 + private const val HIST_COL_GPU_PERCENT = 10 + private const val HIST_COL_BATTERY_PERCENT = 11 + + // JSONCompact column indices — single-host container stats query + // SELECT: name(0), container_id(1), image(2), state(3), cpu_percent(4), + // mem_usage(5), mem_limit(6), net_rx_bytes(7), net_tx_bytes(8) + private const val CONTAINER_COL_MEM_USAGE = 5 + private const val CONTAINER_COL_MEM_LIMIT = 6 + private const val CONTAINER_COL_NET_RX_BYTES = 7 + private const val CONTAINER_COL_NET_TX_BYTES = 8 + + // JSONCompact column indices — infra containers query + // SELECT: host(0), container_id(1), name(2), image(3), state(4), cpu_percent(5), + // mem_usage(6), mem_limit(7), net_rx_bytes(8), net_tx_bytes(9), + // tags(10), timestamp(11) + private const val INFRA_COL_IMAGE = 3 + private const val INFRA_COL_STATE = 4 + private const val INFRA_COL_CPU_PERCENT = 5 + private const val INFRA_COL_MEM_USAGE = 6 + private const val INFRA_COL_MEM_LIMIT = 7 + private const val INFRA_COL_NET_RX_BYTES = 8 + private const val INFRA_COL_NET_TX_BYTES = 9 + private const val INFRA_COL_TAGS = 10 + private const val INFRA_COL_TIMESTAMP = 11 + + // JSONCompact column indices — container historical metrics query + // SELECT: ts(0), cpu(1), mem_used(2), mem_limit(3), net_recv(4), net_sent(5) + private const val CONT_HIST_COL_MEM_LIMIT = 3 + private const val CONT_HIST_COL_NET_RECV = 4 + private const val CONT_HIST_COL_NET_SENT = 5 + + private const val PERCENT_MULTIPLIER = 100 } private val clickhouseDb: String get() = ClickHouseClient.getDatabase() @@ -186,48 +280,48 @@ class MonitorService( ?.toLongOrNull() ?: 0 val memAvailable = data - .getOrNull(3) + .getOrNull(LATEST_COL_MEM_AVAILABLE) ?.toString() ?.replace("\"", "") ?.toLongOrNull() ?: 0 val diskTotal = data - .getOrNull(4) + .getOrNull(LATEST_COL_DISK_TOTAL) ?.toString() ?.replace("\"", "") ?.toLongOrNull() ?: 0 val diskUsed = data - .getOrNull(5) + .getOrNull(LATEST_COL_DISK_USED) ?.toString() ?.replace("\"", "") ?.toLongOrNull() ?: 0 val netRecvBytes = data - .getOrNull(6) + .getOrNull(LATEST_COL_NET_RECV_BYTES) ?.toString() ?.replace("\"", "") ?.toLongOrNull() ?: 0 val netSentBytes = data - .getOrNull(7) + .getOrNull(LATEST_COL_NET_SENT_BYTES) ?.toString() ?.replace("\"", "") ?.toLongOrNull() ?: 0 - val load1 = data.getOrNull(8)?.toString()?.toFloatOrNull() ?: 0f - val tempMax = data.getOrNull(9)?.toString()?.toFloatOrNull() - val gpuPercent = data.getOrNull(10)?.toString()?.toFloatOrNull() - val batteryPercent = data.getOrNull(11)?.toString()?.toFloatOrNull() + val load1 = data.getOrNull(LATEST_COL_LOAD_1)?.toString()?.toFloatOrNull() ?: 0f + val tempMax = data.getOrNull(LATEST_COL_TEMP_MAX)?.toString()?.toFloatOrNull() + val gpuPercent = data.getOrNull(LATEST_COL_GPU_PERCENT)?.toString()?.toFloatOrNull() + val batteryPercent = data.getOrNull(LATEST_COL_BATTERY_PERCENT)?.toString()?.toFloatOrNull() val effectiveMemUsed = if (memAvailable > 0) memTotal - memAvailable else memUsed return LatestMetrics( cpu_percent = cpuPercent, mem_total = memTotal, mem_used = effectiveMemUsed, - mem_percent = if (memTotal > 0) (effectiveMemUsed.toFloat() / memTotal * 100) else 0f, + mem_percent = if (memTotal > 0) (effectiveMemUsed.toFloat() / memTotal * PERCENT_MULTIPLIER) else 0f, disk_total = diskTotal, disk_used = diskUsed, - disk_percent = if (diskTotal > 0) (diskUsed.toFloat() / diskTotal * 100) else 0f, + disk_percent = if (diskTotal > 0) (diskUsed.toFloat() / diskTotal * PERCENT_MULTIPLIER) else 0f, net_recv_bytes = netRecvBytes, net_sent_bytes = netSentBytes, net_recv_mbps = null, @@ -288,25 +382,33 @@ class MonitorService( val rowHostId = arr.getOrNull(0)?.toString()?.replace("\"", "")?.toIntOrNull() ?: continue val cpuPercent = arr.getOrNull(1)?.toString()?.toFloatOrNull() ?: 0f val memTotal = arr.getOrNull(2)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val memUsed = arr.getOrNull(3)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val memAvailable = arr.getOrNull(4)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val diskTotal = arr.getOrNull(5)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val diskUsed = arr.getOrNull(6)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val netRecvBytes = arr.getOrNull(7)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val netSentBytes = arr.getOrNull(8)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L - val load1 = arr.getOrNull(9)?.toString()?.toFloatOrNull() ?: 0f - val tempMax = arr.getOrNull(10)?.toString()?.toFloatOrNull() - val gpuPercent = arr.getOrNull(11)?.toString()?.toFloatOrNull() - val batteryPercent = arr.getOrNull(12)?.toString()?.toFloatOrNull() + val memUsed = + arr.getOrNull(BATCH_COL_MEM_USED)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val memAvailable = + arr.getOrNull(BATCH_COL_MEM_AVAILABLE)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val diskTotal = + arr.getOrNull(BATCH_COL_DISK_TOTAL)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val diskUsed = + arr.getOrNull(BATCH_COL_DISK_USED)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val netRecvBytes = + arr.getOrNull(BATCH_COL_NET_RECV_BYTES)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val netSentBytes = + arr.getOrNull(BATCH_COL_NET_SENT_BYTES)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0L + val load1 = arr.getOrNull(BATCH_COL_LOAD_1)?.toString()?.toFloatOrNull() ?: 0f + val tempMax = arr.getOrNull(BATCH_COL_TEMP_MAX)?.toString()?.toFloatOrNull() + val gpuPercent = arr.getOrNull(BATCH_COL_GPU_PERCENT)?.toString()?.toFloatOrNull() + val batteryPercent = arr.getOrNull(BATCH_COL_BATTERY_PERCENT)?.toString()?.toFloatOrNull() val effectiveMemUsed = if (memAvailable > 0) memTotal - memAvailable else memUsed result[rowHostId] = LatestMetrics( cpu_percent = cpuPercent, mem_total = memTotal, mem_used = effectiveMemUsed, - mem_percent = if (memTotal > 0) (effectiveMemUsed.toFloat() / memTotal * 100) else 0f, + mem_percent = if (memTotal > 0) { + effectiveMemUsed.toFloat() / memTotal * PERCENT_MULTIPLIER + } else { 0f }, disk_total = diskTotal, disk_used = diskUsed, - disk_percent = if (diskTotal > 0) (diskUsed.toFloat() / diskTotal * 100) else 0f, + disk_percent = if (diskTotal > 0) (diskUsed.toFloat() / diskTotal * PERCENT_MULTIPLIER) else 0f, net_recv_bytes = netRecvBytes, net_sent_bytes = netSentBytes, net_recv_mbps = null, @@ -362,11 +464,11 @@ class MonitorService( val timeRange = effectiveTo - effectiveFrom val calculatedInterval = intervalSeconds ?: when { - timeRange <= 3600 -> 10 - timeRange <= 21600 -> 60 - timeRange <= 86400 -> 300 - timeRange <= 604800 -> 1800 - else -> 3600 + timeRange <= ONE_HOUR_SECONDS -> INTERVAL_TEN_SECONDS + timeRange <= SIX_HOURS_SECONDS -> INTERVAL_ONE_MINUTE + timeRange <= ONE_DAY_SECONDS -> INTERVAL_FIVE_MINUTES + timeRange <= ONE_WEEK_SECONDS -> INTERVAL_THIRTY_MINUTES + else -> INTERVAL_ONE_HOUR } val query = @@ -389,8 +491,8 @@ class MonitorService( FROM `$clickhouseDb`.metrics WHERE organization_id = ${host.organizationId} AND tags['host_id'] = '$hostId' - AND timestamp >= fromUnixTimestamp64Milli(${effectiveFrom * 1000}) - AND timestamp <= fromUnixTimestamp64Milli(${effectiveTo * 1000}) + AND timestamp >= fromUnixTimestamp64Milli(${effectiveFrom * MILLIS_PER_SECOND}) + AND timestamp <= fromUnixTimestamp64Milli(${effectiveTo * MILLIS_PER_SECOND}) GROUP BY ts ORDER BY ts FORMAT JSONCompact @@ -417,25 +519,25 @@ class MonitorService( timestamp = arr[0].toString().replace("\"", "").toLong(), cpu_percent = arr.getOrNull(1)?.toString()?.toFloatOrNull(), mem_percent = arr.getOrNull(2)?.toString()?.toFloatOrNull(), - disk_percent = arr.getOrNull(3)?.toString()?.toFloatOrNull(), + disk_percent = arr.getOrNull(HIST_COL_DISK_PERCENT)?.toString()?.toFloatOrNull(), net_recv_bytes = arr - .getOrNull(4) + .getOrNull(HIST_COL_NET_RECV_BYTES) ?.toString() ?.replace("\"", "") ?.toLongOrNull(), net_sent_bytes = arr - .getOrNull(5) + .getOrNull(HIST_COL_NET_SENT_BYTES) ?.toString() ?.replace("\"", "") ?.toLongOrNull(), - load_1 = arr.getOrNull(6)?.toString()?.toFloatOrNull(), - load_5 = arr.getOrNull(7)?.toString()?.toFloatOrNull(), - load_15 = arr.getOrNull(8)?.toString()?.toFloatOrNull(), - temp_max = arr.getOrNull(9)?.toString()?.toFloatOrNull(), - gpu_percent = arr.getOrNull(10)?.toString()?.toFloatOrNull(), - battery_percent = arr.getOrNull(11)?.toString()?.toFloatOrNull() + load_1 = arr.getOrNull(HIST_COL_LOAD_1)?.toString()?.toFloatOrNull(), + load_5 = arr.getOrNull(HIST_COL_LOAD_5)?.toString()?.toFloatOrNull(), + load_15 = arr.getOrNull(HIST_COL_LOAD_15)?.toString()?.toFloatOrNull(), + temp_max = arr.getOrNull(HIST_COL_TEMP_MAX)?.toString()?.toFloatOrNull(), + gpu_percent = arr.getOrNull(HIST_COL_GPU_PERCENT)?.toString()?.toFloatOrNull(), + battery_percent = arr.getOrNull(HIST_COL_BATTERY_PERCENT)?.toString()?.toFloatOrNull() ) } }.getOrElse { e -> @@ -460,7 +562,10 @@ class MonitorService( val host = getHostById(hostId) ?: return emptyList() val retentionDays = retentionPolicyService.getRetentionDaysForHost(hostId) ?: PricingTier.FREE.retentionDays val monitorIntervalSeconds = getTierConfig(host.organizationId).monitorIntervalSeconds - val freshnessWindowSeconds = max(monitorIntervalSeconds * 3, 300) + val freshnessWindowSeconds = max( + monitorIntervalSeconds * FRESHNESS_MONITOR_MULTIPLIER, + FRESHNESS_MIN_WINDOW_SECONDS, + ) val query = """ @@ -487,10 +592,12 @@ class MonitorService( data.map { row -> val arr = row.jsonArray - val memUsed = arr[5].toString().replace("\"", "").toLongOrNull() ?: 0 - val memLimit = arr[6].toString().replace("\"", "").toLongOrNull() ?: 1 - val netRecvBytes = arr.getOrNull(7)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0 - val netSentBytes = arr.getOrNull(8)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0 + val memUsed = arr[CONTAINER_COL_MEM_USAGE].toString().replace("\"", "").toLongOrNull() ?: 0 + val memLimit = arr[CONTAINER_COL_MEM_LIMIT].toString().replace("\"", "").toLongOrNull() ?: 1 + val netRecvBytes = + arr.getOrNull(CONTAINER_COL_NET_RX_BYTES)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0 + val netSentBytes = + arr.getOrNull(CONTAINER_COL_NET_TX_BYTES)?.toString()?.replace("\"", "")?.toLongOrNull() ?: 0 ContainerStats( name = arr[0].toString().replace("\"", ""), @@ -502,7 +609,7 @@ class MonitorService( mem_limit = memLimit, net_recv_bytes = netRecvBytes, net_sent_bytes = netSentBytes, - mem_percent = if (memLimit > 0) (memUsed.toFloat() / memLimit * 100) else 0f + mem_percent = if (memLimit > 0) (memUsed.toFloat() / memLimit * PERCENT_MULTIPLIER) else 0f ) } }.getOrElse { e -> @@ -586,7 +693,7 @@ class MonitorService( data.map { row -> val arr = row.jsonArray val tagsObj = suspendRunCatching { - arr.getOrNull(10)?.toString()?.let { t -> + arr.getOrNull(INFRA_COL_TAGS)?.toString()?.let { t -> json.parseToJsonElement(t.replace("\\\"", "\"")) .jsonObject .entries @@ -595,25 +702,25 @@ class MonitorService( }.getOrElse { _ -> emptyMap() } - val ts = arr.getOrNull(11)?.toString()?.replace("\"", "") ?: "" + val ts = arr.getOrNull(INFRA_COL_TIMESTAMP)?.toString()?.replace("\"", "") ?: "" mapOf( "host" to arr.getOrNull(0)?.toString()?.replace("\"", ""), "container_id" to arr.getOrNull(1)?.toString()?.replace("\"", ""), "containerId" to arr.getOrNull(1)?.toString()?.replace("\"", ""), "name" to arr.getOrNull(2)?.toString()?.replace("\"", ""), - "image" to arr.getOrNull(3)?.toString()?.replace("\"", ""), - "state" to arr.getOrNull(4)?.toString()?.replace("\"", ""), - "cpu_percent" to (arr.getOrNull(5)?.toString()?.toFloatOrNull() ?: 0f), - "cpuPercent" to (arr.getOrNull(5)?.toString()?.toFloatOrNull() ?: 0f), - "mem_usage" to (arr.getOrNull(6)?.toString()?.toLongOrNull() ?: 0L), - "memUsage" to (arr.getOrNull(6)?.toString()?.toLongOrNull() ?: 0L), - "mem_limit" to (arr.getOrNull(7)?.toString()?.toLongOrNull() ?: 0L), - "memLimit" to (arr.getOrNull(7)?.toString()?.toLongOrNull() ?: 0L), - "net_rx_bytes" to (arr.getOrNull(8)?.toString()?.toLongOrNull() ?: 0L), - "netRxBytes" to (arr.getOrNull(8)?.toString()?.toLongOrNull() ?: 0L), - "net_tx_bytes" to (arr.getOrNull(9)?.toString()?.toLongOrNull() ?: 0L), - "netTxBytes" to (arr.getOrNull(9)?.toString()?.toLongOrNull() ?: 0L), + "image" to arr.getOrNull(INFRA_COL_IMAGE)?.toString()?.replace("\"", ""), + "state" to arr.getOrNull(INFRA_COL_STATE)?.toString()?.replace("\"", ""), + "cpu_percent" to (arr.getOrNull(INFRA_COL_CPU_PERCENT)?.toString()?.toFloatOrNull() ?: 0f), + "cpuPercent" to (arr.getOrNull(INFRA_COL_CPU_PERCENT)?.toString()?.toFloatOrNull() ?: 0f), + "mem_usage" to (arr.getOrNull(INFRA_COL_MEM_USAGE)?.toString()?.toLongOrNull() ?: 0L), + "memUsage" to (arr.getOrNull(INFRA_COL_MEM_USAGE)?.toString()?.toLongOrNull() ?: 0L), + "mem_limit" to (arr.getOrNull(INFRA_COL_MEM_LIMIT)?.toString()?.toLongOrNull() ?: 0L), + "memLimit" to (arr.getOrNull(INFRA_COL_MEM_LIMIT)?.toString()?.toLongOrNull() ?: 0L), + "net_rx_bytes" to (arr.getOrNull(INFRA_COL_NET_RX_BYTES)?.toString()?.toLongOrNull() ?: 0L), + "netRxBytes" to (arr.getOrNull(INFRA_COL_NET_RX_BYTES)?.toString()?.toLongOrNull() ?: 0L), + "net_tx_bytes" to (arr.getOrNull(INFRA_COL_NET_TX_BYTES)?.toString()?.toLongOrNull() ?: 0L), + "netTxBytes" to (arr.getOrNull(INFRA_COL_NET_TX_BYTES)?.toString()?.toLongOrNull() ?: 0L), "tags" to tagsObj, "timestamp" to ts, "id" to arr.getOrNull(1)?.toString()?.replace("\"", "") @@ -657,11 +764,11 @@ class MonitorService( val timeRange = effectiveTo - effectiveFrom val calculatedInterval = intervalSeconds ?: when { - timeRange <= 3600 -> 10 - timeRange <= 21600 -> 60 - timeRange <= 86400 -> 300 - timeRange <= 604800 -> 1800 - else -> 3600 + timeRange <= ONE_HOUR_SECONDS -> INTERVAL_TEN_SECONDS + timeRange <= SIX_HOURS_SECONDS -> INTERVAL_ONE_MINUTE + timeRange <= ONE_DAY_SECONDS -> INTERVAL_FIVE_MINUTES + timeRange <= ONE_WEEK_SECONDS -> INTERVAL_THIRTY_MINUTES + else -> INTERVAL_ONE_HOUR } val escapedName = escapeSql(containerName) @@ -678,8 +785,8 @@ class MonitorService( WHERE organization_id = ${host.organizationId} AND tags['host_id'] = '$hostId' AND name = '$escapedName' - AND timestamp >= fromUnixTimestamp64Milli(${effectiveFrom * 1000}) - AND timestamp <= fromUnixTimestamp64Milli(${effectiveTo * 1000}) + AND timestamp >= fromUnixTimestamp64Milli(${effectiveFrom * MILLIS_PER_SECOND}) + AND timestamp <= fromUnixTimestamp64Milli(${effectiveTo * MILLIS_PER_SECOND}) GROUP BY ts ORDER BY ts FORMAT JSONCompact @@ -712,19 +819,19 @@ class MonitorService( ?.toLongOrNull(), mem_limit = arr - .getOrNull(3) + .getOrNull(CONT_HIST_COL_MEM_LIMIT) ?.toString() ?.replace("\"", "") ?.toLongOrNull(), net_recv_bytes = arr - .getOrNull(4) + .getOrNull(CONT_HIST_COL_NET_RECV) ?.toString() ?.replace("\"", "") ?.toLongOrNull(), net_sent_bytes = arr - .getOrNull(5) + .getOrNull(CONT_HIST_COL_NET_SENT) ?.toString() ?.replace("\"", "") ?.toLongOrNull() @@ -1009,7 +1116,7 @@ class MonitorService( ): Pair? { val retentionDays = retentionPolicyService.getRetentionDaysForHost(hostId) ?: PricingTier.FREE.retentionDays val nowEpochSeconds = Clock.System.now().epochSeconds - val oldestAllowed = nowEpochSeconds - (retentionDays * 86_400L) + val oldestAllowed = nowEpochSeconds - (retentionDays * ONE_DAY_SECONDS) val clampedFrom = max(fromTimestamp, oldestAllowed) val clampedTo = min(toTimestamp, nowEpochSeconds) if (clampedFrom > clampedTo) return null diff --git a/backend/src/main/kotlin/com/moneat/notifications/services/DiscordService.kt b/backend/src/main/kotlin/com/moneat/notifications/services/DiscordService.kt index 1904aa996..9ebb4d8b2 100644 --- a/backend/src/main/kotlin/com/moneat/notifications/services/DiscordService.kt +++ b/backend/src/main/kotlin/com/moneat/notifications/services/DiscordService.kt @@ -161,13 +161,17 @@ class DiscordService( ) companion object { + private const val DISCORD_COLOR_RED = 0xE01E5A + private const val DISCORD_COLOR_GREEN = 0x2EB67D + private const val DISCORD_COLOR_YELLOW = 0xECB22E + /** Builds embed for host metric alerts. Exposed for unit testing. */ internal fun buildHostAlertEmbed(p: DiscordService.HostAlertParams): DiscordEmbed = DiscordEmbed( title = "⚠️ Host Alert", description = "**${p.hostName}** triggered an alert", url = "${p.baseUrl}/monitoring/hosts/${p.hostId}", - color = 0xECB22E, + color = DISCORD_COLOR_YELLOW, fields = listOf( DiscordField("Host", p.hostName, true), DiscordField("Metric", p.metric, true), @@ -190,7 +194,7 @@ class DiscordService( title = "🔴 Host Down", description = "**$hostName** is not responding", url = "$baseUrl/monitoring/hosts/$hostId", - color = 0xE01E5A, + color = DISCORD_COLOR_RED, fields = listOf( DiscordField("Host", hostName, true), DiscordField("Last Seen", lastSeen, true) @@ -210,7 +214,7 @@ class DiscordService( title = "✅ Host Recovered", description = "**$hostName** is back online", url = "$baseUrl/monitoring/hosts/$hostId", - color = 0x2EB67D, + color = DISCORD_COLOR_GREEN, fields = listOf( DiscordField("Host", hostName, true), DiscordField("Status", "Online", true) @@ -222,7 +226,7 @@ class DiscordService( /** Builds embed for uptime monitor alerts. Exposed for unit testing. */ internal fun buildUptimeAlertEmbed(p: DiscordService.UptimeAlertParams): DiscordEmbed { val title = if (p.isDown) "🔴 Uptime Monitor Down" else "✅ Uptime Monitor Recovered" - val color = if (p.isDown) 0xE01E5A else 0x2EB67D + val color = if (p.isDown) DISCORD_COLOR_RED else DISCORD_COLOR_GREEN val statusText = when { p.errorMessage != null -> p.errorMessage p.statusCode != null -> "HTTP ${p.statusCode}" @@ -249,9 +253,9 @@ class DiscordService( /** Builds embed for dashboard alerts. Exposed for unit testing. */ internal fun buildDashboardAlertEmbed(p: DiscordService.DashboardAlertParams): DiscordEmbed { val color = when (p.severity) { - "CRITICAL", "HIGH" -> 0xE01E5A - "MEDIUM" -> 0xECB22E - else -> 0xECB22E + "CRITICAL", "HIGH" -> DISCORD_COLOR_RED + "MEDIUM" -> DISCORD_COLOR_YELLOW + else -> DISCORD_COLOR_YELLOW } return DiscordEmbed( title = "📊 Dashboard Alert: ${p.alertName}", @@ -272,9 +276,9 @@ class DiscordService( /** Builds embed for error/issue alerts. Exposed for unit testing. */ internal fun buildErrorAlertEmbed(p: DiscordService.ErrorAlertParams): DiscordEmbed { val color = when (p.level.lowercase()) { - "fatal", "error" -> 0xE01E5A - "warning" -> 0xECB22E - else -> 0x2EB67D + "fatal", "error" -> DISCORD_COLOR_RED + "warning" -> DISCORD_COLOR_YELLOW + else -> DISCORD_COLOR_GREEN } return DiscordEmbed( title = "🐛 New Issue Detected", @@ -303,7 +307,7 @@ class DiscordService( title = "✅ Discord Integration Test", description = "Your Discord integration is working correctly!", url = baseUrl, - color = 0x2EB67D, + color = DISCORD_COLOR_GREEN, fields = listOf( DiscordField("Status", "Connected", true), DiscordField("Guild ID", guildId, true) diff --git a/backend/src/main/kotlin/com/moneat/notifications/services/EmailService.kt b/backend/src/main/kotlin/com/moneat/notifications/services/EmailService.kt index 0c0e955d1..25788bbb5 100644 --- a/backend/src/main/kotlin/com/moneat/notifications/services/EmailService.kt +++ b/backend/src/main/kotlin/com/moneat/notifications/services/EmailService.kt @@ -59,6 +59,7 @@ private const val BADGE_NEGATIVE = "background-color:#fef2f2;border:1px solid #fecaca;color:#dc2626;" private const val BADGE_NEUTRAL = "font-weight:500;background-color:#f5f5f5;" + "border:1px solid #e5e5e5;color:#737373;" +private const val TOP_ISSUES_COUNT = 5 class EmailService { private val config = ApplicationConfig("application.conf") @@ -529,6 +530,8 @@ class EmailService { ) { val subject = "Your Weekly Summary: ${data.totalEvents} events, ${data.newIssues} new issues" val htmlBody = loadWeeklySummaryTemplate(data) + val topIssuesList = data.topIssues.take(TOP_ISSUES_COUNT) + .joinToString("\n") { "- ${it.title} (${it.project}): ${it.count} events" } val textBody = """ Your Weekly Summary (${data.startDate} – ${data.endDate}) @@ -539,7 +542,7 @@ class EmailService { - Affected Users: ${data.affectedUsers} (${if (data.usersTrend > 0) "+" else ""}${data.usersTrend}%) TOP ISSUES: - ${data.topIssues.take(5).joinToString("\n") { "- ${it.title} (${it.project}): ${it.count} events" }} + $topIssuesList Open Dashboard: ${data.dashboardUrl} Manage preferences: ${data.settingsUrl} diff --git a/backend/src/main/kotlin/com/moneat/notifications/services/NotificationService.kt b/backend/src/main/kotlin/com/moneat/notifications/services/NotificationService.kt index d35074e4e..29250f507 100644 --- a/backend/src/main/kotlin/com/moneat/notifications/services/NotificationService.kt +++ b/backend/src/main/kotlin/com/moneat/notifications/services/NotificationService.kt @@ -57,6 +57,15 @@ import java.util.concurrent.TimeUnit private val logger = KotlinLogging.logger {} +private const val STACK_FRAMES_COUNT = 5 +private const val EPOCH_SECONDS_TO_MILLIS = 1000 +private const val WEEKLY_SUMMARY_DAYS = 7L +private const val ERROR_BODY_PREVIEW_CHARS = 500 +private const val WEEKLY_SUMMARY_HOUR = 9 +private const val FULL_PERCENTAGE = 100 +private const val MILLION = 1_000_000L +private const val THOUSAND = 1_000L + class NotificationService( private val emailService: EmailService, private val slackService: SlackService = SlackService(), @@ -161,7 +170,7 @@ class NotificationService( ?.firstOrNull() ?.stacktrace ?.frames - ?.takeLast(5) + ?.takeLast(STACK_FRAMES_COUNT) ?.joinToString("\n") { frame -> " at ${frame.function ?: "unknown"} (${frame.filename}:${frame.lineno})" } ?: "No stack trace available" @@ -187,7 +196,7 @@ class NotificationService( timestamp = event.timestamp?.let { java.time.Instant - .ofEpochMilli((it * 1000).toLong()) + .ofEpochMilli((it * EPOCH_SECONDS_TO_MILLIS).toLong()) .toString() } ?: java.time.Instant .now() @@ -286,8 +295,8 @@ class NotificationService( val now = Instant.now() val endDate = now - val startDate = now.minus(Duration.ofDays(7)) - val priorStartDate = startDate.minus(Duration.ofDays(7)) + val startDate = now.minus(Duration.ofDays(WEEKLY_SUMMARY_DAYS)) + val priorStartDate = startDate.minus(Duration.ofDays(WEEKLY_SUMMARY_DAYS)) // Get all users with weekly summary enabled val usersToNotify = @@ -323,8 +332,8 @@ class NotificationService( suspend fun sendWeeklySummaryForUser(userId: Int, email: String) { val now = Instant.now() val endDate = now - val startDate = now.minus(Duration.ofDays(7)) - val priorStartDate = startDate.minus(Duration.ofDays(7)) + val startDate = now.minus(Duration.ofDays(WEEKLY_SUMMARY_DAYS)) + val priorStartDate = startDate.minus(Duration.ofDays(WEEKLY_SUMMARY_DAYS)) sendUserWeeklySummary(userId, email, startDate, endDate, priorStartDate) } @@ -435,7 +444,8 @@ class NotificationService( val response = ClickHouseClient.execute(query) val responseBody = response.bodyAsText() if (!response.status.isSuccess()) { - logger.error("ClickHouse query failed in getStatsForPeriod: ${response.status} - ${responseBody.take(500)}") + val preview = responseBody.take(ERROR_BODY_PREVIEW_CHARS) + logger.error("ClickHouse query failed in getStatsForPeriod: ${response.status} - $preview") return PeriodStats(totalEvents = 0, uniqueIssues = 0, uniqueUsers = 0) } val jsonResponse = Json.parseToJsonElement(responseBody).jsonObject @@ -511,7 +521,8 @@ class NotificationService( val response = ClickHouseClient.execute(query) val responseBody = response.bodyAsText() if (!response.status.isSuccess()) { - logger.error("ClickHouse query failed in getTopIssues: ${response.status} - ${responseBody.take(500)}") + val preview = responseBody.take(ERROR_BODY_PREVIEW_CHARS) + logger.error("ClickHouse query failed in getTopIssues: ${response.status} - $preview") return emptyList() } val jsonResponse = Json.parseToJsonElement(responseBody).jsonObject @@ -597,7 +608,7 @@ class NotificationService( var nextRun = now .with(DayOfWeek.MONDAY) - .withHour(9) + .withHour(WEEKLY_SUMMARY_HOUR) .withMinute(0) .withSecond(0) .withNano(0) @@ -608,7 +619,7 @@ class NotificationService( } val initialDelay = Duration.between(now, nextRun).toMillis() - val period = Duration.ofDays(7).toMillis() + val period = Duration.ofDays(WEEKLY_SUMMARY_DAYS).toMillis() logger.info { "Scheduling weekly summary for $nextRun" } @@ -632,14 +643,14 @@ class NotificationService( current: Long, previous: Long ): Int { - if (previous == 0L) return if (current > 0) 100 else 0 - return ((current - previous) * 100 / previous).toInt() + if (previous == 0L) return if (current > 0) FULL_PERCENTAGE else 0 + return ((current - previous) * FULL_PERCENTAGE / previous).toInt() } private fun formatNumber(num: Long): String { return when { - num >= 1_000_000 -> String.format(Locale.US, "%.1fM", num / 1_000_000.0) - num >= 1_000 -> String.format(Locale.US, "%.1fK", num / 1_000.0) + num >= MILLION -> String.format(Locale.US, "%.1fM", num / MILLION.toDouble()) + num >= THOUSAND -> String.format(Locale.US, "%.1fK", num / THOUSAND.toDouble()) else -> num.toString() } } diff --git a/backend/src/main/kotlin/com/moneat/notifications/services/SlackService.kt b/backend/src/main/kotlin/com/moneat/notifications/services/SlackService.kt index d1bbd75f7..1bbb28d51 100644 --- a/backend/src/main/kotlin/com/moneat/notifications/services/SlackService.kt +++ b/backend/src/main/kotlin/com/moneat/notifications/services/SlackService.kt @@ -44,6 +44,8 @@ import org.slf4j.LoggerFactory import com.moneat.utils.suspendRunCatching import java.util.* +private const val SLACK_CHANNEL_FETCH_LIMIT = 200 + class SlackService { private val logger = LoggerFactory.getLogger(SlackService::class.java) private val json = Json { ignoreUnknownKeys = true } @@ -835,7 +837,7 @@ class SlackService { httpClient.get("https://slack.com/api/conversations.list") { header("Authorization", "Bearer $accessToken") parameter("types", "public_channel,private_channel") - parameter("limit", 200) + parameter("limit", SLACK_CHANNEL_FETCH_LIMIT) } val result = json.decodeFromString(response.bodyAsText()) diff --git a/backend/src/main/kotlin/com/moneat/org/routes/AdminRoutes.kt b/backend/src/main/kotlin/com/moneat/org/routes/AdminRoutes.kt index 871be9357..f820dadc4 100644 --- a/backend/src/main/kotlin/com/moneat/org/routes/AdminRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/org/routes/AdminRoutes.kt @@ -123,7 +123,7 @@ private suspend fun queryReceivedTelemetry(): ReceivedTelemetryStatus { val body = response.bodyAsText().trim() if (response.isClickHouseError(body)) { - logger.warn { "Failed to query telemetry_pulses: ${body.take(200)}" } + logger.warn { "Failed to query telemetry_pulses: ${body.take(LOG_BODY_PREVIEW_LENGTH)}" } return ReceivedTelemetryStatus(deploymentCount = 0, lastSeenAt = null, deployments = emptyList()) } @@ -168,6 +168,12 @@ private data class AdminUsersResponse( val limit: Int ) +private const val LOG_BODY_PREVIEW_LENGTH = 200 +private const val DEFAULT_PAGE_LIMIT = 25 +private const val SMALL_PAGE_LIMIT = 10 +private const val LARGE_PAGE_LIMIT = 500 +private const val MEDIUM_PAGE_LIMIT = 100 + @Serializable private data class AdminImpersonationTokenResponse( val token: String @@ -236,7 +242,7 @@ fun Route.adminRoutes() { get("/organizations") { val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 25 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_PAGE_LIMIT val orgs = adminService.getAllOrganizations(page, limit) call.respond(orgs) } @@ -292,7 +298,7 @@ fun Route.adminRoutes() { } get("/top-consumers") { - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 10 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: SMALL_PAGE_LIMIT val consumers = adminService.getTopConsumers(limit) call.respond(consumers) } @@ -787,7 +793,7 @@ fun Route.adminRoutes() { get("/users") { val page = call.request.queryParameters["page"]?.toIntOrNull() ?: 1 - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 25 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_PAGE_LIMIT val search = call.request.queryParameters["search"] val users = adminService.getAllUsers(page, limit, search) val total = adminService.getTotalUserCount(search) @@ -887,7 +893,7 @@ fun Route.adminRoutes() { } get("/subscriptions") { - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 500 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: LARGE_PAGE_LIMIT call.respond(pricingTierService.listAdminSubscriptions(limit)) } @@ -972,7 +978,7 @@ fun Route.adminRoutes() { } get("/promotional-credits") { - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 100 + val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: MEDIUM_PAGE_LIMIT val grants = adminBillingService.getAllPromotionalCreditGrants(limit) call.respond(grants) } diff --git a/backend/src/main/kotlin/com/moneat/org/routes/IntegrationRoutes.kt b/backend/src/main/kotlin/com/moneat/org/routes/IntegrationRoutes.kt index 75f59171c..5af716d9a 100644 --- a/backend/src/main/kotlin/com/moneat/org/routes/IntegrationRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/org/routes/IntegrationRoutes.kt @@ -110,6 +110,14 @@ data class SlackChannel( ) // Helper functions for secure state management +private const val MILLIS_PER_SECOND = 1000 +private const val SLACK_REQUEST_MAX_AGE_SECONDS = 300 +private const val NONCE_BYTES_SIZE = 16 +private const val OAUTH_PARTS_COUNT = 5 +private const val OAUTH_STATE_NONCE_INDEX = 3 +private const val OAUTH_STATE_SIGNATURE_INDEX = 4 +private const val OAUTH_STATE_MAX_AGE_MS = 600_000 +private const val DISCORD_BOT_PERMISSIONS = 85504 // 0x14C00 private val secureRandom = SecureRandom() private fun getStateSecret(): String { @@ -144,10 +152,10 @@ private fun verifySlackRequestSignature( val timestamp = headers["X-Slack-Request-Timestamp"] ?: return false val signature = headers["X-Slack-Signature"] ?: return false val requestTs = timestamp.toLongOrNull() ?: return false - val nowTs = System.currentTimeMillis() / 1000 + val nowTs = System.currentTimeMillis() / MILLIS_PER_SECOND // Reject stale/replayed payloads. - if (abs(nowTs - requestTs) > 60 * 5) return false + if (abs(nowTs - requestTs) > SLACK_REQUEST_MAX_AGE_SECONDS) return false val secret = getSlackSigningSecret() ?: run { @@ -166,7 +174,7 @@ private fun generateSecureState( userId: Int, organizationId: Int ): String { - val nonce = ByteArray(16) + val nonce = ByteArray(NONCE_BYTES_SIZE) secureRandom.nextBytes(nonce) val timestamp = System.currentTimeMillis() val payload = "$userId:$organizationId:$timestamp:${Base64.getUrlEncoder().withoutPadding().encodeToString(nonce)}" @@ -184,16 +192,16 @@ private fun validateAndDecodeState(state: String): Pair? { suspendRunCatching { val decoded = String(Base64.getUrlDecoder().decode(state)) val parts = decoded.split(":") - if (parts.size != 5) return null + if (parts.size != OAUTH_PARTS_COUNT) return null val userId = parts[0].toInt() val organizationId = parts[1].toInt() val timestamp = parts[2].toLong() - val nonce = parts[3] - val signature = parts[4] + val nonce = parts[OAUTH_STATE_NONCE_INDEX] + val signature = parts[OAUTH_STATE_SIGNATURE_INDEX] // Check if state is expired (10 minutes) - if (System.currentTimeMillis() - timestamp > 10 * 60 * 1000) { + if (System.currentTimeMillis() - timestamp > OAUTH_STATE_MAX_AGE_MS) { return null } @@ -544,7 +552,7 @@ fun Route.integrationRoutes() { // Discord permissions: Send Messages (0x800), Embed Links (0x4000), // Read Message History (0x10000), View Channels (0x400) - val permissions = 85504 // 0x14C00 + val permissions = DISCORD_BOT_PERMISSIONS val scopes = "bot+guilds" val state = generateSecureState(userId, organizationId) diff --git a/backend/src/main/kotlin/com/moneat/org/services/AdminService.kt b/backend/src/main/kotlin/com/moneat/org/services/AdminService.kt index 8cdfa1b9f..bb0ab3617 100644 --- a/backend/src/main/kotlin/com/moneat/org/services/AdminService.kt +++ b/backend/src/main/kotlin/com/moneat/org/services/AdminService.kt @@ -253,6 +253,21 @@ class AdminService( companion object { private const val USABLE_STORAGE_BYTES = 35L * 1024 * 1024 * 1024 // 35GB from MONETIZATION.md + private const val PRO_PLAN_PRICE = 19.0 + private const val TEAM_PLAN_PRICE = 49.0 + private const val TEAM_CLOUD_COST_PER_ORG = 6.0 + private const val EVENTS_HISTORY_DAYS = 365 + private const val PERCENT_MULTIPLIER = 100.0 + private const val PERIOD_7D_DAYS_BACK = 6 + private const val PERIOD_30D_DAYS_BACK = 29 + private const val CLICKHOUSE_TABLE_MIN_COLUMNS = 4 + private const val CLICKHOUSE_BYTES_COLUMN_INDEX = 3 + private const val STORAGE_WARNING_THRESHOLD_PCT = 70.0 + private const val STORAGE_CRITICAL_THRESHOLD_PCT = 80.0 + private const val SES_COST_PER_EMAIL = 0.0001 + private const val BYTES_PER_KB = 1024L + private const val BYTES_PER_MB = BYTES_PER_KB * BYTES_PER_KB + private const val BYTES_PER_GB = BYTES_PER_MB * BYTES_PER_KB } private fun applyUserSearchFilter( @@ -293,7 +308,7 @@ class AdminService( val (allTimeEvents, last30Events, eventsTimeline) = queryClickHouseEvents( - today.minus(365, DateTimeUnit.DAY), + today.minus(EVENTS_HISTORY_DAYS, DateTimeUnit.DAY), today ) @@ -304,8 +319,8 @@ class AdminService( .where { Subscriptions.status eq "active" } .mapNotNull { row -> when (row[Subscriptions.plan].lowercase()) { - "pro" -> 19.0 - "team" -> 49.0 + "pro" -> PRO_PLAN_PRICE + "team" -> TEAM_PLAN_PRICE else -> null } }.sum() @@ -377,12 +392,12 @@ class AdminService( val tier = pricingTierService.getEffectiveTierForOrganization(orgId).tier val quotaPct = when { - tier.monthlyGbLimit > 0 -> (bytesCount.toDouble() / tier.monthlyGbLimit * 100).coerceAtMost( - 100.0 - ) - tier.monthlyUnitLimit > 0 -> (usage.toDouble() / tier.monthlyUnitLimit * 100).coerceAtMost( - 100.0 - ) + tier.monthlyGbLimit > 0 -> + (bytesCount.toDouble() / tier.monthlyGbLimit * PERCENT_MULTIPLIER) + .coerceAtMost(PERCENT_MULTIPLIER) + tier.monthlyUnitLimit > 0 -> + (usage.toDouble() / tier.monthlyUnitLimit * PERCENT_MULTIPLIER) + .coerceAtMost(PERCENT_MULTIPLIER) else -> null } @@ -443,10 +458,12 @@ class AdminService( val tier = pricingTierService.getEffectiveTierForOrganization(orgId).tier val quotaPct = when { - tier.monthlyGbLimit > 0 -> (bytesCount.toDouble() / tier.monthlyGbLimit * 100).coerceAtMost(100.0) - tier.monthlyUnitLimit > 0 -> (eventCount.toDouble() / tier.monthlyUnitLimit * 100).coerceAtMost( - 100.0 - ) + tier.monthlyGbLimit > 0 -> + (bytesCount.toDouble() / tier.monthlyGbLimit * PERCENT_MULTIPLIER) + .coerceAtMost(PERCENT_MULTIPLIER) + tier.monthlyUnitLimit > 0 -> + (eventCount.toDouble() / tier.monthlyUnitLimit * PERCENT_MULTIPLIER) + .coerceAtMost(PERCENT_MULTIPLIER) else -> null } @@ -505,9 +522,9 @@ class AdminService( val daysBack = when (period) { "24h" -> 0 - "7d" -> 6 - "30d" -> 29 - else -> 6 + "7d" -> PERIOD_7D_DAYS_BACK + "30d" -> PERIOD_30D_DAYS_BACK + else -> PERIOD_7D_DAYS_BACK } val startDate = today.minus(daysBack, DateTimeUnit.DAY) @@ -562,8 +579,8 @@ class AdminService( val mrr = subsByPlan.entries.sumOf { (plan, count) -> when (plan) { - "pro" -> count * 19.0 - "team" -> count * 49.0 + "pro" -> count * PRO_PLAN_PRICE + "team" -> count * TEAM_PLAN_PRICE else -> 0.0 } } @@ -579,7 +596,7 @@ class AdminService( mapOf( "free" to 0.0, "pro" to 2.0, - "team" to 6.0 + "team" to TEAM_CLOUD_COST_PER_ORG ) return AdminRevenueMetrics( mrr = mrr, @@ -604,39 +621,24 @@ class AdminService( """.trimIndent() val response = ClickHouseClient.execute(query) if (response.status.isSuccess()) { - val text = response.bodyAsText() - val lines = text.trim().split("\n").filter { it.isNotBlank() } - for (line in lines) { - val parts = line.split("\t") - if (parts.size >= 4) { - val db = parts[0] - val rawTable = parts[1] - val rows = parts[2].toLongOrNull() ?: 0L - val bytes = parts[3].toLongOrNull() ?: 0L - // Prefix with database name for non-app tables to disambiguate - val tableName = if (db == clickhouseDb) rawTable else "$db.$rawTable" - tables.add( - TableSize( - table = tableName, - rows = rows, - bytesOnDisk = bytes, - bytesOnDiskFormatted = formatBytes(bytes) - ) - ) - totalBytes += bytes - totalRows += rows - } - } + val parsed = parseClickHouseParts(response.bodyAsText()) + tables.addAll(parsed) + totalBytes += parsed.sumOf { it.bytesOnDisk } + totalRows += parsed.sumOf { it.rows } } }.getOrElse { e -> logger.error(e) { "Failed to query ClickHouse system.parts" } } val storageUsedPercent = - if (USABLE_STORAGE_BYTES > 0) (totalBytes.toDouble() / USABLE_STORAGE_BYTES * 100) else 0.0 + if (USABLE_STORAGE_BYTES > 0) (totalBytes.toDouble() / USABLE_STORAGE_BYTES * PERCENT_MULTIPLIER) else 0.0 val alerts = mutableListOf() - if (storageUsedPercent > 70) alerts.add("Storage > 70% (consider adding block storage)") - if (storageUsedPercent > 80) alerts.add("Storage > 80% (scaling trigger)") + if (storageUsedPercent > STORAGE_WARNING_THRESHOLD_PCT) { + alerts.add("Storage > 70% (consider adding block storage)") + } + if (storageUsedPercent > STORAGE_CRITICAL_THRESHOLD_PCT) { + alerts.add("Storage > 80% (scaling trigger)") + } return AdminInfrastructureHealth( clickhouseTables = tables.sortedByDescending { it.bytesOnDisk }, @@ -647,6 +649,30 @@ class AdminService( ) } + private fun parseClickHouseParts(text: String): List { + val result = mutableListOf() + for (line in text.trim().split("\n").filter { it.isNotBlank() }) { + val parts = line.split("\t") + if (parts.size >= CLICKHOUSE_TABLE_MIN_COLUMNS) { + val db = parts[0] + val rawTable = parts[1] + val rows = parts[2].toLongOrNull() ?: 0L + val bytes = parts[CLICKHOUSE_BYTES_COLUMN_INDEX].toLongOrNull() ?: 0L + // Prefix with database name for non-app tables to disambiguate + val tableName = if (db == clickhouseDb) rawTable else "$db.$rawTable" + result.add( + TableSize( + table = tableName, + rows = rows, + bytesOnDisk = bytes, + bytesOnDiskFormatted = formatBytes(bytes), + ), + ) + } + } + return result + } + fun getTopConsumers(limit: Int): List { usageTracker.flushBuffer() val today = @@ -711,9 +737,9 @@ class AdminService( val daysBack = when (period) { "24h" -> 1 - "7d" -> 7 - "30d" -> 30 - else -> 7 + "7d" -> PERIOD_7D_DAYS_BACK + "30d" -> PERIOD_30D_DAYS_BACK + else -> PERIOD_7D_DAYS_BACK } val startDate = today.minus(daysBack, DateTimeUnit.DAY) return usageTracker.getUsageForOrg(orgId, startDate, today) @@ -727,9 +753,9 @@ class AdminService( .date val daysBack = when (period) { - "7d" -> 7 - "30d" -> 30 - else -> 30 + "7d" -> PERIOD_7D_DAYS_BACK + "30d" -> PERIOD_30D_DAYS_BACK + else -> PERIOD_30D_DAYS_BACK } val startDate = today.minus(daysBack, DateTimeUnit.DAY) @@ -751,7 +777,7 @@ class AdminService( .mapValues { it.value.size.toLong() } // Timeline for last 7 days - val last7DaysStart = today.minus(7, DateTimeUnit.DAY).atStartOfDayIn(TimeZone.UTC) + val last7DaysStart = today.minus(PERIOD_7D_DAYS_BACK, DateTimeUnit.DAY).atStartOfDayIn(TimeZone.UTC) val last7Days = EmailsSent .selectAll() @@ -781,7 +807,7 @@ class AdminService( }.sortedBy { it.date } // Estimate cost - AWS SES costs $0.10 per 1,000 emails - val estimatedCost = totalSent * 0.0001 + val estimatedCost = totalSent * SES_COST_PER_EMAIL AdminEmailStats( totalSent = totalSent, @@ -845,10 +871,10 @@ class AdminService( private fun formatBytes(bytes: Long): String { return when { - bytes < 1024 -> "$bytes B" - bytes < 1024 * 1024 -> "%.1f KB".format(bytes / 1024.0) - bytes < 1024 * 1024 * 1024 -> "%.1f MB".format(bytes / (1024.0 * 1024)) - else -> "%.1f GB".format(bytes / (1024.0 * 1024 * 1024)) + bytes < BYTES_PER_KB -> "$bytes B" + bytes < BYTES_PER_MB -> "%.1f KB".format(bytes.toDouble() / BYTES_PER_KB) + bytes < BYTES_PER_GB -> "%.1f MB".format(bytes.toDouble() / BYTES_PER_MB) + else -> "%.1f GB".format(bytes.toDouble() / BYTES_PER_GB) } } diff --git a/backend/src/main/kotlin/com/moneat/org/services/OrgInvitationService.kt b/backend/src/main/kotlin/com/moneat/org/services/OrgInvitationService.kt index 154afce22..b1bfa3a8a 100644 --- a/backend/src/main/kotlin/com/moneat/org/services/OrgInvitationService.kt +++ b/backend/src/main/kotlin/com/moneat/org/services/OrgInvitationService.kt @@ -39,8 +39,12 @@ class OrgInvitationService( private val logger = LoggerFactory.getLogger(OrgInvitationService::class.java) private val random = SecureRandom() + companion object { + private const val TOKEN_BYTES_SIZE = 32 + } + private fun generateToken(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(TOKEN_BYTES_SIZE) random.nextBytes(bytes) return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes) } diff --git a/backend/src/main/kotlin/com/moneat/otlp/OtlpProtobufParser.kt b/backend/src/main/kotlin/com/moneat/otlp/OtlpProtobufParser.kt index 7496d3054..7afbb019c 100644 --- a/backend/src/main/kotlin/com/moneat/otlp/OtlpProtobufParser.kt +++ b/backend/src/main/kotlin/com/moneat/otlp/OtlpProtobufParser.kt @@ -23,6 +23,9 @@ import io.opentelemetry.proto.resource.v1.Resource private const val NANOS_PER_MILLI = 1_000_000L private val HEX_CHARS = "0123456789abcdef".toCharArray() +private const val BYTE_MASK = 0xFF +private const val HEX_SHIFT = 4 +private const val NIBBLE_MASK = 0x0F /** * Shared utilities for converting OTLP protobuf objects to Moneat internal models. @@ -66,14 +69,13 @@ object OtlpProtobufParser { return nanos / NANOS_PER_MILLI } - @Suppress("MagicNumber") fun bytesToHex(bytes: ByteString): String { if (bytes.isEmpty) return "" val hex = CharArray(bytes.size() * 2) bytes.forEachIndexed { i, b -> - val v = b.toInt() and 0xFF - hex[i * 2] = HEX_CHARS[v ushr 4] - hex[i * 2 + 1] = HEX_CHARS[v and 0x0F] + val v = b.toInt() and BYTE_MASK + hex[i * 2] = HEX_CHARS[v ushr HEX_SHIFT] + hex[i * 2 + 1] = HEX_CHARS[v and NIBBLE_MASK] } return String(hex) } diff --git a/backend/src/main/kotlin/com/moneat/otlp/services/OtlpMetricsService.kt b/backend/src/main/kotlin/com/moneat/otlp/services/OtlpMetricsService.kt index 31a4754ef..e5ac74304 100644 --- a/backend/src/main/kotlin/com/moneat/otlp/services/OtlpMetricsService.kt +++ b/backend/src/main/kotlin/com/moneat/otlp/services/OtlpMetricsService.kt @@ -47,6 +47,7 @@ private val logger = KotlinLogging.logger {} // OTLP AggregationTemporality (opentelemetry.proto.metrics.v1.AggregationTemporality) private const val AGGREGATION_TEMPORALITY_DELTA = 1 private const val AGGREGATION_TEMPORALITY_CUMULATIVE = 2 +private const val ERROR_MESSAGE_PREVIEW_LENGTH = 500 @Serializable data class OtlpMetricInsert( @@ -137,7 +138,9 @@ class OtlpMetricsService( try { ExportMetricsServiceRequest.parseFrom(bytes) } catch (e: InvalidProtocolBufferException) { - logger.warn { "Invalid OTLP protobuf metrics payload: ${e.message?.take(500)}" } + logger.warn { + "Invalid OTLP protobuf metrics payload: ${e.message?.take(ERROR_MESSAGE_PREVIEW_LENGTH)}" + } return null } diff --git a/backend/src/main/kotlin/com/moneat/otlp/services/OtlpTraceService.kt b/backend/src/main/kotlin/com/moneat/otlp/services/OtlpTraceService.kt index b0ac80ec4..5c7fa8908 100644 --- a/backend/src/main/kotlin/com/moneat/otlp/services/OtlpTraceService.kt +++ b/backend/src/main/kotlin/com/moneat/otlp/services/OtlpTraceService.kt @@ -58,6 +58,7 @@ private const val OTLP_SPAN_KIND_SERVER = 2 private const val OTLP_SPAN_KIND_CLIENT = 3 private const val OTLP_SPAN_KIND_PRODUCER = 4 private const val OTLP_SPAN_KIND_CONSUMER = 5 +private const val ERROR_MESSAGE_PREVIEW_LENGTH = 500 @Serializable data class OtlpSpanInsert( @@ -102,7 +103,7 @@ class OtlpTraceService( try { ExportTraceServiceRequest.parseFrom(bytes) } catch (e: InvalidProtocolBufferException) { - logger.warn { "Invalid OTLP protobuf traces payload: ${e.message?.take(500)}" } + logger.warn { "Invalid OTLP protobuf traces payload: ${e.message?.take(ERROR_MESSAGE_PREVIEW_LENGTH)}" } return null } diff --git a/backend/src/main/kotlin/com/moneat/plugins/BackgroundJobs.kt b/backend/src/main/kotlin/com/moneat/plugins/BackgroundJobs.kt index ff4004919..7babace3d 100644 --- a/backend/src/main/kotlin/com/moneat/plugins/BackgroundJobs.kt +++ b/backend/src/main/kotlin/com/moneat/plugins/BackgroundJobs.kt @@ -47,6 +47,7 @@ import kotlin.time.Duration.Companion.hours import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val DEFAULT_WORKER_THREADS = 4 fun Application.configureBackgroundJobs() { val backgroundJobsEnabled = @@ -154,7 +155,7 @@ fun Application.configureBackgroundJobs() { .propertyOrNull("pulse.intervalHours") ?.getString() ?.toIntOrNull() - ?.takeIf { it > 0 } ?: 4 + ?.takeIf { it > 0 } ?: DEFAULT_WORKER_THREADS PulseService(interval = telemetryIntervalHours.hours).also { logger.info { "Telemetry pulse enabled for self-hosted deployment" } it.start(jobScope) diff --git a/backend/src/main/kotlin/com/moneat/plugins/Databases.kt b/backend/src/main/kotlin/com/moneat/plugins/Databases.kt index 25f978a85..1952a1597 100644 --- a/backend/src/main/kotlin/com/moneat/plugins/Databases.kt +++ b/backend/src/main/kotlin/com/moneat/plugins/Databases.kt @@ -35,6 +35,9 @@ private const val CLICKHOUSE_MIGRATION_LOCK_KEY = 8675309L val ExposedDatabaseKey = AttributeKey("ExposedDatabase") private const val CLICKHOUSE_MIGRATION_LOCK_WAIT_TIMEOUT_MS = 120_000L private const val CLICKHOUSE_MIGRATION_LOCK_POLL_INTERVAL_MS = 1_000L +private const val DB_POOL_MIN_IDLE = 5 +private const val DB_CONNECTION_TIMEOUT_MS = 10000L +private const val DB_LEAK_DETECTION_THRESHOLD_MS = 30000L private fun tryAcquireAdvisoryLock(connection: Connection, lockKey: Long): Boolean = connection.createStatement().use { statement -> @@ -109,9 +112,9 @@ fun Application.configureDatabases() { username = config.property("database.postgres.user").getString() password = config.property("database.postgres.password").getString() maximumPoolSize = config.property("database.postgres.maxPoolSize").getString().toInt() - minimumIdle = 5 - connectionTimeout = 10000 - leakDetectionThreshold = 30000 + minimumIdle = DB_POOL_MIN_IDLE + connectionTimeout = DB_CONNECTION_TIMEOUT_MS + leakDetectionThreshold = DB_LEAK_DETECTION_THRESHOLD_MS isAutoCommit = false transactionIsolation = "TRANSACTION_READ_COMMITTED" validate() diff --git a/backend/src/main/kotlin/com/moneat/plugins/Monitoring.kt b/backend/src/main/kotlin/com/moneat/plugins/Monitoring.kt index 59f54a660..b9492561d 100644 --- a/backend/src/main/kotlin/com/moneat/plugins/Monitoring.kt +++ b/backend/src/main/kotlin/com/moneat/plugins/Monitoring.kt @@ -48,6 +48,13 @@ private val logger = KotlinLogging.logger {} private val SentryTransactionKey = AttributeKey("SentryTransaction") private val ingestPathRegex = Regex("^/api/[^/]+/(envelope|logs|store|security)/?$") +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 +private const val HTTP_CLIENT_ERROR_MIN = 400 +private const val HTTP_CLIENT_ERROR_MAX = 499 +private const val HTTP_SERVER_ERROR_MIN = 500 +private const val HTTP_SERVER_ERROR_MAX = 599 + fun Application.configureMonitoring() { // Sentry transaction interceptor for non-ingestion, non-health requests intercept(ApplicationCallPipeline.Setup) { @@ -90,9 +97,9 @@ fun Application.configureMonitoring() { // Determine transaction status based on HTTP status code transaction.status = when (status?.value) { - in 200..299 -> SpanStatus.OK - in 400..499 -> SpanStatus.INVALID_ARGUMENT - in 500..599 -> SpanStatus.INTERNAL_ERROR + in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX -> SpanStatus.OK + in HTTP_CLIENT_ERROR_MIN..HTTP_CLIENT_ERROR_MAX -> SpanStatus.INVALID_ARGUMENT + in HTTP_SERVER_ERROR_MIN..HTTP_SERVER_ERROR_MAX -> SpanStatus.INTERNAL_ERROR else -> SpanStatus.UNKNOWN_ERROR } diff --git a/backend/src/main/kotlin/com/moneat/plugins/RateLimiting.kt b/backend/src/main/kotlin/com/moneat/plugins/RateLimiting.kt index 61742d1a5..4d5fd94dd 100644 --- a/backend/src/main/kotlin/com/moneat/plugins/RateLimiting.kt +++ b/backend/src/main/kotlin/com/moneat/plugins/RateLimiting.kt @@ -64,13 +64,13 @@ private object TrustedProxies { val addr = InetAddress.getByName(ip).address val network = InetAddress.getByName(networkStr).address if (addr.size != network.size) return false - val fullBytes = prefixLen / 8 - val remainBits = prefixLen % 8 + val fullBytes = prefixLen / BITS_PER_BYTE + val remainBits = prefixLen % BITS_PER_BYTE for (i in 0 until fullBytes) { if (addr[i] != network[i]) return false } if (remainBits > 0) { - val mask = (0xFF shl (8 - remainBits)).toByte() + val mask = (BYTE_MASK shl (BITS_PER_BYTE - remainBits)).toByte() val addrBits = addr[fullBytes].toInt() and mask.toInt() val networkBits = network[fullBytes].toInt() and mask.toInt() if (addrBits != networkBits) return false @@ -104,6 +104,8 @@ private const val INGEST_RATE_LIMIT = 100 private const val INGEST_REFILL_SECONDS = 1 private const val TELEMETRY_RATE_LIMIT = 10 private const val TELEMETRY_REFILL_SECONDS = 60 +private const val BITS_PER_BYTE = 8 +private const val BYTE_MASK = 0xFF fun Application.configureRateLimiting() { install(RateLimit) { diff --git a/backend/src/main/kotlin/com/moneat/security/routes/SecurityRoutes.kt b/backend/src/main/kotlin/com/moneat/security/routes/SecurityRoutes.kt index ba9fdf680..e55747a86 100644 --- a/backend/src/main/kotlin/com/moneat/security/routes/SecurityRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/security/routes/SecurityRoutes.kt @@ -45,6 +45,11 @@ private val json = Json { ignoreUnknownKeys = true } private const val DEFAULT_LIMIT = 100 private const val MAX_LIMIT = 500 +private const val MAX_IDENTIFIER_LENGTH = 255 +private const val MAX_NAME_LENGTH = 64 +private const val MAX_SHORT_NAME_LENGTH = 32 +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 private fun getOrgIdsForUser(userId: Int): List { return transaction { @@ -71,13 +76,13 @@ private fun sanitizeSeverity(value: String?): String? = value?.takeIf { it.lowercase() in ALLOWED_SEVERITY }?.let { it } private fun sanitizeIdentifier(value: String?): String? = - value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= 255 } + value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= MAX_IDENTIFIER_LENGTH } private fun sanitizeFramework(value: String?): String? = - value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= 64 } + value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= MAX_NAME_LENGTH } private fun sanitizeStatus(value: String?): String? = - value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= 32 } + value?.takeIf { it.matches(SAFE_IDENTIFIER_REGEX) && it.length <= MAX_SHORT_NAME_LENGTH } private fun snakeToCamel(snake: String): String { return snake.split('_').mapIndexed { index, part -> @@ -103,7 +108,7 @@ private fun parseJsonEachRow(body: String): List { private suspend fun executeChQuery(query: String): List? { return runCatching { val response = ClickHouseClient.execute(query) - if (response.status.value !in 200..299) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { logger.warn { "ClickHouse query failed: ${response.status}" } return null } diff --git a/backend/src/main/kotlin/com/moneat/shared/services/AnalyticsSaltService.kt b/backend/src/main/kotlin/com/moneat/shared/services/AnalyticsSaltService.kt index 6e22ef93c..7e2c1d2f4 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/AnalyticsSaltService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/AnalyticsSaltService.kt @@ -41,6 +41,7 @@ private val logger = KotlinLogging.logger {} object AnalyticsSaltService { private const val SALT_TTL_SECONDS = 48L * 60 * 60 // 48 hours + private const val SALT_BYTES_SIZE = 32 fun getDailySalt(): String { val date = LocalDate.now().toString() @@ -65,7 +66,7 @@ object AnalyticsSaltService { } private fun generateSalt(): String { - val bytes = ByteArray(32) + val bytes = ByteArray(SALT_BYTES_SIZE) SecureRandom().nextBytes(bytes) return bytes.joinToString("") { "%02x".format(it) } } diff --git a/backend/src/main/kotlin/com/moneat/shared/services/ArtifactCleanupService.kt b/backend/src/main/kotlin/com/moneat/shared/services/ArtifactCleanupService.kt index 2f6f0106b..5186f358e 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/ArtifactCleanupService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/ArtifactCleanupService.kt @@ -41,6 +41,10 @@ class ArtifactCleanupService( ) { private var cleanupJob: Job? = null + companion object { + private const val INVITATION_EXPIRY_DAYS = 90 + } + fun start(scope: CoroutineScope) { logger.info { "Starting artifact cleanup service (auth tokens, invitations)" } cleanupJob = @@ -51,7 +55,7 @@ class ArtifactCleanupService( if (authDeleted > 0) { logger.info { "Cleaned up $authDeleted expired auth tokens" } } - val inviteDeleted = orgInvitationService.purgeOldInvitations(90) + val inviteDeleted = orgInvitationService.purgeOldInvitations(INVITATION_EXPIRY_DAYS) if (inviteDeleted > 0) { logger.info { "Purged $inviteDeleted old invitations" } } diff --git a/backend/src/main/kotlin/com/moneat/shared/services/AttributionAnalyticsService.kt b/backend/src/main/kotlin/com/moneat/shared/services/AttributionAnalyticsService.kt index 3eb1ef5a8..a78d67c4c 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/AttributionAnalyticsService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/AttributionAnalyticsService.kt @@ -56,6 +56,12 @@ data class AttributionSummary( class AttributionAnalyticsService { + companion object { + private const val MONTHS_PER_YEAR = 12 + private const val CENTS_PER_DOLLAR = 100 + private const val PERCENT_MULTIPLIER = 100.0 + } + fun getAttributionMetrics( groupBy: String = "campaign" // "source", "medium", "campaign", "all" ): AttributionAnalyticsResponse { @@ -143,7 +149,7 @@ class AttributionAnalyticsService { if (interval == "yearly") { BigDecimal( basePriceCents - ).divide(BigDecimal(12), 2, RoundingMode.HALF_UP) + ).divide(BigDecimal(MONTHS_PER_YEAR), 2, RoundingMode.HALF_UP) } else { BigDecimal(basePriceCents) } @@ -152,11 +158,15 @@ class AttributionAnalyticsService { null } }.fold(BigDecimal.ZERO) { acc, value -> acc + value } - .divide(BigDecimal(100), 2, RoundingMode.HALF_UP) // Convert cents to dollars + .divide( + BigDecimal(CENTS_PER_DOLLAR), + 2, + RoundingMode.HALF_UP + ) // Convert cents to dollars val conversionRate = if (signups > 0) { - (paidOrgs.toDouble() / signups.toDouble()) * 100 + (paidOrgs.toDouble() / signups.toDouble()) * PERCENT_MULTIPLIER } else { 0.0 } @@ -169,7 +179,7 @@ class AttributionAnalyticsService { } // Estimated LTV (assuming 12-month retention for simplicity) - val estimatedLtv = totalMrr.multiply(BigDecimal(12)) + val estimatedLtv = totalMrr.multiply(BigDecimal(MONTHS_PER_YEAR)) AttributionMetrics( source = source, @@ -203,7 +213,7 @@ class AttributionAnalyticsService { totalPaidOrganizations = totalPaid, overallConversionRate = if (totalSignups > 0) { - (totalPaid.toDouble() / totalSignups.toDouble()) * 100 + (totalPaid.toDouble() / totalSignups.toDouble()) * PERCENT_MULTIPLIER } else { 0.0 }, diff --git a/backend/src/main/kotlin/com/moneat/shared/services/PulseService.kt b/backend/src/main/kotlin/com/moneat/shared/services/PulseService.kt index 0f8c933c4..818684286 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/PulseService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/PulseService.kt @@ -74,7 +74,7 @@ class PulseService( pulseJob = scope.launch { // Delay the first pulse to let the application fully initialise - delay(60_000) + delay(PULSE_INTERVAL_MS) while (true) { suspendRunCatching { @@ -210,6 +210,8 @@ class PulseService( } companion object { + private const val PULSE_INTERVAL_MS = 60_000L + fun isEnabled(): Boolean { val selfHost = EnvConfig.SelfHost.enabled val telemetryEnabled = EnvConfig.get("TELEMETRY_ENABLED", "true").toBoolean() diff --git a/backend/src/main/kotlin/com/moneat/shared/services/RetentionBackgroundService.kt b/backend/src/main/kotlin/com/moneat/shared/services/RetentionBackgroundService.kt index 25b071a4c..0440c6c84 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/RetentionBackgroundService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/RetentionBackgroundService.kt @@ -55,6 +55,12 @@ class RetentionBackgroundService( private var sweepJob: Job? = null + companion object { + private const val MILLIS_PER_SECOND = 1000L + private const val HTTP_SUCCESS_MIN = 200 + private const val HTTP_SUCCESS_MAX = 299 + } + fun start(scope: CoroutineScope) { if (!enabled) { logger.info { "Retention background job is disabled by config" } @@ -69,7 +75,7 @@ class RetentionBackgroundService( }.onFailure { e -> logger.error(e) { "Retention sweep failed" } } - delay(sweepIntervalSeconds * 1000L) + delay(sweepIntervalSeconds * MILLIS_PER_SECOND) } } } @@ -284,7 +290,7 @@ class RetentionBackgroundService( } return suspendRunCatching { val response = ClickHouseClient.execute(query) - if (response.status.value !in 200..299) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { logger.error { "Retention mutation failed for $label (status=${response.status})" } false } else { diff --git a/backend/src/main/kotlin/com/moneat/shared/services/RetentionPolicyService.kt b/backend/src/main/kotlin/com/moneat/shared/services/RetentionPolicyService.kt index be1e9bcb2..40625f7a3 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/RetentionPolicyService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/RetentionPolicyService.kt @@ -31,6 +31,8 @@ class RetentionPolicyService( ) { companion object { const val RETENTION_CACHE_TTL_SECONDS = 300L + private const val FREE_TIER_LOG_RETENTION_DAYS = 3 + private const val DEFAULT_ANALYTICS_RETENTION_DAYS = 1095 } suspend fun getRetentionDaysForOrganization(organizationId: Int): Int { @@ -110,7 +112,7 @@ class RetentionPolicyService( logRetentionByOrg[orgId] = runCatching { getLogRetentionDaysForOrganization(orgId) - }.getOrDefault(3) // Default to FREE tier log retention (3 days) + }.getOrDefault(FREE_TIER_LOG_RETENTION_DAYS) // Default to FREE tier log retention } return logRetentionByOrg } @@ -164,7 +166,7 @@ class RetentionPolicyService( for (orgId in orgIds) { analyticsRetentionByOrg[orgId] = runCatching { getAnalyticsRetentionDaysForOrganization(orgId) } - .getOrDefault(1095) // Default 3 years for analytics + .getOrDefault(DEFAULT_ANALYTICS_RETENTION_DAYS) // Default 3 years for analytics } return analyticsRetentionByOrg } diff --git a/backend/src/main/kotlin/com/moneat/shared/services/SdkVersionService.kt b/backend/src/main/kotlin/com/moneat/shared/services/SdkVersionService.kt index d69b755c7..bd4db01bb 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/SdkVersionService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/SdkVersionService.kt @@ -70,6 +70,7 @@ object SdkVersionService { private const val MIN_CACHE_TTL_SECONDS = 300 private const val MAX_CACHE_TTL_SECONDS = 86_400 private const val GITHUB_API_BASE = "https://api.github.com" + private const val GITHUB_RELEASES_PER_PAGE = 20 private val versionTargets = listOf( @@ -222,7 +223,7 @@ object SdkVersionService { suspendRunCatching { httpClient.get("$GITHUB_API_BASE/repos/$repository/tags") { applyGitHubHeaders() - parameter("per_page", 20) + parameter("per_page", GITHUB_RELEASES_PER_PAGE) } }.getOrElse { e -> sdkVersionLogger.warn(e) { "Failed to fetch tags for $repository" } diff --git a/backend/src/main/kotlin/com/moneat/shared/services/UsageTrackingService.kt b/backend/src/main/kotlin/com/moneat/shared/services/UsageTrackingService.kt index d0786ed21..d442d32d2 100644 --- a/backend/src/main/kotlin/com/moneat/shared/services/UsageTrackingService.kt +++ b/backend/src/main/kotlin/com/moneat/shared/services/UsageTrackingService.kt @@ -70,6 +70,7 @@ class UsageTrackingService { /** Sentinel project ID for org-level usage (logs, etc.) when no specific project applies. */ const val ORG_PROJECT_ID_SENTINEL = 0L + private const val USAGE_RECORD_PARTS_COUNT = 4 } private val buffer = ConcurrentHashMap>() @@ -184,7 +185,7 @@ class UsageTrackingService { toFlush.mapNotNull { key -> val pair = buffer.remove(key) ?: return@mapNotNull null val parts = key.split("|") - if (parts.size != 4) return@mapNotNull null + if (parts.size != USAGE_RECORD_PARTS_COUNT) return@mapNotNull null val (orgId, projectId, eventType, dateStr) = parts UsageRecord( organizationId = orgId.toIntOrNull() ?: return@mapNotNull null, diff --git a/backend/src/main/kotlin/com/moneat/statuspage/services/StatusPageService.kt b/backend/src/main/kotlin/com/moneat/statuspage/services/StatusPageService.kt index 41f86286a..a66af1527 100644 --- a/backend/src/main/kotlin/com/moneat/statuspage/services/StatusPageService.kt +++ b/backend/src/main/kotlin/com/moneat/statuspage/services/StatusPageService.kt @@ -69,6 +69,10 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val HOURS_PER_DAY = 24 +private const val PERCENT_MULTIPLIER = 100.0 +private const val TOKEN_BYTES_SIZE = 32 + class StatusPageService( private val uptimeService: UptimeService = UptimeService(BillingQuotaService(), UptimeMonitorRepositoryImpl()) ) { @@ -666,7 +670,7 @@ class StatusPageService( val currentStatus = getCurrentMonitorStatus(data.monitorId) // Calculate uptime percentage - val uptimePercentage = uptimeService.getUptimePercentage(data.monitorId, historyDays * 24) + val uptimePercentage = uptimeService.getUptimePercentage(data.monitorId, historyDays * HOURS_PER_DAY) // Get uptime history if enabled val uptimeHistory = @@ -767,7 +771,7 @@ class StatusPageService( days: Int ): List { val now = Clock.System.now() - val from = now.minus((days * 24).hours) + val from = now.minus((days * HOURS_PER_DAY).hours) // Get daily uptime percentages val query = @@ -797,7 +801,8 @@ class StatusPageService( val upCount = json["up_count"]?.jsonPrimitive?.long ?: 0L val totalCount = json["total_count"]?.jsonPrimitive?.long ?: 0L - val uptime = if (totalCount == 0L) 0.0 else (upCount.toDouble() / totalCount.toDouble() * 100.0) + val uptime = + if (totalCount == 0L) 0.0 else (upCount.toDouble() / totalCount.toDouble() * PERCENT_MULTIPLIER) UptimeDataPoint(date = date, uptime = uptime) } catch (e: SerializationException) { @@ -815,7 +820,7 @@ class StatusPageService( private fun generateVerificationToken(): String { val random = SecureRandom() - val bytes = ByteArray(32) + val bytes = ByteArray(TOKEN_BYTES_SIZE) random.nextBytes(bytes) return bytes.joinToString("") { "%02x".format(it) } } diff --git a/backend/src/main/kotlin/com/moneat/summary/services/SummaryService.kt b/backend/src/main/kotlin/com/moneat/summary/services/SummaryService.kt index fb4f07a9a..79ce2b083 100644 --- a/backend/src/main/kotlin/com/moneat/summary/services/SummaryService.kt +++ b/backend/src/main/kotlin/com/moneat/summary/services/SummaryService.kt @@ -86,6 +86,16 @@ private const val STATUS_DOWN = "down" private const val SPIKE_FACTOR = 2 private const val WEEK_SECONDS = 7 * 24 * 3600L private const val TWO_WEEKS_SECONDS = 14 * 24 * 3600L +private const val SECONDS_PER_MINUTE = 60L +private const val MILLIS_PER_SECOND = 1000L +private const val SECONDS_PER_HOUR = 3600L +private const val HOURS_PER_DAY = 24L +private const val DAYS_IN_WEEK = 7L +private const val DAYS_IN_MONTH = 30L +private const val MILLIS_PER_HOUR = SECONDS_PER_HOUR * MILLIS_PER_SECOND +private const val MILLIS_PER_DAY = HOURS_PER_DAY * MILLIS_PER_HOUR +private const val MILLIS_7_DAYS = DAYS_IN_WEEK * MILLIS_PER_DAY +private const val MILLIS_30_DAYS = DAYS_IN_MONTH * MILLIS_PER_DAY class SummaryService( private val monitorService: MonitorService = MonitorService(HostRepositoryImpl(), HostAlertRepositoryImpl()), @@ -342,7 +352,7 @@ class SummaryService( val projectIds = getProjectIds(organizationId) val now = Instant.now() - val windowStart = now.minusSeconds(METRICS_WINDOW_MINUTES * 60) + val windowStart = now.minusSeconds(METRICS_WINDOW_MINUTES * SECONDS_PER_MINUTE) // Load the specific incident from IncidentEventLog by ID and org, then // parse the deduplication key to find the exact triggering host/alert. @@ -581,10 +591,10 @@ class SummaryService( private fun periodToMillis(period: String): Long { return when (period) { - "24h" -> 24L * 3600 * 1000 - "7d" -> 7L * 24 * 3600 * 1000 - "30d" -> 30L * 24 * 3600 * 1000 - else -> 24L * 3600 * 1000 + "24h" -> MILLIS_PER_DAY + "7d" -> MILLIS_7_DAYS + "30d" -> MILLIS_30_DAYS + else -> MILLIS_PER_DAY } } diff --git a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsCheckExecutor.kt b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsCheckExecutor.kt index 6dbf4edb5..07bd9bd4a 100644 --- a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsCheckExecutor.kt +++ b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsCheckExecutor.kt @@ -116,7 +116,7 @@ open class SyntheticsCheckExecutor { ) } - val timeoutMs = test.timeoutSeconds * 1000L + val timeoutMs = test.timeoutSeconds * MILLIS_PER_SECOND val client = buildClient(timeoutMs) val totalStart = System.nanoTime() @@ -544,7 +544,7 @@ open class SyntheticsCheckExecutor { return SyntheticCheckResult(status = "failed", durationMs = 0, errorMessage = "No steps configured") } - val timeoutMs = test.timeoutSeconds * 1000L + val timeoutMs = test.timeoutSeconds * MILLIS_PER_SECOND val client = buildClient(timeoutMs) val variables = mutableMapOf() val startTime = System.currentTimeMillis() @@ -847,5 +847,6 @@ open class SyntheticsCheckExecutor { companion object { private const val NS_PER_MS = 1_000_000L private const val REQUEST_FAILED = "Request failed" + private const val MILLIS_PER_SECOND = 1000L } } diff --git a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsRoutes.kt b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsRoutes.kt index 3f0328f12..13240e148 100644 --- a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsRoutes.kt @@ -54,6 +54,8 @@ private val json = Json { ignoreUnknownKeys = true } private const val DEFAULT_LIMIT = 100 private const val MAX_LIMIT = 500 +private const val HTTP_SUCCESS_MIN = 200 +private const val HTTP_SUCCESS_MAX = 299 private fun getOrgIdsForUser(userId: Int): List { return transaction { @@ -97,7 +99,7 @@ private fun parseJsonEachRow(body: String): List { private suspend fun executeChQuery(query: String): List? { return runCatching { val response = ClickHouseClient.execute(query) - if (response.status.value !in 200..299) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { logger.warn { "ClickHouse query failed: ${response.status}" } return null } diff --git a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsScheduler.kt b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsScheduler.kt index 830174814..f0ef1a0ed 100644 --- a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsScheduler.kt +++ b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsScheduler.kt @@ -36,6 +36,8 @@ import com.moneat.utils.suspendRunCatching private val logger = KotlinLogging.logger {} +private const val SCHEDULER_STARTUP_DELAY_MS = 2000L + class SyntheticsScheduler( private val service: SyntheticsService = SyntheticsService(), private val executor: SyntheticsCheckExecutor = SyntheticsCheckExecutor() @@ -58,7 +60,7 @@ class SyntheticsScheduler( TaskLock.tryWithLock("synthetics-scheduler", lockAtMostFor = 5.minutes, lockAtLeastFor = 1.seconds) { checkTests(this) } - delay(2000) + delay(SCHEDULER_STARTUP_DELAY_MS) } } diff --git a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsService.kt b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsService.kt index 834c72520..79ad226c6 100644 --- a/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsService.kt +++ b/backend/src/main/kotlin/com/moneat/synthetics/routes/SyntheticsService.kt @@ -61,6 +61,10 @@ class SyntheticsService( private const val PRO_TIER_LIMIT = 20 private const val TEAM_TIER_LIMIT = 50 private const val BUSINESS_TIER_LIMIT = Int.MAX_VALUE + private const val MILLIS_PER_SECOND = 1000L + private const val TIMEOUT_BUFFER_MS = 5000L + private const val HTTP_SUCCESS_MIN = 200 + private const val HTTP_SUCCESS_MAX = 299 } fun createTest( @@ -327,7 +331,7 @@ class SyntheticsService( for (attempt in 1..maxAttempts) { lastResult = suspendRunCatching { - withTimeout(test.timeoutSeconds * 1000L + 5000) { + withTimeout(test.timeoutSeconds * MILLIS_PER_SECOND + TIMEOUT_BUFFER_MS) { executor.executeTest(test) } }.getOrElse { e -> @@ -672,7 +676,7 @@ class SyntheticsService( ClickHouseClient.execute(query) }.getOrNull() ?: return null - if (response.status.value !in 200..299) { + if (response.status.value !in HTTP_SUCCESS_MIN..HTTP_SUCCESS_MAX) { logger.warn { "ClickHouse summary query failed: ${response.status}" } diff --git a/backend/src/main/kotlin/com/moneat/uptime/routes/UptimeRoutes.kt b/backend/src/main/kotlin/com/moneat/uptime/routes/UptimeRoutes.kt index a24d7f2de..f9b42855b 100644 --- a/backend/src/main/kotlin/com/moneat/uptime/routes/UptimeRoutes.kt +++ b/backend/src/main/kotlin/com/moneat/uptime/routes/UptimeRoutes.kt @@ -80,6 +80,8 @@ private fun getOrganizationIdsForUser(userId: Int): List { } } +private const val MIN_INTERVAL_SECONDS = 10 + /** * Uptime monitoring routes. */ @@ -458,78 +460,40 @@ fun Route.uptimeRoutes( * Validate monitor creation request. */ private fun validateMonitorRequest(request: CreateUptimeMonitorRequest): String? { - if (request.name.isBlank()) { - return "Monitor name is required" - } - - when (request.type.lowercase()) { - "http", "keyword", "json_query" -> { - if (request.url.isNullOrBlank()) { - return "URL is required for ${request.type} monitors" - } - } - - "tcp" -> { - if (request.hostname.isNullOrBlank()) { - return "Hostname is required for TCP monitors" - } - if (request.port == null) { - return "Port is required for TCP monitors" - } - } + if (request.name.isBlank()) return "Monitor name is required" - "ping" -> { - if (request.hostname.isNullOrBlank()) { - return "Hostname is required for ping monitors" - } - } + val typeError = validateMonitorType(request) + if (typeError != null) return typeError - "dns" -> { - if (request.hostname.isNullOrBlank()) { - return "Hostname is required for DNS monitors" - } - } + if (request.intervalSeconds < MIN_INTERVAL_SECONDS) return "Check interval must be at least 10 seconds" + if (request.timeoutSeconds < 1) return "Timeout must be at least 1 second" - "websocket" -> { - if (request.url.isNullOrBlank()) { - return "URL is required for WebSocket monitors" - } - } - - "ssl" -> { - if (request.hostname.isNullOrBlank()) { - return "Hostname is required for SSL monitors" - } - } - - "database" -> { - if (request.dbConnectionString.isNullOrBlank()) { - return "Database connection string is required" - } - } - - "docker" -> { - if (request.dockerContainerName.isNullOrBlank()) { - return "Container name is required for Docker monitors" - } - } - - "push" -> { - // No additional validation for push monitors - } - - else -> { - return "Unknown monitor type: ${request.type}" - } - } - - if (request.intervalSeconds < 10) { - return "Check interval must be at least 10 seconds" - } + return null +} - if (request.timeoutSeconds < 1) { - return "Timeout must be at least 1 second" +private fun validateMonitorType(request: CreateUptimeMonitorRequest): String? = + when (request.type.lowercase()) { + "http", "keyword", "json_query" -> + if (request.url.isNullOrBlank()) "URL is required for ${request.type} monitors" else null + "tcp" -> validateTcpMonitor(request) + "ping" -> + if (request.hostname.isNullOrBlank()) "Hostname is required for ping monitors" else null + "dns" -> + if (request.hostname.isNullOrBlank()) "Hostname is required for DNS monitors" else null + "websocket" -> + if (request.url.isNullOrBlank()) "URL is required for WebSocket monitors" else null + "ssl" -> + if (request.hostname.isNullOrBlank()) "Hostname is required for SSL monitors" else null + "database" -> + if (request.dbConnectionString.isNullOrBlank()) "Database connection string is required" else null + "docker" -> + if (request.dockerContainerName.isNullOrBlank()) "Container name is required for Docker monitors" else null + "push" -> null // No additional validation for push monitors + else -> "Unknown monitor type: ${request.type}" } +private fun validateTcpMonitor(request: CreateUptimeMonitorRequest): String? { + if (request.hostname.isNullOrBlank()) return "Hostname is required for TCP monitors" + if (request.port == null) return "Port is required for TCP monitors" return null } diff --git a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeCheckExecutor.kt b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeCheckExecutor.kt index dd9162e55..7cebae3e2 100644 --- a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeCheckExecutor.kt +++ b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeCheckExecutor.kt @@ -56,6 +56,12 @@ private val logger = KotlinLogging.logger {} */ class UptimeCheckExecutor { + companion object { + private const val MILLIS_PER_SECOND = 1000 + private const val HTTP_OK = 200 + private const val DEFAULT_HTTPS_PORT = 443 + } + private val httpClient = HttpClient(CIO) { install(HttpTimeout) { @@ -73,7 +79,7 @@ class UptimeCheckExecutor { */ suspend fun executeCheck(monitor: UptimeMonitorData): CheckResult { return try { - withTimeout(monitor.timeoutSeconds * 1000L) { + withTimeout(monitor.timeoutSeconds * MILLIS_PER_SECOND.toLong()) { when (monitor.type.lowercase()) { "http" -> checkHttp(monitor) "keyword" -> checkKeyword(monitor) @@ -172,7 +178,7 @@ class UptimeCheckExecutor { monitor.expectedStatusCodes ?.split(",") ?.mapNotNull { it.trim().toIntOrNull() } - ?: listOf(200) + ?: listOf(HTTP_OK) val isSuccess = statusCode in expectedCodes @@ -301,7 +307,7 @@ class UptimeCheckExecutor { return try { Socket().use { socket -> - socket.connect(InetSocketAddress(hostname, port), monitor.timeoutSeconds * 1000) + socket.connect(InetSocketAddress(hostname, port), monitor.timeoutSeconds * MILLIS_PER_SECOND) val responseTime = (System.currentTimeMillis() - startTime).toInt() CheckResult(1, responseTime, 0, "TCP connection successful") } @@ -321,7 +327,7 @@ class UptimeCheckExecutor { return try { val address = InetAddress.getByName(hostname) - val reachable = address.isReachable(monitor.timeoutSeconds * 1000) + val reachable = address.isReachable(monitor.timeoutSeconds * MILLIS_PER_SECOND) val responseTime = (System.currentTimeMillis() - startTime).toInt() if (reachable) { @@ -443,32 +449,8 @@ class UptimeCheckExecutor { // Docker API check return suspendRunCatching { - val startTime = System.currentTimeMillis() - - // For HTTP-based Docker API if (dockerHost.startsWith("http")) { - val dockerUrl = "$dockerHost/containers/$containerName/json" - try { - UrlValidator.validateExternalUrl(dockerUrl) - } catch (e: UrlValidator.SsrfException) { - return CheckResult(0, -1, 0, "Blocked: ${e.message}") - } - val response = httpClient.get(dockerUrl) - val responseTime = (System.currentTimeMillis() - startTime).toInt() - - if (response.status.value == 200) { - val body = response.bodyAsText() - val running = body.contains("\"Running\":true") - - CheckResult( - status = if (running) 1 else 0, - responseTimeMs = responseTime, - statusCode = response.status.value, - message = if (running) "Container is running" else "Container is not running" - ) - } else { - CheckResult(0, responseTime, response.status.value, "Container not found or Docker API error") - } + checkHttpDockerContainer(containerName, dockerHost) } else { // Unix socket not easily supported here CheckResult(0, -1, 0, "Docker Unix socket not supported. Use HTTP Docker API.") @@ -478,6 +460,31 @@ class UptimeCheckExecutor { } } + private suspend fun checkHttpDockerContainer(containerName: String, dockerHost: String): CheckResult { + val dockerUrl = "$dockerHost/containers/$containerName/json" + try { + UrlValidator.validateExternalUrl(dockerUrl) + } catch (e: UrlValidator.SsrfException) { + return CheckResult(0, -1, 0, "Blocked: ${e.message}") + } + val startTime = System.currentTimeMillis() + val response = httpClient.get(dockerUrl) + val responseTime = (System.currentTimeMillis() - startTime).toInt() + + return if (response.status.value == HTTP_OK) { + val body = response.bodyAsText() + val running = body.contains("\"Running\":true") + CheckResult( + status = if (running) 1 else 0, + responseTimeMs = responseTime, + statusCode = response.status.value, + message = if (running) "Container is running" else "Container is not running" + ) + } else { + CheckResult(0, responseTime, response.status.value, "Container not found or Docker API error") + } + } + /** * Database connection check */ @@ -518,14 +525,14 @@ class UptimeCheckExecutor { */ private suspend fun checkSsl(monitor: UptimeMonitorData): CheckResult { val hostname = monitor.hostname ?: return CheckResult(0, -1, 0, "No hostname configured") - val port = monitor.port ?: 443 + val port = monitor.port ?: DEFAULT_HTTPS_PORT val startTime = System.currentTimeMillis() return try { val url = java.net.URI("https://$hostname:$port").toURL() val conn = url.openConnection() as HttpsURLConnection - conn.connectTimeout = monitor.timeoutSeconds * 1000 + conn.connectTimeout = monitor.timeoutSeconds * MILLIS_PER_SECOND conn.connect() val responseTime = (System.currentTimeMillis() - startTime).toInt() diff --git a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeScheduler.kt b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeScheduler.kt index 1d085e228..0b2480663 100644 --- a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeScheduler.kt +++ b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeScheduler.kt @@ -58,6 +58,11 @@ class UptimeScheduler( private val emailService: EmailService = EmailService(), private val prefsService: AlertNotificationPreferencesService = AlertNotificationPreferencesService(), ) { + companion object { + private const val MILLIS_PER_SECOND = 1000L + private const val TIMEOUT_BUFFER_MS = 5000L + } + private var schedulerJob: Job? = null private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private val runningChecks = Collections.synchronizedSet(mutableSetOf()) @@ -81,7 +86,7 @@ class UptimeScheduler( } // Check every second - delay(1000) + delay(MILLIS_PER_SECOND) } } @@ -162,7 +167,7 @@ class UptimeScheduler( // Execute the check val result = try { - withTimeout(monitor.timeoutSeconds * 1000L + 5000) { // Add 5s buffer + withTimeout(monitor.timeoutSeconds * MILLIS_PER_SECOND + TIMEOUT_BUFFER_MS) { // Add 5s buffer checkExecutor.executeCheck(monitor) } } catch (e: TimeoutCancellationException) { @@ -232,11 +237,11 @@ class UptimeScheduler( var lastResult = initialResult for (retry in 1..monitor.retries) { - delay(monitor.retryIntervalSeconds * 1000L) + delay(monitor.retryIntervalSeconds * MILLIS_PER_SECOND) val retryResult = try { - withTimeout(monitor.timeoutSeconds * 1000L + 5000) { + withTimeout(monitor.timeoutSeconds * MILLIS_PER_SECOND + TIMEOUT_BUFFER_MS) { checkExecutor.executeCheck(monitor) } } catch (e: TimeoutCancellationException) { diff --git a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeService.kt b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeService.kt index b813ebe2f..61b157a46 100644 --- a/backend/src/main/kotlin/com/moneat/uptime/services/UptimeService.kt +++ b/backend/src/main/kotlin/com/moneat/uptime/services/UptimeService.kt @@ -56,6 +56,12 @@ class UptimeService( private const val TEAM_TIER_QUOTA = 25 private const val BUSINESS_TIER_QUOTA = Int.MAX_VALUE private const val DEFAULT_TIER_QUOTA = 3 + private const val MILLIS_PER_SECOND = 1000 + private const val PERCENT_MULTIPLIER = 100f + private const val HOURS_PER_DAY = 24 + private const val HOURS_PER_WEEK = 168 + private const val HOURS_PER_MONTH = 720 + private const val TOKEN_BYTES_SIZE = 32 } /** @@ -161,7 +167,7 @@ class UptimeService( monitorId: UUID, result: CheckResult ) { - val timestamp = Clock.System.now().toEpochMilliseconds() / 1000.0 + val timestamp = Clock.System.now().toEpochMilliseconds() / MILLIS_PER_SECOND.toDouble() val sql = """ @@ -169,7 +175,7 @@ class UptimeService( (monitor_id, timestamp, status, response_time_ms, status_code, message, ping_ms) VALUES ( '$monitorId', - fromUnixTimestamp64Milli(${(timestamp * 1000).toLong()}), + fromUnixTimestamp64Milli(${(timestamp * MILLIS_PER_SECOND).toLong()}), ${result.status}, ${result.responseTimeMs}, ${result.statusCode}, @@ -279,7 +285,7 @@ class UptimeService( val upCount = json["up_count"]?.jsonPrimitive?.long ?: 0L val totalCount = json["total_count"]?.jsonPrimitive?.long ?: 0L - if (totalCount == 0L) 0f else (upCount.toFloat() / totalCount.toFloat() * 100f) + if (totalCount == 0L) 0f else (upCount.toFloat() / totalCount.toFloat() * PERCENT_MULTIPLIER) }.getOrElse { e -> logger.error(e) { "Failed to calculate uptime for monitor $monitorId" } 0f @@ -364,9 +370,9 @@ class UptimeService( // Run async calls synchronously (in real impl, could be suspended) kotlinx.coroutines.runBlocking { Triple( - getUptimePercentage(monitor.id, 24), - getUptimePercentage(monitor.id, 168), - getUptimePercentage(monitor.id, 720) + getUptimePercentage(monitor.id, HOURS_PER_DAY), + getUptimePercentage(monitor.id, HOURS_PER_WEEK), + getUptimePercentage(monitor.id, HOURS_PER_MONTH) ) } } else { @@ -376,7 +382,7 @@ class UptimeService( val avgResponseTime = if (includeStats) { kotlinx.coroutines.runBlocking { - getAverageResponseTime(monitor.id, 24) + getAverageResponseTime(monitor.id, HOURS_PER_DAY) } } else { null @@ -438,7 +444,7 @@ class UptimeService( private fun generatePushToken(): String { val random = SecureRandom() - val bytes = ByteArray(32) + val bytes = ByteArray(TOKEN_BYTES_SIZE) random.nextBytes(bytes) return bytes.joinToString("") { "%02x".format(it) } } diff --git a/backend/src/main/kotlin/com/moneat/utils/ClickHouseQueryUtils.kt b/backend/src/main/kotlin/com/moneat/utils/ClickHouseQueryUtils.kt index f445325d9..f85c12378 100644 --- a/backend/src/main/kotlin/com/moneat/utils/ClickHouseQueryUtils.kt +++ b/backend/src/main/kotlin/com/moneat/utils/ClickHouseQueryUtils.kt @@ -24,6 +24,9 @@ package com.moneat.utils */ object ClickHouseQueryUtils { + private const val MILLIS_PER_SECOND = 1000.0 + private const val DATETIME64_PRECISION_MS = 3 + /** * Generate a project_id comparison clause that works with negative IDs. * @@ -78,7 +81,7 @@ object ClickHouseQueryUtils { ): String { val nowClause = if (demoEpochMs != null) { - "toDateTime64(${demoEpochMs / 1000.0}, 3)" + "toDateTime64(${demoEpochMs / MILLIS_PER_SECOND}, $DATETIME64_PRECISION_MS)" } else { "now()" } diff --git a/backend/src/main/kotlin/com/moneat/utils/DemoClock.kt b/backend/src/main/kotlin/com/moneat/utils/DemoClock.kt index 509a4371c..5999d9fcb 100644 --- a/backend/src/main/kotlin/com/moneat/utils/DemoClock.kt +++ b/backend/src/main/kotlin/com/moneat/utils/DemoClock.kt @@ -25,6 +25,9 @@ import kotlin.time.Instant * When demo mode is active, returns the configured demo epoch instead of the current time. */ object DemoClock { + + private const val MILLIS_PER_SECOND = 1000L + /** * Get current timestamp in milliseconds. * In demo mode, returns the demo epoch; otherwise returns actual current time. @@ -54,6 +57,6 @@ object DemoClock { * In demo mode, returns the demo epoch; otherwise returns actual current time. */ fun nowSeconds(isDemo: Boolean): Long { - return nowMs(isDemo) / 1000 + return nowMs(isDemo) / MILLIS_PER_SECOND } } diff --git a/backend/src/main/kotlin/com/moneat/utils/UrlValidator.kt b/backend/src/main/kotlin/com/moneat/utils/UrlValidator.kt index 88d5b7ee4..0327849c0 100644 --- a/backend/src/main/kotlin/com/moneat/utils/UrlValidator.kt +++ b/backend/src/main/kotlin/com/moneat/utils/UrlValidator.kt @@ -26,6 +26,18 @@ import java.net.UnknownHostException object UrlValidator { + private const val IPV6_ADDRESS_BYTE_LENGTH = 16 + private const val IPV4_MAPPED_ZERO_RANGE_END = 9 + private const val IPV4_MAPPED_FF_FIRST_IDX = 10 + private const val IPV4_MAPPED_FF_SECOND_IDX = 11 + private const val IPV4_IN_IPV6_START_IDX = 12 + private const val IPV4_IN_IPV6_END_IDX = 15 + private const val IPV4_BYTE_LENGTH = 4 + private const val IPV4_LAST_BYTE_IDX = 3 + private const val BYTE_MASK_UNSIGNED = 0xFF + private const val IPV6_ULA_FC_PREFIX = 0xFC + private const val IPV6_ULA_FD_PREFIX = 0xFD + class SsrfException(message: String) : IllegalArgumentException(message) /** @@ -105,12 +117,12 @@ object UrlValidator { if (addr is Inet6Address) { val bytes = addr.address // Check for ::ffff:x.x.x.x pattern (bytes 0-9 = 0, 10-11 = 0xff) - val isMapped = bytes.size == 16 && - bytes.slice(0..9).all { it == 0.toByte() } && - bytes[10] == 0xFF.toByte() && - bytes[11] == 0xFF.toByte() + val isMapped = bytes.size == IPV6_ADDRESS_BYTE_LENGTH && + bytes.slice(0..IPV4_MAPPED_ZERO_RANGE_END).all { it == 0.toByte() } && + bytes[IPV4_MAPPED_FF_FIRST_IDX] == 0xFF.toByte() && + bytes[IPV4_MAPPED_FF_SECOND_IDX] == 0xFF.toByte() if (isMapped) { - val ipv4Bytes = bytes.sliceArray(12..15) + val ipv4Bytes = bytes.sliceArray(IPV4_IN_IPV6_START_IDX..IPV4_IN_IPV6_END_IDX) return Inet4Address.getByAddress(ipv4Bytes) } } @@ -140,17 +152,17 @@ object UrlValidator { private fun isMetadataAddress(addr: InetAddress): Boolean { val bytes = addr.address - if (bytes.size != 4) return false + if (bytes.size != IPV4_BYTE_LENGTH) return false return bytes[0] == 169.toByte() && bytes[1] == 254.toByte() && bytes[2] == 169.toByte() && - bytes[3] == 254.toByte() + bytes[IPV4_LAST_BYTE_IDX] == 254.toByte() } private fun isIPv6UniqueLocal(addr: InetAddress): Boolean { if (addr !is Inet6Address) return false - val firstByte = addr.address[0].toInt() and 0xFF + val firstByte = addr.address[0].toInt() and BYTE_MASK_UNSIGNED // fc00::/7 means first byte is 0xFC or 0xFD - return firstByte == 0xFC || firstByte == 0xFD + return firstByte == IPV6_ULA_FC_PREFIX || firstByte == IPV6_ULA_FD_PREFIX } } diff --git a/backend/src/test/kotlin/com/moneat/dashboards/RedisHandlerParserTest.kt b/backend/src/test/kotlin/com/moneat/dashboards/RedisHandlerParserTest.kt index 7b4f2aac6..fcdb1dbef 100644 --- a/backend/src/test/kotlin/com/moneat/dashboards/RedisHandlerParserTest.kt +++ b/backend/src/test/kotlin/com/moneat/dashboards/RedisHandlerParserTest.kt @@ -152,4 +152,34 @@ class RedisHandlerParserTest { RedisHandler.parseCommandStats("# Commandstats\n").isEmpty() ) } + + @Test + fun `buildRedisUri brackets bare IPv6 literal`() { + val uri = RedisHandler.buildRedisUri("2001:db8::5", 6379, null) + assertEquals("redis://[2001:db8::5]:6379", uri) + } + + @Test + fun `buildRedisUri accepts already-bracketed IPv6`() { + val uri = RedisHandler.buildRedisUri("[2001:db8::5]", 6379, null) + assertEquals("redis://[2001:db8::5]:6379", uri) + } + + @Test + fun `buildRedisUri brackets bare IPv6 loopback`() { + val uri = RedisHandler.buildRedisUri("::1", 6380, "secret") + assertEquals("redis://:secret@[::1]:6380", uri) + } + + @Test + fun `buildRedisUri leaves plain hostname unchanged`() { + val uri = RedisHandler.buildRedisUri("myhost", 6379, null) + assertEquals("redis://myhost:6379", uri) + } + + @Test + fun `buildRedisUri uses port from host string when present`() { + val uri = RedisHandler.buildRedisUri("myhost:6380", 6379, null) + assertEquals("redis://myhost:6380", uri) + } } diff --git a/backend/src/test/kotlin/com/moneat/routes/DashboardRoutesTest.kt b/backend/src/test/kotlin/com/moneat/routes/DashboardRoutesTest.kt index 1f99321ae..ccac4de31 100644 --- a/backend/src/test/kotlin/com/moneat/routes/DashboardRoutesTest.kt +++ b/backend/src/test/kotlin/com/moneat/routes/DashboardRoutesTest.kt @@ -23,6 +23,7 @@ import com.moneat.dashboards.models.FolderResponse import com.moneat.dashboards.models.NotificationChannels import com.moneat.dashboards.models.SearchResponse import com.moneat.dashboards.models.TestConnectionResult +import com.moneat.dashboards.routes.DashboardTranslators import com.moneat.dashboards.routes.customDashboardRoutes import com.moneat.dashboards.services.CustomDashboardService import com.moneat.dashboards.services.CustomDataSourceExecutor @@ -216,8 +217,7 @@ class DashboardRoutesTest { dashboardService = mockDashboardService, queryEngine = mockQueryEngine, retentionPolicyService = mockRetentionService, - dataDogTranslator = mockDDTranslator, - grafanaTranslator = mockGrafanaTranslator, + translators = DashboardTranslators(mockDDTranslator, mockGrafanaTranslator), dataSourceService = mockDataSourceService, dataSourceExecutor = mockDataSourceExecutor, dashboardAlertService = mockAlertService, diff --git a/backend/src/test/kotlin/com/moneat/services/AdminServiceTest.kt b/backend/src/test/kotlin/com/moneat/services/AdminServiceTest.kt index 420fdbb8a..8b5d614e3 100644 --- a/backend/src/test/kotlin/com/moneat/services/AdminServiceTest.kt +++ b/backend/src/test/kotlin/com/moneat/services/AdminServiceTest.kt @@ -688,4 +688,43 @@ class AdminServiceTest { val result = service.getEmailStats(period = "7d") assertEquals(0L, result.totalSent) } + + @Test + fun `getEmailStats defaults to 30d for unknown period`() { + // Exercises the else branch of the period when-expression in getEmailStats + val result = service.getEmailStats(period = "unknown") + assertEquals(0L, result.totalSent) + } + + @Test + fun `getOrgUsage returns usage for 24h period`() { + // Exercises the "24h" branch of the period when-expression in getOrgUsage + val orgId = seedOrg() + val result = service.getOrgUsage(orgId, "24h") + assertTrue(result.isEmpty()) + } + + @Test + fun `getOrgUsage returns usage for 30d period`() { + // Exercises the "30d" branch of the period when-expression in getOrgUsage + val orgId = seedOrg() + val result = service.getOrgUsage(orgId, "30d") + assertTrue(result.isEmpty()) + } + + @Test + fun `getOrgUsage defaults to 7d for unknown period`() { + // Exercises the else branch of the period when-expression in getOrgUsage + val orgId = seedOrg() + val result = service.getOrgUsage(orgId, "unknown") + assertTrue(result.isEmpty()) + } + + @Test + fun `getUsageBreakdown returns empty for 30d period`() { + // Exercises the "30d" branch of the period when-expression in getUsageBreakdown + val result = service.getUsageBreakdown("30d") + assertTrue(result.daily.isEmpty()) + assertEquals(0L, result.totalBytes) + } } diff --git a/backend/src/test/kotlin/com/moneat/services/LogServicePureLogicTest.kt b/backend/src/test/kotlin/com/moneat/services/LogServicePureLogicTest.kt index 697eb1620..76f4b0df3 100644 --- a/backend/src/test/kotlin/com/moneat/services/LogServicePureLogicTest.kt +++ b/backend/src/test/kotlin/com/moneat/services/LogServicePureLogicTest.kt @@ -658,4 +658,59 @@ class LogServicePureLogicTest { val result = service.parseOtlpProtobuf(request.toByteArray()) assertEquals(1700000000000L, result.first().timestampMs) } + + // ──── estimateBillableBytes — private helper branch coverage ──── + + @Test + fun `estimateBillableBytes covers non-empty tags including blank-key filtering`() { + // Exercises sanitizeMap with a non-empty map; the blank key is filtered out + val entries = listOf( + LogIngestEntry( + message = "Hello", + body = "World", + tags = hashMapOf("env" to "production", " " to "blank-key-skipped"), + ), + ) + assertEquals(10L, service.estimateBillableBytes(entries)) + } + + @Test + fun `estimateBillableBytes covers trace warning and fatal log level normalization`() { + // Exercises normalizeLevel branches: trace, warn/warning, fatal/critical/panic + val entries = listOf( + LogIngestEntry(message = "a", body = "b", level = "trace"), + LogIngestEntry(message = "c", body = "d", level = "warning"), + LogIngestEntry(message = "e", body = "f", level = "fatal"), + ) + assertEquals(6L, service.estimateBillableBytes(entries)) + } + + @Test + fun `estimateBillableBytes covers agent and unknown source normalization`() { + // Exercises normalizeSource branches: agent_stdout, agent_stderr, else (unknown) + val entries = listOf( + LogIngestEntry(message = "x", body = "y", source = "agent_stdout"), + LogIngestEntry(message = "p", body = "q", source = "agent_stderr"), + LogIngestEntry(message = "r", body = "s", source = "custom_source"), + ) + assertEquals(6L, service.estimateBillableBytes(entries)) + } + + @Test + fun `estimateBillableBytes handles 10-digit Unix seconds timestamp`() { + // Exercises parseTimeToMillis digit-count branch: 10 digits → seconds → multiply by 1000 + val entries = listOf( + LogIngestEntry(message = "Hello", body = "World", timestamp = "1700000000"), + ) + assertEquals(10L, service.estimateBillableBytes(entries)) + } + + @Test + fun `estimateBillableBytes handles invalid unparseable timestamp`() { + // Exercises parseTimeToMillis getOrElse fallback when Instant.parse throws + val entries = listOf( + LogIngestEntry(message = "msg", body = "log", timestamp = "not-a-timestamp"), + ) + assertEquals(6L, service.estimateBillableBytes(entries)) + } } diff --git a/backend/src/test/kotlin/com/moneat/utils/RecoverOnExpectedFailuresTest.kt b/backend/src/test/kotlin/com/moneat/utils/RecoverOnExpectedFailuresTest.kt new file mode 100644 index 000000000..8b34d417a --- /dev/null +++ b/backend/src/test/kotlin/com/moneat/utils/RecoverOnExpectedFailuresTest.kt @@ -0,0 +1,110 @@ +// Moneat - observability platform +// Copyright (C) 2026 Moneat +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package com.moneat.utils + +import io.lettuce.core.RedisException +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.SerializationException +import mu.KotlinLogging +import java.io.IOException +import java.sql.SQLException +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class RecoverOnExpectedFailuresTest { + + private val logger = KotlinLogging.logger {} + + // ──── Happy path ──── + + @Test + fun `returns block result when no exception is thrown`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { "success" } + assertEquals("success", result) + } + + @Test + fun `works with non-string fallback type`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", -1) { 42 } + assertEquals(42, result) + } + + // ──── CancellationException propagates ──── + + @Test + fun `rethrows CancellationException`() { + assertFailsWith { + runBlocking { + recoverOnExpectedFailures(logger, "op", "fallback") { + throw CancellationException("cancelled") + } + } + } + } + + // ──── Expected failure types return fallback ──── + + @Test + fun `returns fallback on IOException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw IOException("io error") + } + assertEquals("fallback", result) + } + + @Test + fun `returns fallback on SQLException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw SQLException("sql error") + } + assertEquals("fallback", result) + } + + @Test + fun `returns fallback on SerializationException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw SerializationException("json error") + } + assertEquals("fallback", result) + } + + @Test + fun `returns fallback on IllegalStateException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw IllegalStateException("bad state") + } + assertEquals("fallback", result) + } + + @Test + fun `returns fallback on IllegalArgumentException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw IllegalArgumentException("bad arg") + } + assertEquals("fallback", result) + } + + @Test + fun `returns fallback on RedisException`() = runBlocking { + val result = recoverOnExpectedFailures(logger, "op", "fallback") { + throw RedisException("redis error") + } + assertEquals("fallback", result) + } +} diff --git a/ee/backend/detekt.yml b/ee/backend/detekt.yml index a273ceee6..51d0527b9 100644 --- a/ee/backend/detekt.yml +++ b/ee/backend/detekt.yml @@ -11,7 +11,14 @@ comments: # Mirror core's disabled rules for consistency. style: MagicNumber: - active: false + active: true + excludes: + - '**/test/**' + - '**/integrationTest/**' + - '**/DemoDataReseeder.kt' + ignorePropertyDeclaration: true + ignoreEnums: true + ignoreRanges: true WildcardImport: active: false ReturnCount: diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/llm/costs/Calculators.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/llm/costs/Calculators.kt index 5d91463b9..7bd9ae874 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/llm/costs/Calculators.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/llm/costs/Calculators.kt @@ -9,6 +9,8 @@ import com.moneat.enterprise.ai.llm.LlmCostCalculator import java.math.BigDecimal import java.math.RoundingMode +private const val COST_DECIMAL_SCALE = 6 + /** GPT-4o-mini: $0.15 / 1M input, $0.60 / 1M output */ class Gpt4oMiniCalculator : LlmCostCalculator { override fun model() = "gpt-4o-mini" @@ -17,9 +19,9 @@ class Gpt4oMiniCalculator : LlmCostCalculator { val inputCost = BigDecimal(inputTokens).multiply(BigDecimal("0.00000015")) val outputCost = BigDecimal(outputTokens).multiply(BigDecimal("0.0000006")) return LlmCost( - inputCost = inputCost.setScale(6, RoundingMode.HALF_UP), - outputCost = outputCost.setScale(6, RoundingMode.HALF_UP), - totalCost = inputCost.add(outputCost).setScale(6, RoundingMode.HALF_UP), + inputCost = inputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + outputCost = outputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + totalCost = inputCost.add(outputCost).setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), ) } } @@ -32,9 +34,9 @@ class Gpt4oCalculator : LlmCostCalculator { val inputCost = BigDecimal(inputTokens).multiply(BigDecimal("0.0000025")) val outputCost = BigDecimal(outputTokens).multiply(BigDecimal("0.00001")) return LlmCost( - inputCost = inputCost.setScale(6, RoundingMode.HALF_UP), - outputCost = outputCost.setScale(6, RoundingMode.HALF_UP), - totalCost = inputCost.add(outputCost).setScale(6, RoundingMode.HALF_UP), + inputCost = inputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + outputCost = outputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + totalCost = inputCost.add(outputCost).setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), ) } } @@ -47,9 +49,9 @@ class Claude4SonnetCalculator : LlmCostCalculator { val inputCost = BigDecimal(inputTokens).multiply(BigDecimal("0.000003")) val outputCost = BigDecimal(outputTokens).multiply(BigDecimal("0.000015")) return LlmCost( - inputCost = inputCost.setScale(6, RoundingMode.HALF_UP), - outputCost = outputCost.setScale(6, RoundingMode.HALF_UP), - totalCost = inputCost.add(outputCost).setScale(6, RoundingMode.HALF_UP), + inputCost = inputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + outputCost = outputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + totalCost = inputCost.add(outputCost).setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), ) } } @@ -62,9 +64,9 @@ class Claude4OpusCalculator : LlmCostCalculator { val inputCost = BigDecimal(inputTokens).multiply(BigDecimal("0.000015")) val outputCost = BigDecimal(outputTokens).multiply(BigDecimal("0.000075")) return LlmCost( - inputCost = inputCost.setScale(6, RoundingMode.HALF_UP), - outputCost = outputCost.setScale(6, RoundingMode.HALF_UP), - totalCost = inputCost.add(outputCost).setScale(6, RoundingMode.HALF_UP), + inputCost = inputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + outputCost = outputCost.setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), + totalCost = inputCost.add(outputCost).setScale(COST_DECIMAL_SCALE, RoundingMode.HALF_UP), ) } } diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/AiContextAggregator.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/AiContextAggregator.kt index 9f8226d02..73b3ce024 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/AiContextAggregator.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/AiContextAggregator.kt @@ -21,6 +21,7 @@ import java.time.LocalDate private val logger = KotlinLogging.logger {} private val json = Json { ignoreUnknownKeys = true } +private const val CHARS_PER_TOKEN = 4 /** Describes the time window to query observability data. */ sealed class AiTimeFilter { @@ -313,6 +314,6 @@ class AiContextAggregator { /** Estimate token count for the aggregated context (rough: ~4 chars per token). */ fun estimateTokens(context: AggregatedContext): Int { val contextJson = json.encodeToString(AggregatedContext.serializer(), context) - return contextJson.length / 4 + return contextJson.length / CHARS_PER_TOKEN } } diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/EnterpriseAiChatService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/EnterpriseAiChatService.kt index 30ad643c4..7be3fc776 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/EnterpriseAiChatService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/ai/services/EnterpriseAiChatService.kt @@ -28,6 +28,7 @@ import org.jetbrains.exposed.v1.jdbc.update import java.io.Writer import java.math.BigDecimal import java.time.LocalDate +import java.time.Month import kotlin.time.Clock private val logger = KotlinLogging.logger {} @@ -269,7 +270,7 @@ $contextStr""" // Conversation history (last 20 messages) val history = loadHistory(conversationId) - messages.addAll(history.takeLast(20)) + messages.addAll(history.takeLast(MAX_HISTORY_MESSAGES)) return messages } @@ -306,7 +307,7 @@ $contextStr""" AiConversations.insert { it[organization_id] = orgId it[user_id] = userId - it[title] = firstMessage.take(100) + it[title] = firstMessage.take(MAX_TITLE_LENGTH) it[created_at] = now it[updated_at] = now } get AiConversations.id @@ -365,20 +366,13 @@ $contextStr""" } private fun parseDateFromMessage(message: String): LocalDate? { - val months = mapOf( - "january" to 1, "february" to 2, "march" to 3, "april" to 4, - "may" to 5, "june" to 6, "july" to 7, "august" to 8, - "september" to 9, "october" to 10, "november" to 11, "december" to 12, - "jan" to 1, "feb" to 2, "mar" to 3, "apr" to 4, - "jun" to 6, "jul" to 7, "aug" to 8, "sep" to 9, "oct" to 10, "nov" to 11, "dec" to 12, - ) val pattern = Regex( """(january|february|march|april|may|june|july|august|september|october|november|december|""" + """jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+(\d{1,2})(?:st|nd|rd|th)?""", RegexOption.IGNORE_CASE, ) val match = pattern.find(message) ?: return null - val month = months[match.groupValues[1].lowercase()] ?: return null + val month = MONTH_NUMBERS[match.groupValues[1].lowercase()] ?: return null val day = match.groupValues[2].toIntOrNull() ?: return null val today = LocalDate.now() return try { @@ -388,14 +382,14 @@ $contextStr""" } private fun parseTimeRangeHours(timeRange: String?): Int { - if (timeRange.isNullOrBlank()) return 24 + if (timeRange.isNullOrBlank()) return HOURS_24H return when (timeRange.lowercase()) { "1h" -> 1 - "6h" -> 6 - "24h" -> 24 - "7d" -> 168 - "30d" -> 720 - else -> 24 + "6h" -> HOURS_6H + "24h" -> HOURS_24H + "7d" -> HOURS_7D + "30d" -> HOURS_30D + else -> HOURS_24H } } @@ -404,5 +398,21 @@ $contextStr""" writer.write("data: $data\n\n") writer.flush() } + + private const val MAX_HISTORY_MESSAGES = 20 + private const val MAX_TITLE_LENGTH = 100 + private const val HOURS_6H = 6 + private const val HOURS_24H = 24 + private const val HOURS_7D = 168 + private const val HOURS_30D = 720 + + private val MONTH_NUMBERS: Map = Month.values().flatMap { m -> + val name = m.name.lowercase() + if (name.length > 3) { + listOf(name to m.value, name.take(3) to m.value) + } else { + listOf(name to m.value) + } + }.toMap() } } diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/services/SummaryService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/services/SummaryService.kt index a667366c4..b4ca31f51 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/services/SummaryService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/services/SummaryService.kt @@ -28,6 +28,14 @@ import java.time.format.DateTimeFormatter private val logger = KotlinLogging.logger {} private val json = Json { ignoreUnknownKeys = true } +private const val MAX_ALERTS_DISPLAY = 10 +private const val HOURS_IN_7_DAYS = 168 +private const val HOURS_IN_30_DAYS = 720 +private const val DEFAULT_PERIOD_HOURS = 24 +private const val OVERNIGHT_END_HOUR = 8 +private const val OVERNIGHT_WINDOW_HOURS = 10L +private const val DAYS_IN_WEEK = 7L + class SummaryService( private val monitorService: MonitorService = MonitorService(HostRepositoryImpl(), HostAlertRepositoryImpl()), private val uptimeService: UptimeService = UptimeService(BillingQuotaService(), UptimeMonitorRepositoryImpl()), @@ -65,7 +73,7 @@ class SummaryService( val topAlerts = systems.flatMap { sys -> collectAlertsForSystem(sys) - }.sortedByDescending { it.lastTriggeredAt }.take(10) + }.sortedByDescending { it.lastTriggeredAt }.take(MAX_ALERTS_DISPLAY) val topErrorHosts = queryTopErrorHosts(organizationId, normalizedPeriod) @@ -83,9 +91,9 @@ class SummaryService( period: String, ): List { val intervalHours = when (period) { - "7d" -> 168 - "30d" -> 720 - else -> 24 + "7d" -> HOURS_IN_7_DAYS + "30d" -> HOURS_IN_30_DAYS + else -> DEFAULT_PERIOD_HOURS } return try { val query = """ @@ -130,9 +138,9 @@ class SummaryService( } val resolvedTimezone = resolvedZoneId.id val now = ZonedDateTime.now(resolvedZoneId) - val windowEnd = now.with(LocalTime.of(8, 0)) + val windowEnd = now.with(LocalTime.of(OVERNIGHT_END_HOUR, 0)) .let { if (it.isAfter(now)) it.minusDays(1) else it } - val windowStart = windowEnd.minusHours(10) + val windowStart = windowEnd.minusHours(OVERNIGHT_WINDOW_HOURS) val fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss") val startStr = windowStart.withZoneSameInstant(ZoneId.of("UTC")).format(fmt) @@ -188,7 +196,7 @@ class SummaryService( suspend fun getWeeklyReport(organizationId: Int): WeeklyReportResponse { val today = LocalDate.now() - val weekAgo = today.minusDays(7) + val weekAgo = today.minusDays(DAYS_IN_WEEK) val uptimeMonitors = uptimeService.listMonitors(organizationId).map { m -> UptimeMonitorSummary( @@ -273,7 +281,7 @@ class SummaryService( val relatedAlerts = systems.flatMap { sys -> collectAlertsForSystem(sys) - }.sortedByDescending { it.lastTriggeredAt }.take(10) + }.sortedByDescending { it.lastTriggeredAt }.take(MAX_ALERTS_DISPLAY) val recentLogErrors = queryRecentLogErrors(organizationId) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DashboardTemplateTool.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DashboardTemplateTool.kt index 31f8ab339..899999df7 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DashboardTemplateTool.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DashboardTemplateTool.kt @@ -34,6 +34,7 @@ private val dataDogTranslator = DataDogTranslator() private val grafanaTranslator = GrafanaTranslator() private val jsonParser = Json { ignoreUnknownKeys = true } private val logger = mu.KotlinLogging.logger {} +private const val DEFAULT_RETENTION_DAYS = 90 class GetDashboardTemplatesTool : McpTool { override val name = "get_dashboard_templates" @@ -163,7 +164,7 @@ class ExecuteDashboardQueryTool : McpTool { val queryConfigJson = args["query_config"] as? JsonObject ?: return errorResult("query_config must be an object") val retentionDays = args["retention_days"]?.jsonPrimitive - ?.content?.toIntOrNull() ?: 90 + ?.content?.toIntOrNull() ?: DEFAULT_RETENTION_DAYS val dsl = try { jsonParser.decodeFromJsonElement( diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DataSourceTool.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DataSourceTool.kt index f179daa27..9b35f453a 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DataSourceTool.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/DataSourceTool.kt @@ -18,6 +18,7 @@ import kotlinx.serialization.json.jsonPrimitive private val dataSourceService = CustomDataSourceService() private val dataSourceExecutor = CustomDataSourceExecutor() +private const val DEFAULT_QUERY_LIMIT = 100 class ListDataSourcesTool : McpTool { override val name = "list_datasources" @@ -141,7 +142,7 @@ class ExecuteDataSourceQueryTool : McpTool { ?: return errorResult("datasource_id is required") val query = args["query"]?.jsonPrimitive?.content ?: return errorResult("query is required") - val limit = args["limit"]?.jsonPrimitive?.intOrNull ?: 100 + val limit = args["limit"]?.jsonPrimitive?.intOrNull ?: DEFAULT_QUERY_LIMIT val orgId = context.organizationId.toLong() val source = dataSourceService.getDataSource(id, orgId) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MetricsTool.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MetricsTool.kt index ede7733e6..813b139a9 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MetricsTool.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MetricsTool.kt @@ -19,6 +19,7 @@ private val metricsMonitorService = MonitorService(HostRepositoryImpl(), HostAle private const val DEFAULT_METRIC_HOURS = 24 private const val MAX_METRIC_HOURS = 168 +private const val MILLIS_PER_HOUR = 3_600_000L class GetHostMetricsTool : McpTool { override val name = "get_host_metrics" @@ -48,7 +49,7 @@ class GetHostMetricsTool : McpTool { ?.coerceIn(1, MAX_METRIC_HOURS) ?: DEFAULT_METRIC_HOURS val now = System.currentTimeMillis() - val from = now - hrs * 3600 * 1000L + val from = now - hrs * MILLIS_PER_HOUR val metrics = metricsMonitorService.getHistoricalMetrics( hostId, from, diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MonitorTool.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MonitorTool.kt index d51ad2c19..0d276470d 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MonitorTool.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/MonitorTool.kt @@ -17,6 +17,7 @@ import kotlinx.serialization.json.jsonPrimitive private val monitorService = MonitorService(HostRepositoryImpl(), HostAlertRepositoryImpl()) private val agentApiKeyService = AgentApiKeyService() +private const val KEY_LOG_PREFIX_LENGTH = 4 class ListHostsTool : McpTool { override val name = "list_hosts" @@ -86,7 +87,7 @@ class CreateAgentKeyTool : McpTool { ) return textResult( "Agent API key created: ${response.name} " + - "(id=${response.id}, key=${response.key.take(4)}***)" + "(id=${response.id}, key=${response.key.take(KEY_LOG_PREFIX_LENGTH)}***)" ) } } diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/UptimeTool.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/UptimeTool.kt index f9b733006..9b033478d 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/UptimeTool.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/mcp/tools/UptimeTool.kt @@ -21,6 +21,7 @@ private val uptimeService = UptimeService(BillingQuotaService(), UptimeMonitorRe private const val DEFAULT_HEARTBEAT_HOURS = 24 private const val MAX_HEARTBEAT_HOURS = 168 private const val DEFAULT_INTERVAL = 60 +private const val MILLIS_PER_HOUR = 3_600_000L class ListUptimeMonitorsTool : McpTool { override val name = "list_uptime_monitors" @@ -68,7 +69,7 @@ class GetMonitorHeartbeatsTool : McpTool { System.currentTimeMillis() ) val from = kotlin.time.Instant.fromEpochMilliseconds( - System.currentTimeMillis() - hrs * 3600 * 1000L + System.currentTimeMillis() - hrs * MILLIS_PER_HOUR ) val heartbeats = uptimeService.getHeartbeats(uuid, from, to) return jsonResult(heartbeats) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/IncidentRoutes.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/IncidentRoutes.kt index 16ced09a7..637a989e0 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/IncidentRoutes.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/IncidentRoutes.kt @@ -10,6 +10,7 @@ import com.moneat.utils.ErrorResponse import com.moneat.utils.MessageResponse import io.ktor.http.HttpStatusCode import io.ktor.server.auth.authenticate +import io.ktor.server.plugins.BadRequestException import io.ktor.server.auth.jwt.JWTPrincipal import io.ktor.server.auth.principal import io.ktor.server.request.receive @@ -20,6 +21,10 @@ import io.ktor.server.routing.post import io.ktor.server.routing.route import kotlinx.serialization.Serializable +private const val DEFAULT_INCIDENT_LIMIT = 50 +private const val MIN_INCIDENT_LIMIT = 1 +private const val MAX_INCIDENT_LIMIT = 200 + @Serializable data class DeclareIncidentRequest( val title: String, @@ -59,7 +64,14 @@ fun Route.incidentRoutes(incidentServiceProvider: () -> IncidentManagementServic val status = call.request.queryParameters["status"] val priorityLevel = call.request.queryParameters["priority"] - val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: 50 + val rawLimit = call.request.queryParameters["limit"]?.toIntOrNull() + val limit = + when { + rawLimit == null -> DEFAULT_INCIDENT_LIMIT + rawLimit < MIN_INCIDENT_LIMIT -> + throw BadRequestException("limit must be >= $MIN_INCIDENT_LIMIT") + else -> rawLimit.coerceAtMost(MAX_INCIDENT_LIMIT) + } val offset = call.request.queryParameters["offset"]?.toIntOrNull() ?: 0 val incidentService = incidentServiceProvider() diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/NotificationPreferencesRoutes.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/NotificationPreferencesRoutes.kt index 9bc410216..ad580f98a 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/NotificationPreferencesRoutes.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/NotificationPreferencesRoutes.kt @@ -418,7 +418,9 @@ private fun maskEmail(email: String): String { return "$masked@${parts[1]}" } +private const val PHONE_VISIBLE_DIGITS = 4 + private fun maskPhone(phone: String): String { - if (phone.length <= 4) return phone - return "*".repeat(phone.length - 4) + phone.takeLast(4) + if (phone.length <= PHONE_VISIBLE_DIGITS) return phone + return "*".repeat(phone.length - PHONE_VISIBLE_DIGITS) + phone.takeLast(PHONE_VISIBLE_DIGITS) } diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/OnCallRoutes.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/OnCallRoutes.kt index 220e4fbf2..b0eed8920 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/OnCallRoutes.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/routes/OnCallRoutes.kt @@ -47,6 +47,8 @@ import kotlin.time.Clock import kotlin.time.Duration.Companion.days import kotlin.time.Instant +private const val WEEKLY_ROTATION_DAYS = 7L + @Serializable data class ScheduleTimelineEntry( val userId: Int, @@ -586,7 +588,7 @@ private fun buildScheduleTimeline( } val rotationType = scheduleRow[OnCallSchedules.rotationType] - val rotationDays = when (rotationType) { "DAILY" -> 1L; else -> 7L } + val rotationDays = when (rotationType) { "DAILY" -> 1L; else -> WEEKLY_ROTATION_DAYS } val zoneId = ZoneId.of(scheduleRow[OnCallSchedules.timezone]) val handoffLocalTime = scheduleRow[OnCallSchedules.handoffTime] diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/BusinessHoursService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/BusinessHoursService.kt index d1e0e0c67..c8ba5d99f 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/BusinessHoursService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/BusinessHoursService.kt @@ -19,6 +19,11 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.update import kotlin.time.Clock +private const val DAY_OF_WEEK_WEDNESDAY = 3 +private const val DAY_OF_WEEK_THURSDAY = 4 +private const val DAY_OF_WEEK_FRIDAY = 5 +private const val DAY_OF_WEEK_SATURDAY = 6 + class BusinessHoursService { fun getBusinessHours(organizationId: Int): BusinessHoursConfig? = transaction { @@ -70,10 +75,10 @@ class BusinessHoursService { kotlinx.datetime.DayOfWeek.SUNDAY -> 0 kotlinx.datetime.DayOfWeek.MONDAY -> 1 kotlinx.datetime.DayOfWeek.TUESDAY -> 2 - kotlinx.datetime.DayOfWeek.WEDNESDAY -> 3 - kotlinx.datetime.DayOfWeek.THURSDAY -> 4 - kotlinx.datetime.DayOfWeek.FRIDAY -> 5 - kotlinx.datetime.DayOfWeek.SATURDAY -> 6 + kotlinx.datetime.DayOfWeek.WEDNESDAY -> DAY_OF_WEEK_WEDNESDAY + kotlinx.datetime.DayOfWeek.THURSDAY -> DAY_OF_WEEK_THURSDAY + kotlinx.datetime.DayOfWeek.FRIDAY -> DAY_OF_WEEK_FRIDAY + kotlinx.datetime.DayOfWeek.SATURDAY -> DAY_OF_WEEK_SATURDAY } val currentTime = java.time.LocalTime.of(now.hour, now.minute, now.second) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallHandoffService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallHandoffService.kt index b6e44c833..1aa314d3c 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallHandoffService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallHandoffService.kt @@ -30,6 +30,7 @@ class OnCallHandoffService( companion object { private const val CHECK_INTERVAL_MS = 60_000L // 60 seconds private const val REDIS_KEY_PREFIX = "oncall:current:" + private const val UNINITIALIZED_USER_ID = -2 } fun start() { @@ -84,7 +85,7 @@ class OnCallHandoffService( val lastUserId = lastUserIdStr?.toIntOrNull() // If state changed - if (currentUserId != (lastUserId ?: -2)) { // -2 as uninitialized sentinel + if (currentUserId != (lastUserId ?: UNINITIALIZED_USER_ID)) { // uninitialized sentinel // Update cache redisClient.set(cacheKey, currentUserId.toString()) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallScheduleService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallScheduleService.kt index 4cea1695e..50d12eefe 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallScheduleService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/OnCallScheduleService.kt @@ -33,6 +33,8 @@ import java.time.temporal.ChronoUnit import kotlin.time.Clock import kotlin.time.Instant +private const val WEEKLY_ROTATION_DAYS = 7 + class OnCallScheduleService { fun getOnCallUsedSeats(organizationId: Int): Int = transaction { @@ -185,12 +187,12 @@ class OnCallScheduleService { when (rotationType) { "DAILY" -> 1 - "WEEKLY" -> 7 + "WEEKLY" -> WEEKLY_ROTATION_DAYS - "CUSTOM" -> 7 + "CUSTOM" -> WEEKLY_ROTATION_DAYS // default to weekly for custom - else -> 7 + else -> WEEKLY_ROTATION_DAYS } // Calculate days since epoch using schedule timezone and handoffTime diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/PushNotificationService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/PushNotificationService.kt index 05fbbecd2..52a13f5b7 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/PushNotificationService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/PushNotificationService.kt @@ -30,6 +30,8 @@ import org.jetbrains.exposed.v1.jdbc.update import org.slf4j.LoggerFactory import kotlin.time.Clock +private const val TOKEN_LOG_PREFIX_LENGTH = 40 + class PushNotificationService { private val logger = LoggerFactory.getLogger(PushNotificationService::class.java) @@ -130,7 +132,7 @@ class PushNotificationService { removeDeviceToken(tokens[index]) } } else { - val prefix = tokens[index].take(40) + val prefix = tokens[index].take(TOKEN_LOG_PREFIX_LENGTH) logger.info( "Push ticket ok for user $userId, ticketId=${ticket.id}, token prefix=$prefix", ) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/ShiftChangeNotifier.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/ShiftChangeNotifier.kt index 394c89858..7ac84c834 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/ShiftChangeNotifier.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/ShiftChangeNotifier.kt @@ -30,6 +30,8 @@ import java.time.temporal.ChronoUnit import kotlin.time.Clock import kotlin.time.Duration.Companion.minutes +private const val WEEKLY_ROTATION_DAYS = 7L + class ShiftChangeNotifier( private val onCallScheduleService: OnCallScheduleService, private val pushNotificationService: PushNotificationService, @@ -181,7 +183,7 @@ class ShiftChangeNotifier( val rotationType = scheduleRow[OnCallSchedules.rotationType] val rotationDays = when (rotationType) { "DAILY" -> 1L - else -> 7L // WEEKLY, CUSTOM + else -> WEEKLY_ROTATION_DAYS // WEEKLY, CUSTOM } val zoneId = ZoneId.of(scheduleRow[OnCallSchedules.timezone]) diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/SlackUserGroupSyncService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/SlackUserGroupSyncService.kt index 436dd975c..12453f956 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/SlackUserGroupSyncService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/oncall/services/SlackUserGroupSyncService.kt @@ -25,6 +25,8 @@ import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.slf4j.LoggerFactory import kotlin.time.Duration.Companion.minutes +private const val ONE_HOUR_SECONDS = 3600L + /** * Background service that syncs on-call schedules to Slack user groups. * Runs every 60 seconds and updates Slack user groups to contain: @@ -167,7 +169,7 @@ class SlackUserGroupSyncService( ) // Cache the new state redisClient.set(cacheKey, targetState) - redisClient.expire(cacheKey, 3600) // Expire in 1 hour as a safety net + redisClient.expire(cacheKey, ONE_HOUR_SECONDS) // Expire in 1 hour as a safety net } else { logger.error( "Failed to sync Slack usergroup ${schedule.usergroupHandle} for schedule ${schedule.scheduleName}", diff --git a/ee/backend/src/main/kotlin/com/moneat/enterprise/sso/services/SsoService.kt b/ee/backend/src/main/kotlin/com/moneat/enterprise/sso/services/SsoService.kt index fec58768f..04b404fc8 100644 --- a/ee/backend/src/main/kotlin/com/moneat/enterprise/sso/services/SsoService.kt +++ b/ee/backend/src/main/kotlin/com/moneat/enterprise/sso/services/SsoService.kt @@ -595,16 +595,23 @@ class SsoService { .withIssuer(jwtIssuer) .withClaim("userId", userId) .withClaim("email", email) - .withExpiresAt(Date(System.currentTimeMillis() + 3600000)) + .withExpiresAt(Date(System.currentTimeMillis() + SSO_JWT_EXPIRY_MS)) .sign(Algorithm.HMAC256(jwtSecret)) private companion object { private const val SSO_NONCE_PREFIX = "sso:nonce:" private const val SSO_NONCE_TTL_SECONDS = 600L // 10 minutes + private const val SSO_NONCE_TTL_MS = SSO_NONCE_TTL_SECONDS * 1000L + private const val SSO_JWT_EXPIRY_MS = 3_600_000L // 1 hour + private const val SSO_STATE_NONCE_BYTES = 16 + private const val SSO_STATE_PART_COUNT = 4 + private const val SSO_STATE_SIGNATURE_IDX = 3 + private const val AES_GCM_IV_LENGTH = 12 + private const val AES_GCM_TAG_BITS = 128 } private fun generateSecureState(orgId: Int): String { - val nonce = ByteArray(16) + val nonce = ByteArray(SSO_STATE_NONCE_BYTES) secureRandom.nextBytes(nonce) val nonceB64 = Base64.getUrlEncoder().withoutPadding().encodeToString(nonce) val timestamp = System.currentTimeMillis() @@ -641,14 +648,14 @@ class SsoService { try { val decoded = String(Base64.getUrlDecoder().decode(state)) val parts = decoded.split(":") - if (parts.size != 4) { + if (parts.size != SSO_STATE_PART_COUNT) { throw IllegalArgumentException("Invalid state format") } val orgId = parts[0].toInt() val nonceB64 = parts[1] val timestamp = parts[2].toLong() - val signature = parts[3] + val signature = parts[SSO_STATE_SIGNATURE_IDX] // Verify HMAC signature val payload = "${parts[0]}:${parts[1]}:${parts[2]}" @@ -666,7 +673,7 @@ class SsoService { } // Check expiry (10 minutes) - if (System.currentTimeMillis() - timestamp > SSO_NONCE_TTL_SECONDS * 1000) { + if (System.currentTimeMillis() - timestamp > SSO_NONCE_TTL_MS) { throw IllegalArgumentException("State expired") } @@ -794,11 +801,11 @@ class SsoService { private fun encryptSecret(plaintext: String): String { val cipher = Cipher.getInstance("AES/GCM/NoPadding") - val iv = ByteArray(12) + val iv = ByteArray(AES_GCM_IV_LENGTH) secureRandom.nextBytes(iv) val keySpec = SecretKeySpec(encryptionKey, "AES") - val gcmSpec = GCMParameterSpec(128, iv) + val gcmSpec = GCMParameterSpec(AES_GCM_TAG_BITS, iv) cipher.init(Cipher.ENCRYPT_MODE, keySpec, gcmSpec) val ciphertext = cipher.doFinal(plaintext.toByteArray()) @@ -811,12 +818,12 @@ class SsoService { try { val combined = Base64.getDecoder().decode(encrypted) - val iv = combined.copyOfRange(0, 12) - val ciphertext = combined.copyOfRange(12, combined.size) + val iv = combined.copyOfRange(0, AES_GCM_IV_LENGTH) + val ciphertext = combined.copyOfRange(AES_GCM_IV_LENGTH, combined.size) val cipher = Cipher.getInstance("AES/GCM/NoPadding") val keySpec = SecretKeySpec(encryptionKey, "AES") - val gcmSpec = GCMParameterSpec(128, iv) + val gcmSpec = GCMParameterSpec(AES_GCM_TAG_BITS, iv) cipher.init(Cipher.DECRYPT_MODE, keySpec, gcmSpec) return String(cipher.doFinal(ciphertext))