From db5343fba36721cbdbf4cfe0d3fd6a1ea11761ba Mon Sep 17 00:00:00 2001 From: Menci Date: Sat, 7 Dec 2024 02:32:48 +0800 Subject: [PATCH 1/7] Add actuator and micrometer --- build.gradle.kts | 4 ++++ src/main/resources/application.properties | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 172db05d..89da707f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -46,6 +46,10 @@ dependencies { testImplementation("org.springframework.security:spring-security-test") implementation("net.logstash.logback:logstash-logback-encoder:7.4") + // Metrics + implementation("org.springframework.boot:spring-boot-starter-actuator") + implementation("io.micrometer:micrometer-registry-prometheus") + // Database runtimeOnly("org.mariadb.jdbc:mariadb-java-client:3.3.3") runtimeOnly("org.xerial:sqlite-jdbc:3.45.2.0") diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index df763c56..ffc41b93 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -33,3 +33,8 @@ spring.jpa.hibernate.ddl-auto=none # https://github.com/termstandard/colors#truecolor-support-in-output-devices # If you want to read the logs using a script, read the json logs in `logs` folder instead. spring.output.ansi.enabled=always + +# Metrics +management.server.port=8081 +management.endpoint.prometheus.enabled=true +management.endpoints.web.exposure.include=prometheus From 340003c568445983dd510da618eda47b958c4012 Mon Sep 17 00:00:00 2001 From: Menci Date: Tue, 10 Dec 2024 23:30:30 +0800 Subject: [PATCH 2/7] update --- src/main/java/ext/Ext.kt | 46 +++++++++++++++++++ .../icu/samnyan/aqua/net/db/AquaNetUser.kt | 1 + .../samnyan/aqua/net/games/mai2/Mai2Import.kt | 2 +- .../icu/samnyan/aqua/sega/aimedb/AimeDB.kt | 11 ++++- .../icu/samnyan/aqua/sega/allnet/AllNet.kt | 26 ++++++++--- .../samnyan/aqua/sega/allnet/AllNetSecure.kt | 10 ++++ .../aqua/sega/allnet/KeychipSession.kt | 11 +++-- .../sega/chusan/ChusanServletController.kt | 46 +++++++++++++------ .../samnyan/aqua/sega/general/BaseHandler.kt | 1 + .../sega/maimai2/Maimai2ServletController.kt | 30 ++++++++++-- .../handler/UploadUserPlaylogHandler.kt | 24 ++++++++++ .../samnyan/aqua/sega/wacca/WaccaServer.kt | 26 ++++++++++- src/main/resources/application.properties | 3 ++ .../V1000_20__keychip_session_game_id.sql | 2 + 14 files changed, 208 insertions(+), 31 deletions(-) create mode 100644 src/main/resources/db/migration/mariadb/V1000_20__keychip_session_game_id.sql diff --git a/src/main/java/ext/Ext.kt b/src/main/java/ext/Ext.kt index 0d713f18..bde47a10 100644 --- a/src/main/java/ext/Ext.kt +++ b/src/main/java/ext/Ext.kt @@ -5,6 +5,9 @@ import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.plugins.contentnegotiation.* import io.ktor.serialization.kotlinx.json.* +import io.micrometer.core.instrument.* +import io.micrometer.core.instrument.Timer +import io.micrometer.core.instrument.Metrics as Micrometer import jakarta.persistence.Query import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -22,6 +25,7 @@ import java.time.LocalDate import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* +import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KCallable import kotlin.reflect.KClass import kotlin.reflect.KMutableProperty1 @@ -214,3 +218,45 @@ val Pair<*, S>.r get() = component2() // Database val Query.exec get() = resultList.map { (it as Array<*>).toList() } + +// Metrics +object Metrics { + @PublishedApi + internal inline fun expandLabels(labels: T): Array { + return T::class.memberProperties.flatMap { prop -> + listOf(prop.name, prop.get(labels)?.toString() ?: throw IllegalArgumentException("Missing value for label ${prop.name}")) + }.toTypedArray() + } + + @PublishedApi + internal data class MetricCacheKey(val metricName: String, val labels: Any) + + @PublishedApi + internal val counterCache = ConcurrentHashMap() + + inline fun counter(metricName: String): (T) -> Counter { + return { labels -> + counterCache.computeIfAbsent(MetricCacheKey(metricName, labels)) { + Counter + .builder(metricName) + .tags(*expandLabels(labels)) + .register(Micrometer.globalRegistry) + } + } + } + + @PublishedApi + internal val timerCache = ConcurrentHashMap() + + inline fun timer(metricName: String): (T) -> Timer { + return { labels -> + timerCache.computeIfAbsent(MetricCacheKey(metricName, labels)) { + Timer + .builder(metricName) + .publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99) + .tags(*expandLabels(labels)) + .register(Micrometer.globalRegistry) + } + } + } +} diff --git a/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt b/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt index 5c2f7a2b..619e920f 100644 --- a/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt +++ b/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt @@ -6,6 +6,7 @@ import icu.samnyan.aqua.net.components.JWT import icu.samnyan.aqua.sega.allnet.AllNetProps import icu.samnyan.aqua.sega.allnet.KeyChipRepo import icu.samnyan.aqua.sega.allnet.KeychipSession +import icu.samnyan.aqua.sega.allnet.TokenChecker import icu.samnyan.aqua.sega.general.dao.CardRepository import icu.samnyan.aqua.sega.general.model.Card import jakarta.persistence.* diff --git a/src/main/java/icu/samnyan/aqua/net/games/mai2/Mai2Import.kt b/src/main/java/icu/samnyan/aqua/net/games/mai2/Mai2Import.kt index 19fb3fde..284fb273 100644 --- a/src/main/java/icu/samnyan/aqua/net/games/mai2/Mai2Import.kt +++ b/src/main/java/icu/samnyan/aqua/net/games/mai2/Mai2Import.kt @@ -48,4 +48,4 @@ class Mai2Import( ) { override fun createEmpty() = Maimai2DataExport() override val userDataRepo = repos.userData -} \ No newline at end of file +} diff --git a/src/main/java/icu/samnyan/aqua/sega/aimedb/AimeDB.kt b/src/main/java/icu/samnyan/aqua/sega/aimedb/AimeDB.kt index 89ce6285..40b691c9 100644 --- a/src/main/java/icu/samnyan/aqua/sega/aimedb/AimeDB.kt +++ b/src/main/java/icu/samnyan/aqua/sega/aimedb/AimeDB.kt @@ -4,6 +4,7 @@ import ext.toHex import icu.samnyan.aqua.net.db.AquaUserServices import icu.samnyan.aqua.sega.general.model.Card import icu.samnyan.aqua.sega.general.service.CardService +import icu.samnyan.aqua.sega.allnet.AllNetProps import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufUtil import io.netty.buffer.Unpooled @@ -25,6 +26,7 @@ import kotlin.jvm.optionals.getOrNull class AimeDB( val cardService: CardService, val us: AquaUserServices, + val allNetProps: AllNetProps, ): ChannelInboundHandlerAdapter() { val logger: Logger = LoggerFactory.getLogger(AimeDB::class.java) @@ -65,7 +67,14 @@ class AimeDB( logger.info("AimeDB /${handler.name} : (game ${base.gameId}, keychip ${base.keychipId})") // Check keychip - if (!us.validKeychip(base.keychipId)) return logger.warn("> Rejected: Keychip not found") + if (!us.validKeychip(base.keychipId)) { + if (allNetProps.keychipPermissiveForTesting) { + logger.warn("> Accepted invalid keychip ${base.keychipId} in permissive mode") + } else { + logger.warn("> Rejected: Keychip not found") + return + } + } handler.fn(data)?.let { ctx.write(it) } } finally { diff --git a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt index 1a8c3155..936e9b7c 100644 --- a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt +++ b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt @@ -25,6 +25,7 @@ class AllNetProps { var port: Int? = null val keychipSesExpire: Long = 172800000 // milliseconds var checkKeychip: Boolean = false + var keychipPermissiveForTesting: Boolean = false var redirect: String = "web" var placeName: String = "" @@ -102,9 +103,11 @@ class AllNet( // game_id SDEZ, ver 1.35, serial A0000001234, ip, firm_ver 50000, boot_ver 0000, // encode UTF-8, format_ver 3, hops 1, token 2010451813 val reqMap = decodeAllNet(dataStream.readAllBytes()) - var serial = reqMap["serial"] ?: "" + val serial = reqMap["serial"] ?: "" logger.info("AllNet /PowerOn : $reqMap") + var session: String? = null + // Proper keychip authentication if (props.checkKeychip) { // If it's a user keychip, it should be in user database @@ -112,11 +115,20 @@ class AllNet( if (u != null) { // Create a new session for the user logger.info("> Keychip authenticated: ${u.auId} ${u.computedName}") - serial = keychipSessionService.new(u).token + session = keychipSessionService.new(u, reqMap["game_id"] ?: "").token } // Check if it's a whitelisted keychip - else if (serial.isEmpty() || !keychipRepo.existsByKeychipId(serial)) { + else if (!serial.isEmpty() && keychipRepo.existsByKeychipId(serial)) { + session = keychipSessionService.new(null, reqMap["game_id"] ?: "").token + } + + else if (props.keychipPermissiveForTesting) { + logger.warn("> Accepted invalid keychip $serial in permissive mode") + session = keychipSessionService.new(null, reqMap["game_id"] ?: "").token + } + + else { // This will cause an allnet auth bad on client side return "".also { logger.warn("> Rejected: Keychip not found") } } @@ -127,7 +139,7 @@ class AllNet( val formatVer = reqMap["format_ver"] ?: "" val resp = props.map.toMutableMap() + mapOf( - "uri" to switchUri(localAddr, localPort, gameId, ver, serial), + "uri" to switchUri(localAddr, localPort, gameId, ver, session), "host" to props.host.ifBlank { localAddr }, ) @@ -160,15 +172,15 @@ class AllNet( return resp.toUrl() + "\n" } - private fun switchUri(localAddr: Str, localPort: Str, gameId: Str, ver: Str, serial: Str): Str { + private fun switchUri(localAddr: Str, localPort: Str, gameId: Str, ver: Str, session: Str?): Str { val addr = props.host.ifBlank { localAddr } val port = props.port?.toString() ?: localPort // If keychip authentication is enabled, the game URLs will be set to /gs/{token}/{game}/... - val base = if (props.checkKeychip) "gs/$serial" else "g" + val base = if (session != null) "gs/$session" else "g" return "http://$addr:$port/$base/" + when (gameId) { - "SDBT" -> "chu2/$ver/$serial/" + "SDBT" -> "chu2/$ver/$session/" "SDHD" -> "chu3/$ver/" "SDGS" -> "chu3/$ver/" // International (c3exp) "SBZV" -> "diva/" diff --git a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNetSecure.kt b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNetSecure.kt index 23b63957..88a87005 100644 --- a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNetSecure.kt +++ b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNetSecure.kt @@ -6,6 +6,7 @@ import jakarta.servlet.http.HttpServletRequest import jakarta.servlet.http.HttpServletRequestWrapper import jakarta.servlet.http.HttpServletResponse import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.context.annotation.Configuration @@ -30,6 +31,13 @@ class TokenChecker( ) : HandlerInterceptor { val log = LoggerFactory.getLogger(TokenChecker::class.java) + companion object { + private val log = LoggerFactory.getLogger(TokenChecker::class.java) + + private val currentSession = ThreadLocal() + fun getCurrentSession() = currentSession.get() + } + /** * Handle request before it's processed. */ @@ -48,6 +56,8 @@ class TokenChecker( if (token.isNotBlank() && (keyChipRepo.existsByKeychipId(token) || session != null || (frontierProps.enabled && frontierProps.ftk == token))) { + currentSession.set(session) + // Forward the request val w = RewriteWrapper(req, token).apply { setAttribute("token", token) } req.getRequestDispatcher(w.requestURI).forward(w, resp) diff --git a/src/main/java/icu/samnyan/aqua/sega/allnet/KeychipSession.kt b/src/main/java/icu/samnyan/aqua/sega/allnet/KeychipSession.kt index 4681f26f..615f8f6c 100644 --- a/src/main/java/icu/samnyan/aqua/sega/allnet/KeychipSession.kt +++ b/src/main/java/icu/samnyan/aqua/sega/allnet/KeychipSession.kt @@ -21,7 +21,10 @@ import java.security.SecureRandom class KeychipSession( @ManyToOne @JoinColumn(name = "au_id") - var user: AquaNetUser = AquaNetUser(), + var user: AquaNetUser? = null, + + @Column(length = 4) + val gameId: String, @Id @Column(length = 32) @@ -69,8 +72,8 @@ class KeychipSessionService( /** * Create a new session. */ - fun new(user: AquaNetUser): KeychipSession { - val session = KeychipSession(user = user) + fun new(user: AquaNetUser?, gameId: String): KeychipSession { + val session = KeychipSession(user = user, gameId = gameId) return keychipSessionRepo.save(session) } @@ -81,4 +84,4 @@ class KeychipSessionService( lastUse = System.currentTimeMillis() keychipSessionRepo.save(this) } -} \ No newline at end of file +} diff --git a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt index fca58e6f..f4fc9211 100644 --- a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt @@ -4,9 +4,12 @@ import ext.* import icu.samnyan.aqua.sega.chunithm.handler.impl.GetGameIdlistHandler import icu.samnyan.aqua.sega.chusan.handler.* import icu.samnyan.aqua.sega.general.BaseHandler +import io.micrometer.core.instrument.Timer import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.* import kotlin.reflect.full.declaredMemberProperties +import kotlin.time.TimeSource +import kotlin.time.toJavaDuration /** @@ -70,6 +73,12 @@ class ChusanServletController( val getUserNetBattleRankingInfo: GetUserNetBattleRankingInfoHandler, val getGameMapAreaCondition: GetGameMapAreaConditionHandler ) { + data class ApiLabel(val api: String) + data class ApiErrorLabel(val api: String, val error: String) + val apiCountMetric = Metrics.counter("aquadx_chusan_api_count") + val apiErrorCountMetric = Metrics.counter("aquadx_chusan_api_error_count") + val apiLatencyMetric = Metrics.timer("aquadx_chusan_api_latency") + val logger = LoggerFactory.getLogger(ChusanServletController::class.java) val getUserCtoCPlay = BaseHandler { """{"userId":"${it["userId"]}","orderBy":"0","count":"0","userCtoCPlayList":[]}""" } @@ -110,23 +119,34 @@ class ChusanServletController( @API("/{endpoint}") fun handle(@PV endpoint: Str, @RB request: MutableMap, @PV version: Str): Any { var api = endpoint - request["version"] = version + val startTime = TimeSource.Monotonic.markNow() + var timer: Timer? = null + try { + request["version"] = version - // Export version - if (api.endsWith("C3Exp")) { - api = api.removeSuffix("C3Exp") - request["c3exp"] = true - } + // Export version + if (api.endsWith("C3Exp")) { + api = api.removeSuffix("C3Exp") + request["c3exp"] = true + } - logger.info("Chu3 $api : $request") + apiCountMetric(ApiLabel(api)).increment() + logger.info("Chu3 $api : $request") - if (api in noopEndpoint) { - return """{"returnCode":"1"}""" - } + if (api in noopEndpoint) { + return """{"returnCode":"1"}""" + } - return handlers[api]?.handle(request) ?: { - logger.warn("Chu3 $api not found") - """{"returnCode":"1","apiName":"$api"}""" + timer = apiLatencyMetric(ApiLabel(api)) + return handlers[api]?.handle(request) ?: { + logger.warn("Chu3 $api not found") + """{"returnCode":"1","apiName":"$api"}""" + } + } catch (e: Exception) { + apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() + throw e; + } finally { + timer?.record(startTime.elapsedNow().toJavaDuration()) } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt b/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt index d883a6fb..8e6f695e 100644 --- a/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt +++ b/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt @@ -1,6 +1,7 @@ package icu.samnyan.aqua.sega.general import com.fasterxml.jackson.core.JsonProcessingException +import icu.samnyan.aqua.sega.allnet.KeychipSession /** * @author samnyan (privateamusement@protonmail.com) diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt index 3afc7bc3..dc86ee00 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt @@ -2,15 +2,21 @@ package icu.samnyan.aqua.sega.maimai2 import ext.* import icu.samnyan.aqua.net.utils.ApiException +import icu.samnyan.aqua.sega.allnet.KeychipSession import icu.samnyan.aqua.sega.general.BaseHandler import icu.samnyan.aqua.sega.maimai2.handler.* import icu.samnyan.aqua.sega.maimai2.model.Mai2Repos import io.ktor.client.request.* +import io.micrometer.core.instrument.Timer +import jakarta.servlet.http.HttpServletRequest import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.* +import java.time.LocalDateTime import java.time.format.DateTimeFormatter import kotlin.reflect.full.declaredMemberProperties +import kotlin.time.TimeSource +import kotlin.time.toJavaDuration /** * @author samnyan (privateamusement@protonmail.com) @@ -40,6 +46,12 @@ class Maimai2ServletController( private val GAME_SETTING_TIME_FMT = DateTimeFormatter.ofPattern("HH:mm:00") } + data class ApiLabel(val api: String) + data class ApiErrorLabel(val api: String, val error: String) + val apiCountMetric = Metrics.counter("aquadx_maimai2_api_count") + val apiErrorCountMetric = Metrics.counter("aquadx_maimai2_api_error_count") + val apiLatencyMetric = Metrics.timer("aquadx_maimai2_api_latency") + val getUserExtend = UserReqHandler { _, userId -> mapOf( "userId" to userId, "userExtend" to (repos.userExtend.findSingleByUser_Card_ExtId(userId)() ?: (404 - "User not found")) @@ -339,7 +351,10 @@ class Maimai2ServletController( @API("/{api}") fun handle(@PathVariable api: String, @RequestBody request: Map): Any { + val startTime = TimeSource.Monotonic.markNow() + var timer: Timer? = null try { + apiCountMetric(ApiLabel(api)).increment() logger.info("Mai2 < $api : ${request.toJson()}") // TODO: Optimize logging if (api in noopEndpoint) { @@ -352,6 +367,7 @@ class Maimai2ServletController( return staticEndpoint[api]!! } + timer = apiLatencyMetric(ApiLabel(api)) return handlers[api]?.handle(request)?.let { if (it is String) it else it.toJson() }?.also { if (api !in setOf("GetUserItemApi", "GetGameEventApi")) logger.info("Mai2 > $api : $it") @@ -359,9 +375,17 @@ class Maimai2ServletController( logger.warn("Mai2 > $api not found") """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" } - } catch (e: ApiException) { - // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow - return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") + } catch (e: Exception) { + if (e is ApiException) { + apiErrorCountMetric(ApiErrorLabel(api, e.code.toString())).increment() + // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow + return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") + } else { + apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() + throw e; + } + } finally { + timer?.record(startTime.elapsedNow().toJavaDuration()) } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt index 498c8fdf..e2c351f5 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt @@ -1,9 +1,12 @@ package icu.samnyan.aqua.sega.maimai2.handler +import ext.Metrics import ext.millis +import icu.samnyan.aqua.sega.allnet.TokenChecker import icu.samnyan.aqua.sega.maimai2.model.Mai2UserDataRepo import icu.samnyan.aqua.sega.maimai2.model.Mai2UserPlaylogRepo import icu.samnyan.aqua.sega.general.BaseHandler +import icu.samnyan.aqua.sega.maimai2.Maimai2ServletController.ApiLabel import icu.samnyan.aqua.sega.maimai2.model.request.UploadUserPlaylog import icu.samnyan.aqua.sega.maimai2.model.userdata.Mai2UserPlaylog import icu.samnyan.aqua.sega.util.jackson.BasicMapper @@ -24,11 +27,23 @@ class UploadUserPlaylogHandler( companion object { @JvmStatic val playBacklog = mutableMapOf>() + + val VALID_GAME_IDS = setOf("SDEZ", "SDGA", "SDGB") } + data class GameIdVersionLabel(val gameId: String, val version: String) + val gameVersionCountMetric = Metrics.counter("aquadx_maimai2_game_version_count") + override fun handle(request: Map): String { val req = mapper.convert(request, UploadUserPlaylog::class.java) + val version = tryParseGameVersion(req.userPlaylog.version) + if (version != null) { + val session = TokenChecker.getCurrentSession() + val gameId = if (session?.gameId in VALID_GAME_IDS) session!!.gameId else "" + gameVersionCountMetric(GameIdVersionLabel(gameId, version)).increment() + } + // Save if the user is registered val u = userDataRepository.findByCardExtId(req.userId).getOrNull() if (u != null) playlogRepo.save(req.userPlaylog.apply { user = u }) @@ -52,4 +67,13 @@ class UploadUserPlaylogHandler( playBacklog.filter { (_, v) -> v.isEmpty() || v[0].time - now > 300_000 }.toList() .forEach { (k, _) -> playBacklog.remove(k) } } + + private fun tryParseGameVersion(version: Int): String? { + val major = version / 1000000 + val minor = version / 1000 % 1000 + if (major != 1) return null + if (minor !in 0..99) return null + // e.g. "1.30", minor should have two digits + return "$major.${minor.toString().padStart(2, '0')}" + } } diff --git a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt index 7f324ad8..9b3d61ce 100644 --- a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt +++ b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt @@ -13,6 +13,7 @@ import icu.samnyan.aqua.sega.wacca.WaccaOptionType.* import icu.samnyan.aqua.sega.wacca.model.BaseRequest import icu.samnyan.aqua.sega.wacca.model.db.* import io.ktor.client.utils.* +import io.micrometer.core.instrument.Timer import jakarta.servlet.http.HttpServletRequest import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.ResponseEntity @@ -20,6 +21,8 @@ import org.springframework.web.bind.annotation.RestController import java.util.* import kotlin.math.max import kotlin.math.min +import kotlin.time.TimeSource +import kotlin.time.toJavaDuration val empty = emptyList() @@ -32,6 +35,12 @@ class WaccaServer { @Autowired lateinit var rp: WaccaRepos @Autowired lateinit var wacca: Wacca + data class ApiLabel(val api: String) + data class ApiErrorLabel(val api: String, val error: String) + val apiCountMetric = Metrics.counter("aquadx_wacca_api_count") + val apiErrorCountMetric = Metrics.counter("aquadx_wacca_api_error_count") + val apiLatencyMetric = Metrics.timer("aquadx_wacca_api_latency") + val handlerMap = mutableMapOf) -> Any>() val cacheMap = mutableMapOf() @@ -79,14 +88,21 @@ class WaccaServer { /** Handle all requests */ @API("/api/**") fun handle(req: HttpServletRequest, @RB body: String): Any { + val startTime = TimeSource.Monotonic.markNow() + var timer: Timer? = null + var api = "" return try { val path = req.requestURI.removePrefix("/g/wacca").removePrefix("/WaccaServlet") .removePrefix("/api").removePrefix("/").lowercase() + + if (path !in cacheMap && path !in handlerMap) return resp("[]", 1, "Not Found") + api = path + apiCountMetric(ApiLabel(api)).increment() if (path in cacheMap) return resp(cacheMap[path]!!) - if (path !in handlerMap) return resp("[]", 1, "Not Found") log.info("Wacca < $path : $body") + timer = apiLatencyMetric(ApiLabel(api)) val br = JACKSON.parse(body) handlerMap[path]!!(br, br.params).let { when (it) { is String -> resp(it) @@ -94,10 +110,16 @@ class WaccaServer { else -> error("Invalid response type ${it.javaClass}") } }.also { log.info("Wacca > $path : ${it.body}") } } - catch (e: ApiException) { resp("[]", e.code, e.message ?: "") } + catch (e: ApiException) { + apiErrorCountMetric(ApiErrorLabel(api, e.code.toString())).increment() + resp("[]", e.code, e.message ?: "") + } catch (e: Exception) { + apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() log.error("Wacca > Error", e) resp("[]", 500, e.message ?: "") + } finally { + timer?.record(startTime.elapsedNow().toJavaDuration()) } } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ffc41b93..769d910a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -38,3 +38,6 @@ spring.output.ansi.enabled=always management.server.port=8081 management.endpoint.prometheus.enabled=true management.endpoints.web.exposure.include=prometheus +management.metrics.distribution.percentiles.http.server.requests=0.5, 0.75, 0.9, 0.95, 0.99 +management.metrics.distribution.percentiles.tasks.scheduled.execution=0.5, 0.75, 0.9, 0.95, 0.99 +management.metrics.distribution.percentiles.spring.data.repository.invocations=0.5, 0.75, 0.9, 0.95, 0.99 diff --git a/src/main/resources/db/migration/mariadb/V1000_20__keychip_session_game_id.sql b/src/main/resources/db/migration/mariadb/V1000_20__keychip_session_game_id.sql new file mode 100644 index 00000000..162c2a2a --- /dev/null +++ b/src/main/resources/db/migration/mariadb/V1000_20__keychip_session_game_id.sql @@ -0,0 +1,2 @@ +ALTER TABLE allnet_keychip_sessions + ADD game_id VARCHAR(4) NULL; From 70466d0c945e788a4e3956aadbd3c4d0400d9f00 Mon Sep 17 00:00:00 2001 From: Azalea <22280294+hykilpikonna@users.noreply.github.com> Date: Tue, 10 Dec 2024 19:13:40 -0500 Subject: [PATCH 3/7] [-] Remove unused import --- src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt b/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt index 619e920f..5c2f7a2b 100644 --- a/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt +++ b/src/main/java/icu/samnyan/aqua/net/db/AquaNetUser.kt @@ -6,7 +6,6 @@ import icu.samnyan.aqua.net.components.JWT import icu.samnyan.aqua.sega.allnet.AllNetProps import icu.samnyan.aqua.sega.allnet.KeyChipRepo import icu.samnyan.aqua.sega.allnet.KeychipSession -import icu.samnyan.aqua.sega.allnet.TokenChecker import icu.samnyan.aqua.sega.general.dao.CardRepository import icu.samnyan.aqua.sega.general.model.Card import jakarta.persistence.* From ebafb4c05e8677cb45c06e839e00e52f16f31400 Mon Sep 17 00:00:00 2001 From: Azalea <22280294+hykilpikonna@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:56:15 -0500 Subject: [PATCH 4/7] [O] Make code less verbose --- src/main/java/ext/Ext.kt | 47 +------------- .../icu/samnyan/aqua/net/db/Prometheus.kt | 47 ++++++++++++++ .../sega/chusan/ChusanServletController.kt | 43 +++++-------- .../samnyan/aqua/sega/general/BaseHandler.kt | 1 - .../sega/maimai2/Maimai2ServletController.kt | 62 +++++++------------ .../handler/UploadUserPlaylogHandler.kt | 11 ++-- .../samnyan/aqua/sega/wacca/WaccaServer.kt | 34 +++------- 7 files changed, 98 insertions(+), 147 deletions(-) create mode 100644 src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt diff --git a/src/main/java/ext/Ext.kt b/src/main/java/ext/Ext.kt index bde47a10..b7e77c65 100644 --- a/src/main/java/ext/Ext.kt +++ b/src/main/java/ext/Ext.kt @@ -5,9 +5,6 @@ import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.plugins.contentnegotiation.* import io.ktor.serialization.kotlinx.json.* -import io.micrometer.core.instrument.* -import io.micrometer.core.instrument.Timer -import io.micrometer.core.instrument.Metrics as Micrometer import jakarta.persistence.Query import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -25,7 +22,6 @@ import java.time.LocalDate import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.util.* -import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KCallable import kotlin.reflect.KClass import kotlin.reflect.KMutableProperty1 @@ -167,6 +163,7 @@ val Any?.truthy get() = when (this) { // Collections fun ls(vararg args: T) = args.toList() +inline fun arr(vararg args: T) = arrayOf(*args) operator fun Map.plus(map: Map) = (if (this is MutableMap) this else toMutableMap()).apply { putAll(map) } operator fun MutableMap.plusAssign(map: Map) { putAll(map) } @@ -218,45 +215,3 @@ val Pair<*, S>.r get() = component2() // Database val Query.exec get() = resultList.map { (it as Array<*>).toList() } - -// Metrics -object Metrics { - @PublishedApi - internal inline fun expandLabels(labels: T): Array { - return T::class.memberProperties.flatMap { prop -> - listOf(prop.name, prop.get(labels)?.toString() ?: throw IllegalArgumentException("Missing value for label ${prop.name}")) - }.toTypedArray() - } - - @PublishedApi - internal data class MetricCacheKey(val metricName: String, val labels: Any) - - @PublishedApi - internal val counterCache = ConcurrentHashMap() - - inline fun counter(metricName: String): (T) -> Counter { - return { labels -> - counterCache.computeIfAbsent(MetricCacheKey(metricName, labels)) { - Counter - .builder(metricName) - .tags(*expandLabels(labels)) - .register(Micrometer.globalRegistry) - } - } - } - - @PublishedApi - internal val timerCache = ConcurrentHashMap() - - inline fun timer(metricName: String): (T) -> Timer { - return { labels -> - timerCache.computeIfAbsent(MetricCacheKey(metricName, labels)) { - Timer - .builder(metricName) - .publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99) - .tags(*expandLabels(labels)) - .register(Micrometer.globalRegistry) - } - } - } -} diff --git a/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt b/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt new file mode 100644 index 00000000..260189e6 --- /dev/null +++ b/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt @@ -0,0 +1,47 @@ +package icu.samnyan.aqua.net.db + +import ext.arr +import icu.samnyan.aqua.net.utils.ApiException +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Timer +import java.util.concurrent.ConcurrentHashMap +import kotlin.time.TimeSource +import kotlin.time.toJavaDuration +import io.micrometer.core.instrument.Metrics as MMetrics + +operator fun Counter.unaryPlus() = increment() + +class APICounter(val api: String, val metrics: APIMetrics) { + operator fun unaryPlus() = +metrics["api_count", arr("api", api)] + + operator fun rem(err: Exception) = also { + val e = if (err is ApiException) err.code.toString() else err.javaClass.simpleName + +metrics["api_error_count", arr("api", api, "error", e)] + } + + operator fun invoke(fn: () -> T): T { + val start = TimeSource.Monotonic.markNow() + try { return fn().also { +this } } + catch (e: Exception) { throw e.also { this % e } } + finally { metrics.timer("api_latency", arr("api", api)).record(start.elapsedNow().toJavaDuration()) } + } +} + +class APIMetrics(val domain: String) { + val cache = ConcurrentHashMap, Any>() + val reg = MMetrics.globalRegistry + + operator fun get(name: String, vararg pairs: Pair) = + get(name, pairs.flatMap { listOf(it.first, it.second.toString()) }.toTypedArray()) + + operator fun get(name: String, tag: Array) = cache.computeIfAbsent(tag) { + Counter.builder("aquadx_${domain}_$name").tags(*tag).register(reg) + } as Counter + + fun timer(name: String, tag: Array) = cache.computeIfAbsent(tag) { + Timer.builder("aquadx_${domain}_$name").tags(*tag).publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99).register(reg) + } as Timer + + operator fun get(api: String) = APICounter(api, this) + operator fun set(api: String, value: APICounter) {} +} diff --git a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt index f4fc9211..d2a637ff 100644 --- a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt @@ -1,15 +1,13 @@ package icu.samnyan.aqua.sega.chusan import ext.* +import icu.samnyan.aqua.net.db.APIMetrics import icu.samnyan.aqua.sega.chunithm.handler.impl.GetGameIdlistHandler import icu.samnyan.aqua.sega.chusan.handler.* import icu.samnyan.aqua.sega.general.BaseHandler -import io.micrometer.core.instrument.Timer import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.* import kotlin.reflect.full.declaredMemberProperties -import kotlin.time.TimeSource -import kotlin.time.toJavaDuration /** @@ -73,11 +71,7 @@ class ChusanServletController( val getUserNetBattleRankingInfo: GetUserNetBattleRankingInfoHandler, val getGameMapAreaCondition: GetGameMapAreaConditionHandler ) { - data class ApiLabel(val api: String) - data class ApiErrorLabel(val api: String, val error: String) - val apiCountMetric = Metrics.counter("aquadx_chusan_api_count") - val apiErrorCountMetric = Metrics.counter("aquadx_chusan_api_error_count") - val apiLatencyMetric = Metrics.timer("aquadx_chusan_api_latency") + val metrics = APIMetrics("chusan") val logger = LoggerFactory.getLogger(ChusanServletController::class.java) @@ -119,34 +113,25 @@ class ChusanServletController( @API("/{endpoint}") fun handle(@PV endpoint: Str, @RB request: MutableMap, @PV version: Str): Any { var api = endpoint - val startTime = TimeSource.Monotonic.markNow() - var timer: Timer? = null - try { - request["version"] = version + request["version"] = version - // Export version - if (api.endsWith("C3Exp")) { - api = api.removeSuffix("C3Exp") - request["c3exp"] = true - } + // Export version + if (api.endsWith("C3Exp")) { + api = api.removeSuffix("C3Exp") + request["c3exp"] = true + } - apiCountMetric(ApiLabel(api)).increment() - logger.info("Chu3 $api : $request") + logger.info("Chu3 $api : $request") - if (api in noopEndpoint) { - return """{"returnCode":"1"}""" - } + if (api in noopEndpoint) { + return """{"returnCode":"1"}""" + } - timer = apiLatencyMetric(ApiLabel(api)) - return handlers[api]?.handle(request) ?: { + return metrics[api] { + handlers[api]?.handle(request) ?: { logger.warn("Chu3 $api not found") """{"returnCode":"1","apiName":"$api"}""" } - } catch (e: Exception) { - apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() - throw e; - } finally { - timer?.record(startTime.elapsedNow().toJavaDuration()) } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt b/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt index 8e6f695e..d883a6fb 100644 --- a/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt +++ b/src/main/java/icu/samnyan/aqua/sega/general/BaseHandler.kt @@ -1,7 +1,6 @@ package icu.samnyan.aqua.sega.general import com.fasterxml.jackson.core.JsonProcessingException -import icu.samnyan.aqua.sega.allnet.KeychipSession /** * @author samnyan (privateamusement@protonmail.com) diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt index dc86ee00..0f1638de 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt @@ -1,22 +1,17 @@ package icu.samnyan.aqua.sega.maimai2 import ext.* +import icu.samnyan.aqua.net.db.APIMetrics import icu.samnyan.aqua.net.utils.ApiException -import icu.samnyan.aqua.sega.allnet.KeychipSession import icu.samnyan.aqua.sega.general.BaseHandler import icu.samnyan.aqua.sega.maimai2.handler.* import icu.samnyan.aqua.sega.maimai2.model.Mai2Repos import io.ktor.client.request.* -import io.micrometer.core.instrument.Timer -import jakarta.servlet.http.HttpServletRequest import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.* -import java.time.LocalDateTime import java.time.format.DateTimeFormatter import kotlin.reflect.full.declaredMemberProperties -import kotlin.time.TimeSource -import kotlin.time.toJavaDuration /** * @author samnyan (privateamusement@protonmail.com) @@ -46,11 +41,7 @@ class Maimai2ServletController( private val GAME_SETTING_TIME_FMT = DateTimeFormatter.ofPattern("HH:mm:00") } - data class ApiLabel(val api: String) - data class ApiErrorLabel(val api: String, val error: String) - val apiCountMetric = Metrics.counter("aquadx_maimai2_api_count") - val apiErrorCountMetric = Metrics.counter("aquadx_maimai2_api_error_count") - val apiLatencyMetric = Metrics.timer("aquadx_maimai2_api_latency") + val metrics = APIMetrics("maimai2") val getUserExtend = UserReqHandler { _, userId -> mapOf( "userId" to userId, @@ -351,41 +342,32 @@ class Maimai2ServletController( @API("/{api}") fun handle(@PathVariable api: String, @RequestBody request: Map): Any { - val startTime = TimeSource.Monotonic.markNow() - var timer: Timer? = null - try { - apiCountMetric(ApiLabel(api)).increment() - logger.info("Mai2 < $api : ${request.toJson()}") // TODO: Optimize logging + logger.info("Mai2 < $api : ${request.toJson()}") // TODO: Optimize logging - if (api in noopEndpoint) { - logger.info("Mai2 > $api no-op") - return """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" - } + if (api in noopEndpoint) { + logger.info("Mai2 > $api no-op") + return """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" + } - if (api in staticEndpoint) { - logger.info("Mai2 > $api static") - return staticEndpoint[api]!! - } + if (api in staticEndpoint) { + logger.info("Mai2 > $api static") + return staticEndpoint[api]!! + } - timer = apiLatencyMetric(ApiLabel(api)) - return handlers[api]?.handle(request)?.let { if (it is String) it else it.toJson() }?.also { + if (!handlers.containsKey(api)) { + logger.warn("Mai2 > $api not found") + return """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" + } + + return try { metrics[api] { + handlers[api]!!.handle(request).let { if (it is String) it else it.toJson() }.also { if (api !in setOf("GetUserItemApi", "GetGameEventApi")) logger.info("Mai2 > $api : $it") - } ?: { - logger.warn("Mai2 > $api not found") - """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" } - } catch (e: Exception) { - if (e is ApiException) { - apiErrorCountMetric(ApiErrorLabel(api, e.code.toString())).increment() - // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow - return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") - } else { - apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() - throw e; - } - } finally { - timer?.record(startTime.elapsedNow().toJavaDuration()) + } } + catch (e: ApiException) { + // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow + return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt index e2c351f5..64de61f3 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt @@ -1,12 +1,12 @@ package icu.samnyan.aqua.sega.maimai2.handler -import ext.Metrics import ext.millis +import icu.samnyan.aqua.net.db.APIMetrics +import icu.samnyan.aqua.net.db.unaryPlus import icu.samnyan.aqua.sega.allnet.TokenChecker +import icu.samnyan.aqua.sega.general.BaseHandler import icu.samnyan.aqua.sega.maimai2.model.Mai2UserDataRepo import icu.samnyan.aqua.sega.maimai2.model.Mai2UserPlaylogRepo -import icu.samnyan.aqua.sega.general.BaseHandler -import icu.samnyan.aqua.sega.maimai2.Maimai2ServletController.ApiLabel import icu.samnyan.aqua.sega.maimai2.model.request.UploadUserPlaylog import icu.samnyan.aqua.sega.maimai2.model.userdata.Mai2UserPlaylog import icu.samnyan.aqua.sega.util.jackson.BasicMapper @@ -31,8 +31,7 @@ class UploadUserPlaylogHandler( val VALID_GAME_IDS = setOf("SDEZ", "SDGA", "SDGB") } - data class GameIdVersionLabel(val gameId: String, val version: String) - val gameVersionCountMetric = Metrics.counter("aquadx_maimai2_game_version_count") + val metrics = APIMetrics("maimai2") override fun handle(request: Map): String { val req = mapper.convert(request, UploadUserPlaylog::class.java) @@ -41,7 +40,7 @@ class UploadUserPlaylogHandler( if (version != null) { val session = TokenChecker.getCurrentSession() val gameId = if (session?.gameId in VALID_GAME_IDS) session!!.gameId else "" - gameVersionCountMetric(GameIdVersionLabel(gameId, version)).increment() + +metrics["game_version_count", "game_id" to gameId, "version" to version] } // Save if the user is registered diff --git a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt index 9b3d61ce..76265c03 100644 --- a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt +++ b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt @@ -1,6 +1,7 @@ package icu.samnyan.aqua.sega.wacca import ext.* +import icu.samnyan.aqua.net.db.APIMetrics import icu.samnyan.aqua.net.db.AquaGameOptions import icu.samnyan.aqua.net.games.wacca.Wacca import icu.samnyan.aqua.net.utils.ApiException @@ -13,7 +14,6 @@ import icu.samnyan.aqua.sega.wacca.WaccaOptionType.* import icu.samnyan.aqua.sega.wacca.model.BaseRequest import icu.samnyan.aqua.sega.wacca.model.db.* import io.ktor.client.utils.* -import io.micrometer.core.instrument.Timer import jakarta.servlet.http.HttpServletRequest import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.ResponseEntity @@ -21,8 +21,6 @@ import org.springframework.web.bind.annotation.RestController import java.util.* import kotlin.math.max import kotlin.math.min -import kotlin.time.TimeSource -import kotlin.time.toJavaDuration val empty = emptyList() @@ -35,11 +33,7 @@ class WaccaServer { @Autowired lateinit var rp: WaccaRepos @Autowired lateinit var wacca: Wacca - data class ApiLabel(val api: String) - data class ApiErrorLabel(val api: String, val error: String) - val apiCountMetric = Metrics.counter("aquadx_wacca_api_count") - val apiErrorCountMetric = Metrics.counter("aquadx_wacca_api_error_count") - val apiLatencyMetric = Metrics.timer("aquadx_wacca_api_latency") + val metrics = APIMetrics("wacca") val handlerMap = mutableMapOf) -> Any>() val cacheMap = mutableMapOf() @@ -88,38 +82,28 @@ class WaccaServer { /** Handle all requests */ @API("/api/**") fun handle(req: HttpServletRequest, @RB body: String): Any { - val startTime = TimeSource.Monotonic.markNow() - var timer: Timer? = null - var api = "" - return try { - val path = req.requestURI.removePrefix("/g/wacca").removePrefix("/WaccaServlet") - .removePrefix("/api").removePrefix("/").lowercase() + val path = req.requestURI.removePrefix("/g/wacca").removePrefix("/WaccaServlet") + .removePrefix("/api").removePrefix("/").lowercase() - if (path !in cacheMap && path !in handlerMap) return resp("[]", 1, "Not Found") - api = path - apiCountMetric(ApiLabel(api)).increment() - if (path in cacheMap) return resp(cacheMap[path]!!) + if (path in cacheMap) return resp(cacheMap[path]!!) + else if (path !in handlerMap) return resp("[]", 1, "Not Found") - log.info("Wacca < $path : $body") + log.info("Wacca < $path : $body") - timer = apiLatencyMetric(ApiLabel(api)) + return try { metrics[path] { val br = JACKSON.parse(body) handlerMap[path]!!(br, br.params).let { when (it) { is String -> resp(it) is List<*> -> resp(it.toJson()) else -> error("Invalid response type ${it.javaClass}") } }.also { log.info("Wacca > $path : ${it.body}") } - } + } } catch (e: ApiException) { - apiErrorCountMetric(ApiErrorLabel(api, e.code.toString())).increment() resp("[]", e.code, e.message ?: "") } catch (e: Exception) { - apiErrorCountMetric(ApiErrorLabel(api, e.javaClass.name)).increment() log.error("Wacca > Error", e) resp("[]", 500, e.message ?: "") - } finally { - timer?.record(startTime.elapsedNow().toJavaDuration()) } } } From ee88be613c5dc551dd427c161cd0618796479344 Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 11 Dec 2024 11:11:17 +0800 Subject: [PATCH 5/7] format --- .../java/icu/samnyan/aqua/net/db/Prometheus.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt b/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt index 260189e6..963c051f 100644 --- a/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt +++ b/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt @@ -23,7 +23,11 @@ class APICounter(val api: String, val metrics: APIMetrics) { val start = TimeSource.Monotonic.markNow() try { return fn().also { +this } } catch (e: Exception) { throw e.also { this % e } } - finally { metrics.timer("api_latency", arr("api", api)).record(start.elapsedNow().toJavaDuration()) } + finally { + metrics + .timer("api_latency", arr("api", api)) + .record(start.elapsedNow().toJavaDuration()) + } } } @@ -35,11 +39,18 @@ class APIMetrics(val domain: String) { get(name, pairs.flatMap { listOf(it.first, it.second.toString()) }.toTypedArray()) operator fun get(name: String, tag: Array) = cache.computeIfAbsent(tag) { - Counter.builder("aquadx_${domain}_$name").tags(*tag).register(reg) + Counter + .builder("aquadx_${domain}_$name") + .tags(*tag) + .register(reg) } as Counter fun timer(name: String, tag: Array) = cache.computeIfAbsent(tag) { - Timer.builder("aquadx_${domain}_$name").tags(*tag).publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99).register(reg) + Timer + .builder("aquadx_${domain}_$name") + .tags(*tag) + .publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99) + .register(reg) } as Timer operator fun get(api: String) = APICounter(api, this) From 5ed89754b312f21e5cf6a0e03a8f77b374c401c2 Mon Sep 17 00:00:00 2001 From: Menci Date: Thu, 12 Dec 2024 02:28:19 +0800 Subject: [PATCH 6/7] refactor --- .../icu/samnyan/aqua/net/db/Prometheus.kt | 58 ------------------- .../samnyan/aqua/net/utils/ErrorResponse.kt | 2 + .../sega/chusan/ChusanServletController.kt | 29 +++++++--- .../sega/maimai2/Maimai2ServletController.kt | 41 ++++++++----- .../handler/UploadUserPlaylogHandler.kt | 10 ++-- .../samnyan/aqua/sega/wacca/WaccaServer.kt | 49 ++++++++++------ .../java/icu/samnyan/aqua/spring/Metrics.kt | 45 ++++++++++++++ 7 files changed, 129 insertions(+), 105 deletions(-) delete mode 100644 src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt create mode 100644 src/main/java/icu/samnyan/aqua/spring/Metrics.kt diff --git a/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt b/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt deleted file mode 100644 index 963c051f..00000000 --- a/src/main/java/icu/samnyan/aqua/net/db/Prometheus.kt +++ /dev/null @@ -1,58 +0,0 @@ -package icu.samnyan.aqua.net.db - -import ext.arr -import icu.samnyan.aqua.net.utils.ApiException -import io.micrometer.core.instrument.Counter -import io.micrometer.core.instrument.Timer -import java.util.concurrent.ConcurrentHashMap -import kotlin.time.TimeSource -import kotlin.time.toJavaDuration -import io.micrometer.core.instrument.Metrics as MMetrics - -operator fun Counter.unaryPlus() = increment() - -class APICounter(val api: String, val metrics: APIMetrics) { - operator fun unaryPlus() = +metrics["api_count", arr("api", api)] - - operator fun rem(err: Exception) = also { - val e = if (err is ApiException) err.code.toString() else err.javaClass.simpleName - +metrics["api_error_count", arr("api", api, "error", e)] - } - - operator fun invoke(fn: () -> T): T { - val start = TimeSource.Monotonic.markNow() - try { return fn().also { +this } } - catch (e: Exception) { throw e.also { this % e } } - finally { - metrics - .timer("api_latency", arr("api", api)) - .record(start.elapsedNow().toJavaDuration()) - } - } -} - -class APIMetrics(val domain: String) { - val cache = ConcurrentHashMap, Any>() - val reg = MMetrics.globalRegistry - - operator fun get(name: String, vararg pairs: Pair) = - get(name, pairs.flatMap { listOf(it.first, it.second.toString()) }.toTypedArray()) - - operator fun get(name: String, tag: Array) = cache.computeIfAbsent(tag) { - Counter - .builder("aquadx_${domain}_$name") - .tags(*tag) - .register(reg) - } as Counter - - fun timer(name: String, tag: Array) = cache.computeIfAbsent(tag) { - Timer - .builder("aquadx_${domain}_$name") - .tags(*tag) - .publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99) - .register(reg) - } as Timer - - operator fun get(api: String) = APICounter(api, this) - operator fun set(api: String, value: APICounter) {} -} diff --git a/src/main/java/icu/samnyan/aqua/net/utils/ErrorResponse.kt b/src/main/java/icu/samnyan/aqua/net/utils/ErrorResponse.kt index f60f28c2..7c87b979 100644 --- a/src/main/java/icu/samnyan/aqua/net/utils/ErrorResponse.kt +++ b/src/main/java/icu/samnyan/aqua/net/utils/ErrorResponse.kt @@ -17,6 +17,8 @@ class ApiException(val code: Int, message: Str) : RuntimeException(message) { fun resp() = ResponseEntity.status(code).body(message.toString()) } +fun Exception.simpleDescribe(): String = if (this is ApiException) "E${code}" else javaClass.simpleName + @ControllerAdvice(basePackages = ["icu.samnyan"]) class GlobalExceptionHandler { @ExceptionHandler(ApiException::class) diff --git a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt index d2a637ff..bbb92d3d 100644 --- a/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/chusan/ChusanServletController.kt @@ -1,15 +1,15 @@ package icu.samnyan.aqua.sega.chusan import ext.* -import icu.samnyan.aqua.net.db.APIMetrics +import icu.samnyan.aqua.net.utils.simpleDescribe import icu.samnyan.aqua.sega.chunithm.handler.impl.GetGameIdlistHandler import icu.samnyan.aqua.sega.chusan.handler.* import icu.samnyan.aqua.sega.general.BaseHandler +import icu.samnyan.aqua.spring.Metrics import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.* import kotlin.reflect.full.declaredMemberProperties - /** * @author samnyan (privateamusement@protonmail.com) */ @@ -71,8 +71,6 @@ class ChusanServletController( val getUserNetBattleRankingInfo: GetUserNetBattleRankingInfoHandler, val getGameMapAreaCondition: GetGameMapAreaConditionHandler ) { - val metrics = APIMetrics("chusan") - val logger = LoggerFactory.getLogger(ChusanServletController::class.java) val getUserCtoCPlay = BaseHandler { """{"userId":"${it["userId"]}","orderBy":"0","count":"0","userCtoCPlayList":[]}""" } @@ -122,16 +120,31 @@ class ChusanServletController( } logger.info("Chu3 $api : $request") + if (api !in noopEndpoint && !handlers.containsKey(api)) { + logger.warn("Chu3 $api not found") + return """{"returnCode":"1","apiName":"$api"}""" + } + + // Only record the counter metrics if the API is known. + Metrics.counter("aquadx_chusan_api_call", "api" to api).increment() if (api in noopEndpoint) { return """{"returnCode":"1"}""" } - return metrics[api] { - handlers[api]?.handle(request) ?: { - logger.warn("Chu3 $api not found") - """{"returnCode":"1","apiName":"$api"}""" + return try { + Metrics.timer("aquadx_chusan_api_latency", "api" to api).recordCallable { + handlers[api]?.handle(request) ?: { + logger.warn("Chu3 $api not found") + """{"returnCode":"1","apiName":"$api"}""" + } } + } catch (e: Exception) { + Metrics.counter( + "aquadx_chusan_api_error", + "api" to api, "error" to e.simpleDescribe() + ).increment() + throw e } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt index 0f1638de..030d1c69 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/Maimai2ServletController.kt @@ -1,11 +1,12 @@ package icu.samnyan.aqua.sega.maimai2 import ext.* -import icu.samnyan.aqua.net.db.APIMetrics import icu.samnyan.aqua.net.utils.ApiException +import icu.samnyan.aqua.net.utils.simpleDescribe import icu.samnyan.aqua.sega.general.BaseHandler import icu.samnyan.aqua.sega.maimai2.handler.* import icu.samnyan.aqua.sega.maimai2.model.Mai2Repos +import icu.samnyan.aqua.spring.Metrics import io.ktor.client.request.* import org.slf4j.LoggerFactory import org.springframework.http.ResponseEntity @@ -41,8 +42,6 @@ class Maimai2ServletController( private val GAME_SETTING_TIME_FMT = DateTimeFormatter.ofPattern("HH:mm:00") } - val metrics = APIMetrics("maimai2") - val getUserExtend = UserReqHandler { _, userId -> mapOf( "userId" to userId, "userExtend" to (repos.userExtend.findSingleByUser_Card_ExtId(userId)() ?: (404 - "User not found")) @@ -343,6 +342,13 @@ class Maimai2ServletController( @API("/{api}") fun handle(@PathVariable api: String, @RequestBody request: Map): Any { logger.info("Mai2 < $api : ${request.toJson()}") // TODO: Optimize logging + if (api !in noopEndpoint && api !in staticEndpoint && !handlers.containsKey(api)) { + logger.warn("Mai2 > $api not found") + return """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" + } + + // Only record the counter metrics if the API is known. + Metrics.counter("aquadx_maimai2_api_call", "api" to api).increment() if (api in noopEndpoint) { logger.info("Mai2 > $api no-op") @@ -354,20 +360,23 @@ class Maimai2ServletController( return staticEndpoint[api]!! } - if (!handlers.containsKey(api)) { - logger.warn("Mai2 > $api not found") - return """{"returnCode":1,"apiName":"com.sega.maimai2servlet.api.$api"}""" - } - - return try { metrics[api] { - handlers[api]!!.handle(request).let { if (it is String) it else it.toJson() }.also { - if (api !in setOf("GetUserItemApi", "GetGameEventApi")) - logger.info("Mai2 > $api : $it") + return try { + Metrics.timer("aquadx_maimai2_api_latency", "api" to api).recordCallable { + handlers[api]!!.handle(request).let { if (it is String) it else it.toJson() }.also { + if (api !in setOf("GetUserItemApi", "GetGameEventApi")) + logger.info("Mai2 > $api : $it") + } } - } } - catch (e: ApiException) { - // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow - return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") + } catch (e: Exception) { + Metrics.counter( + "aquadx_maimai2_api_error", + "api" to api, "error" to e.simpleDescribe() + ).increment() + + if (e is ApiException) { + // It's a bad practice to return 200 ok on error, but this is what maimai does so we have to follow + return ResponseEntity.ok().body("""{"returnCode":0,"apiName":"com.sega.maimai2servlet.api.$api","message":"${e.message?.replace("\"", "\\\"")} - ${e.code}"}""") + } else throw e } } } diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt index 64de61f3..6f9c7a36 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/handler/UploadUserPlaylogHandler.kt @@ -1,8 +1,6 @@ package icu.samnyan.aqua.sega.maimai2.handler import ext.millis -import icu.samnyan.aqua.net.db.APIMetrics -import icu.samnyan.aqua.net.db.unaryPlus import icu.samnyan.aqua.sega.allnet.TokenChecker import icu.samnyan.aqua.sega.general.BaseHandler import icu.samnyan.aqua.sega.maimai2.model.Mai2UserDataRepo @@ -10,6 +8,7 @@ import icu.samnyan.aqua.sega.maimai2.model.Mai2UserPlaylogRepo import icu.samnyan.aqua.sega.maimai2.model.request.UploadUserPlaylog import icu.samnyan.aqua.sega.maimai2.model.userdata.Mai2UserPlaylog import icu.samnyan.aqua.sega.util.jackson.BasicMapper +import icu.samnyan.aqua.spring.Metrics import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import kotlin.jvm.optionals.getOrNull @@ -31,8 +30,6 @@ class UploadUserPlaylogHandler( val VALID_GAME_IDS = setOf("SDEZ", "SDGA", "SDGB") } - val metrics = APIMetrics("maimai2") - override fun handle(request: Map): String { val req = mapper.convert(request, UploadUserPlaylog::class.java) @@ -40,7 +37,10 @@ class UploadUserPlaylogHandler( if (version != null) { val session = TokenChecker.getCurrentSession() val gameId = if (session?.gameId in VALID_GAME_IDS) session!!.gameId else "" - +metrics["game_version_count", "game_id" to gameId, "version" to version] + Metrics.counter( + "aquadx_maimai2_playlog_game_version", + "game_id" to gameId, "version" to version + ).increment() } // Save if the user is registered diff --git a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt index 76265c03..68d6b39c 100644 --- a/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt +++ b/src/main/java/icu/samnyan/aqua/sega/wacca/WaccaServer.kt @@ -1,10 +1,10 @@ package icu.samnyan.aqua.sega.wacca import ext.* -import icu.samnyan.aqua.net.db.APIMetrics import icu.samnyan.aqua.net.db.AquaGameOptions import icu.samnyan.aqua.net.games.wacca.Wacca import icu.samnyan.aqua.net.utils.ApiException +import icu.samnyan.aqua.net.utils.simpleDescribe import icu.samnyan.aqua.sega.general.dao.CardRepository import icu.samnyan.aqua.sega.wacca.WaccaItemType.* import icu.samnyan.aqua.sega.wacca.WaccaItemType.NOTE_COLOR @@ -13,6 +13,7 @@ import icu.samnyan.aqua.sega.wacca.WaccaItemType.TOUCH_EFFECT import icu.samnyan.aqua.sega.wacca.WaccaOptionType.* import icu.samnyan.aqua.sega.wacca.model.BaseRequest import icu.samnyan.aqua.sega.wacca.model.db.* +import icu.samnyan.aqua.spring.Metrics import io.ktor.client.utils.* import jakarta.servlet.http.HttpServletRequest import org.springframework.beans.factory.annotation.Autowired @@ -33,8 +34,6 @@ class WaccaServer { @Autowired lateinit var rp: WaccaRepos @Autowired lateinit var wacca: Wacca - val metrics = APIMetrics("wacca") - val handlerMap = mutableMapOf) -> Any>() val cacheMap = mutableMapOf() @@ -82,28 +81,42 @@ class WaccaServer { /** Handle all requests */ @API("/api/**") fun handle(req: HttpServletRequest, @RB body: String): Any { + // Normalize path val path = req.requestURI.removePrefix("/g/wacca").removePrefix("/WaccaServlet") .removePrefix("/api").removePrefix("/").lowercase() + if (path !in cacheMap && path !in handlerMap) { + return resp("[]", 1, "Not Found") + } + + // Only record the counter metrics if the API is known. + Metrics.counter("aquadx_wacca_api_call", "api" to path).increment() + if (path in cacheMap) return resp(cacheMap[path]!!) - else if (path !in handlerMap) return resp("[]", 1, "Not Found") log.info("Wacca < $path : $body") - return try { metrics[path] { - val br = JACKSON.parse(body) - handlerMap[path]!!(br, br.params).let { when (it) { - is String -> resp(it) - is List<*> -> resp(it.toJson()) - else -> error("Invalid response type ${it.javaClass}") - } }.also { log.info("Wacca > $path : ${it.body}") } - } } - catch (e: ApiException) { - resp("[]", e.code, e.message ?: "") - } - catch (e: Exception) { - log.error("Wacca > Error", e) - resp("[]", 500, e.message ?: "") + return try { + Metrics.timer("aquadx_wacca_api_latency", "api" to path).recordCallable { + val br = JACKSON.parse(body) + handlerMap[path]!!(br, br.params).let { when (it) { + is String -> resp(it) + is List<*> -> resp(it.toJson()) + else -> error("Invalid response type ${it.javaClass}") + } }.also { log.info("Wacca > $path : ${it.body}") } + } + } catch (e: Exception) { + Metrics.counter( + "aquadx_wacca_api_error", + "api" to path, "error" to e.simpleDescribe() + ).increment() + + if (e is ApiException) { + resp("[]", e.code, e.message ?: "") + } else { + log.error("Wacca > Error", e) + resp("[]", 500, e.message ?: "") + } } } } diff --git a/src/main/java/icu/samnyan/aqua/spring/Metrics.kt b/src/main/java/icu/samnyan/aqua/spring/Metrics.kt new file mode 100644 index 00000000..d37d62b4 --- /dev/null +++ b/src/main/java/icu/samnyan/aqua/spring/Metrics.kt @@ -0,0 +1,45 @@ +package icu.samnyan.aqua.spring + +import io.micrometer.core.instrument.Counter +import io.micrometer.core.instrument.Timer +import java.util.concurrent.ConcurrentHashMap +import io.micrometer.core.instrument.Metrics as MMetrics + +object Metrics { + fun counter(metricName: String, vararg pairs: Pair): Counter { + val expandedLabels = expandLabels(*pairs) + return cache.computeIfAbsent(MetricCacheKey(Counter::class.java, metricName, expandedLabels)) { + Counter + .builder(metricName) + .tags(*expandedLabels) + .register(MMetrics.globalRegistry) + } as Counter + } + + fun timer(metricName: String, vararg pairs: Pair): Timer { + val expandedLabels = expandLabels(*pairs) + return cache.computeIfAbsent(MetricCacheKey(Timer::class.java, metricName, expandedLabels)) { + Timer + .builder(metricName) + .publishPercentiles(0.5, 0.75, 0.90, 0.95, 0.99) + .tags(*expandedLabels) + .register(MMetrics.globalRegistry) + } as Timer + } + + private data class MetricCacheKey( + val type: Class<*>, + val metricName: String, + val expandedLabels: Array, + ) + + private val cache = ConcurrentHashMap() + + private fun expandLabels(vararg pairs: Pair): Array { + return pairs + .flatMap { + listOf(it.first, it.second.toString()) + } + .toTypedArray() + } +} From 1a06033964af18b7140d29ba9b175804bb2f086f Mon Sep 17 00:00:00 2001 From: Azalea <22280294+hykilpikonna@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:06:32 -0500 Subject: [PATCH 7/7] [+] Hide allnet port --- config/application.properties | 1 + src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/application.properties b/config/application.properties index 94da66f8..484f64fb 100644 --- a/config/application.properties +++ b/config/application.properties @@ -13,6 +13,7 @@ billing.server.port=8443 ## Please notice most games won't work with localhost or 127.0.0.1 #allnet.server.host= #allnet.server.port= +allnet.server.hide-port=true ## This is for some games that use shop name for in-game functions. ## Specify the place name here if you need it. By default it is empty. allnet.server.place-name=AquaDX diff --git a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt index 936e9b7c..48ccf5d7 100644 --- a/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt +++ b/src/main/java/icu/samnyan/aqua/sega/allnet/AllNet.kt @@ -23,6 +23,7 @@ import java.util.* class AllNetProps { var host: String = "" var port: Int? = null + var hidePort: Boolean = true val keychipSesExpire: Long = 172800000 // milliseconds var checkKeychip: Boolean = false var keychipPermissiveForTesting: Boolean = false @@ -173,13 +174,13 @@ class AllNet( } private fun switchUri(localAddr: Str, localPort: Str, gameId: Str, ver: Str, session: Str?): Str { - val addr = props.host.ifBlank { localAddr } - val port = props.port?.toString() ?: localPort + val addr = props.host.ifBlank { localAddr } + + if (props.hidePort) "" else ":${props.port?.toString() ?: localPort}" // If keychip authentication is enabled, the game URLs will be set to /gs/{token}/{game}/... val base = if (session != null) "gs/$session" else "g" - return "http://$addr:$port/$base/" + when (gameId) { + return "http://$addr/$base/" + when (gameId) { "SDBT" -> "chu2/$ver/$session/" "SDHD" -> "chu3/$ver/" "SDGS" -> "chu3/$ver/" // International (c3exp)