From b02371e4c3dc8bf60f4936f31a15309fe088512f Mon Sep 17 00:00:00 2001 From: Azalea <22280294+hykilpikonna@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:49:45 -0500 Subject: [PATCH] [O] Optimize auto-ban --- .../icu/samnyan/aqua/net/CardController.kt | 20 ++++++--- src/main/java/icu/samnyan/aqua/net/Safety.kt | 43 +++++++++++-------- .../java/icu/samnyan/aqua/net/games/Models.kt | 21 ++++----- .../sega/maimai/model/userdata/UserData.java | 2 +- .../maimai2/model/userdata/Mai2UserDetail.kt | 2 +- .../sega/ongeki/model/userdata/UserData.java | 2 +- .../aqua/sega/wacca/model/db/WaccaUser.kt | 2 +- 7 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/main/java/icu/samnyan/aqua/net/CardController.kt b/src/main/java/icu/samnyan/aqua/net/CardController.kt index 01131ed6..d9dc22a7 100644 --- a/src/main/java/icu/samnyan/aqua/net/CardController.kt +++ b/src/main/java/icu/samnyan/aqua/net/CardController.kt @@ -13,6 +13,7 @@ import icu.samnyan.aqua.sega.general.model.Card import icu.samnyan.aqua.sega.general.service.CardService import icu.samnyan.aqua.sega.maimai2.model.Mai2UserDataRepo import icu.samnyan.aqua.sega.wacca.model.db.WcUserRepo +import jakarta.persistence.EntityManager import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service import org.springframework.web.bind.annotation.RestController @@ -178,7 +179,8 @@ class CardGameService( val ongeki: icu.samnyan.aqua.sega.ongeki.dao.userdata.UserDataRepository, val diva: icu.samnyan.aqua.sega.diva.dao.userdata.PlayerProfileRepository, val safety: AquaNetSafetyService, - val cardRepo: CardRepository + val cardRepo: CardRepository, + val em: EntityManager ) { companion object { val log = logger() @@ -220,16 +222,20 @@ class CardGameService( @Scheduled(fixedDelay = 3600000) suspend fun autoBan() { log.info("Running auto-ban") + val time = millis() // Ban any players with unacceptable names for (repo in listOf(maimai2, chusan, wacca, ongeki)) { - repo.findAll().filter { it.card != null && !it.card!!.rankingBanned }.forEach { data -> - if (!safety.isSafe(data.userName)) { - log.info("Banning user ${data.userName} ${data.card!!.id}") - data.card!!.rankingBanned = true - async { cardRepo.save(data.card!!) } - } + val all = async { repo.findAllNonBanned() } + val isSafe = safety.isSafeBatch(all.map { it.userName }) + val toSave = all.filterIndexed { i, _ -> !isSafe[i] }.mapNotNull { it.card } + if (toSave.isNotEmpty()) { + log.info("Banning users ${toSave.joinToString(", ")}") + toSave.forEach { it.rankingBanned = true } + async { cardRepo.saveAll(toSave) } } } + + log.info("Auto-ban completed in ${millis() - time}ms") } } diff --git a/src/main/java/icu/samnyan/aqua/net/Safety.kt b/src/main/java/icu/samnyan/aqua/net/Safety.kt index c04f4fdf..33c629c8 100644 --- a/src/main/java/icu/samnyan/aqua/net/Safety.kt +++ b/src/main/java/icu/samnyan/aqua/net/Safety.kt @@ -1,7 +1,6 @@ package icu.samnyan.aqua.net import ext.HTTP -import ext.async import ext.toJson import icu.samnyan.aqua.net.games.BaseEntity import io.ktor.client.call.* @@ -50,25 +49,31 @@ class AquaNetSafetyService( val safety: AquaNetSafetyRepo, val openAIConfig: OpenAIConfig ) { - suspend fun isSafe(rawContent: String): Boolean { - // NFKC normalize - val content = Normalizer.normalize(rawContent, Normalizer.Form.NFKC) - if (content.isBlank()) return true + /** + * It is very inefficient to have query inside a loop, so we batch the query. + */ + suspend fun isSafeBatch(rawContents: List): List { + val contents = rawContents.map { Normalizer.normalize(it, Normalizer.Form.NFKC) } + val map = safety.findAll().associateBy { it.content.lowercase().trim() }.toMutableMap() - async { safety.findByContent(content) }?.let { return it.safe } - - // Query OpenAI - HTTP.post("https://api.openai.com/v1/moderations") { - header("Authorization", "Bearer ${openAIConfig.apiKey}") - header("Content-Type", "application/json") - setBody(mapOf("input" to content).toJson()) - }.let { - if (!it.status.isSuccess()) return true - val body = it.body>() - return AquaNetSafety().apply { - this.content = content - this.safe = !body.results.first().flagged - }.also { safety.save(it) }.safe + // Process unseen content with OpenAI + val news = contents.filter { it.lowercase().trim() !in map }.map { inp -> + HTTP.post("https://api.openai.com/v1/moderations") { + header("Authorization", "Bearer ${openAIConfig.apiKey}") + header("Content-Type", "application/json") + setBody(mapOf("input" to inp).toJson()) + }.let { + if (!it.status.isSuccess()) throw Exception("OpenAI request failed for $inp") + val body = it.body>() + AquaNetSafety().apply { + content = inp + safe = !body.results.first().flagged + } + } } + if (news.isNotEmpty()) safety.saveAll(news) + news.associateByTo(map) { it.content.lowercase().trim() } + + return contents.map { map[it.lowercase().trim()]!!.safe } } } diff --git a/src/main/java/icu/samnyan/aqua/net/games/Models.kt b/src/main/java/icu/samnyan/aqua/net/games/Models.kt index 5db8f7de..67437bac 100644 --- a/src/main/java/icu/samnyan/aqua/net/games/Models.kt +++ b/src/main/java/icu/samnyan/aqua/net/games/Models.kt @@ -1,13 +1,13 @@ package icu.samnyan.aqua.net.games import com.fasterxml.jackson.annotation.JsonIgnore -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.annotation.JsonSerialize import ext.JACKSON import ext.JavaSerializable import icu.samnyan.aqua.sega.general.model.Card -import icu.samnyan.aqua.sega.util.jackson.AccessCodeSerializer -import jakarta.persistence.* +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.MappedSuperclass import kotlinx.serialization.Serializable import org.springframework.data.domain.Page import org.springframework.data.domain.Pageable @@ -95,6 +95,7 @@ interface IUserData { } interface IGenericGamePlaylog { + val user: IUserData val musicId: Int val level: Int val userPlayDate: Any @@ -116,21 +117,15 @@ open class BaseEntity( override fun toString() = JACKSON.writeValueAsString(this) } -@MappedSuperclass -open class UserDataEntity : BaseEntity() { - @JsonSerialize(using = AccessCodeSerializer::class) - @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY) - @OneToOne - @JoinColumn(name = "aime_card_id", unique = true) - var card: Card? = null -} - @NoRepositoryBean interface GenericUserDataRepo : JpaRepository { fun findByCard(card: Card): T? fun findByCard_ExtId(extId: Long): Optional @Query("select count(*) from #{#entityName} e where e.playerRating > :rating and e.card.rankingBanned = false") fun getRanking(rating: Int): Long + + @Query("select e from #{#entityName} e where e.card.rankingBanned = false") + fun findAllNonBanned(): List } @NoRepositoryBean diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai/model/userdata/UserData.java b/src/main/java/icu/samnyan/aqua/sega/maimai/model/userdata/UserData.java index 13783376..6c38b07e 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai/model/userdata/UserData.java +++ b/src/main/java/icu/samnyan/aqua/sega/maimai/model/userdata/UserData.java @@ -31,7 +31,7 @@ public class UserData implements Serializable { @JsonSerialize(using = AccessCodeSerializer.class) @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY) - @OneToOne + @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "aime_card_id") private Card card; diff --git a/src/main/java/icu/samnyan/aqua/sega/maimai2/model/userdata/Mai2UserDetail.kt b/src/main/java/icu/samnyan/aqua/sega/maimai2/model/userdata/Mai2UserDetail.kt index 18f79bde..0dff70f7 100644 --- a/src/main/java/icu/samnyan/aqua/sega/maimai2/model/userdata/Mai2UserDetail.kt +++ b/src/main/java/icu/samnyan/aqua/sega/maimai2/model/userdata/Mai2UserDetail.kt @@ -16,7 +16,7 @@ import jakarta.persistence.* class Mai2UserDetail( @get:JsonSerialize(using = AccessCodeSerializer::class) @get:JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY) - @OneToOne + @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "aime_card_id", unique = true) override var card: Card? = null, diff --git a/src/main/java/icu/samnyan/aqua/sega/ongeki/model/userdata/UserData.java b/src/main/java/icu/samnyan/aqua/sega/ongeki/model/userdata/UserData.java index 6fa5357b..5d15951e 100644 --- a/src/main/java/icu/samnyan/aqua/sega/ongeki/model/userdata/UserData.java +++ b/src/main/java/icu/samnyan/aqua/sega/ongeki/model/userdata/UserData.java @@ -32,7 +32,7 @@ public class UserData implements Serializable, IUserData { @JsonSerialize(using = AccessCodeSerializer.class) @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY) - @OneToOne + @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "aime_card_id", unique = true) private Card card; // Access code in card diff --git a/src/main/java/icu/samnyan/aqua/sega/wacca/model/db/WaccaUser.kt b/src/main/java/icu/samnyan/aqua/sega/wacca/model/db/WaccaUser.kt index 90f0c9f5..c20e60e6 100644 --- a/src/main/java/icu/samnyan/aqua/sega/wacca/model/db/WaccaUser.kt +++ b/src/main/java/icu/samnyan/aqua/sega/wacca/model/db/WaccaUser.kt @@ -16,7 +16,7 @@ import java.util.* */ @Entity @Table(name = "wacca_user") class WaccaUser : BaseEntity(), IUserData { - @OneToOne + @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "aime_card_id", unique = true) override var card: Card? = Card()