diff --git a/assets/openapi.json b/assets/openapi.json index 5262d7c0..dcf58752 100644 Binary files a/assets/openapi.json and b/assets/openapi.json differ diff --git a/assets/schemas.json b/assets/schemas.json index 4764252c..4e4b05f0 100644 Binary files a/assets/schemas.json and b/assets/schemas.json differ diff --git a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs index 811143fe..e4018a1e 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs @@ -173,16 +173,20 @@ public class UserController(ILogger logger, Configuration config total_messages = messages.Count(), total_channels = channels.Count, messages_per_channel = channels.ToDictionary(c => c.ChannelId, c => messages.Count(m => m.ChannelId == c.ChannelId)) }); - var results = channels - .Select(ctx => DeleteMessagesForChannel(ctx.GuildId, ctx.ChannelId!, id, mqChannel, messageDeleteChunkSize)) - .ToList(); - var a = AggregateAsyncEnumerablesWithoutOrder(results); - await foreach (var result in a) { - yield return result; - } + if (messages.Any()) { + var results = channels + .Select(ctx => DeleteMessagesForChannel(ctx.GuildId, ctx.ChannelId!, id, mqChannel, messageDeleteChunkSize)) + .ToList(); + var a = AggregateAsyncEnumerablesWithoutOrder(results); + await foreach (var result in a) { + yield return result; + } - await db.Database.ExecuteSqlRawAsync("VACUUM FULL messages"); - await db.Database.ExecuteSqlRawAsync("REINDEX TABLE messages"); + if (messages.Count() >= 100) { + await db.Database.ExecuteSqlRawAsync("VACUUM FULL messages"); + await db.Database.ExecuteSqlRawAsync("REINDEX TABLE messages"); + } + } } private async IAsyncEnumerable DeleteMessagesForChannel( @@ -203,10 +207,10 @@ public class UserController(ILogger logger, Configuration config var messageIds = _db.Database.SqlQuery($""" DELETE FROM messages WHERE id IN ( - SELECT id FROM messages - WHERE author_id = {authorId} - AND channel_id = {channelId} - AND guild_id = {guildId} + SELECT id FROM messages + WHERE author_id = {authorId} + AND channel_id = {channelId} + AND guild_id = {guildId} LIMIT {messageDeleteChunkSize} ) RETURNING id; """).ToList(); @@ -410,7 +414,7 @@ public class UserController(ILogger logger, Configuration config } } } - + // { // "op": 0, // "t": "GUILD_ROLE_UPDATE", @@ -432,7 +436,7 @@ public class UserController(ILogger logger, Configuration config // }, // "s": 38 // } - + [HttpGet("test")] public async IAsyncEnumerable Test() { var factory = new ConnectionFactory { @@ -446,7 +450,7 @@ public class UserController(ILogger logger, Configuration config var roleId = "1391303296148639051"; //Spacebar Maintainer // int color = 16711680; //Administrator int color = 99839; //Spacebar Maintainer - + await mqChannel.ExchangeDeclareAsync(exchange: guildId, type: ExchangeType.Fanout, durable: false); var props = new BasicProperties() { Type = "GUILD_ROLE_UPDATE" }; @@ -492,4 +496,4 @@ public class UserController(ILogger logger, Configuration config sw.Restart(); } } -} \ No newline at end of file +} diff --git a/src/api/routes/channels/#channel_id/messages/index.ts b/src/api/routes/channels/#channel_id/messages/index.ts index 482db1d7..d6df9995 100644 --- a/src/api/routes/channels/#channel_id/messages/index.ts +++ b/src/api/routes/channels/#channel_id/messages/index.ts @@ -19,30 +19,41 @@ import { handleMessage, postHandleMessage, route } from "@spacebar/api"; import { Attachment, + AutomodRule, + AutomodTriggerTypes, Channel, Config, DmChannelDTO, + emitEvent, FieldErrors, + getPermission, + getUrlSignature, Member, Message, MessageCreateEvent, + NewUrlSignatureData, + NewUrlUserSignatureData, ReadState, Rights, Snowflake, - User, - emitEvent, - getPermission, - getUrlSignature, uploadFile, - NewUrlSignatureData, - NewUrlUserSignatureData, + User, } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { HTTPError } from "lambert-server"; import multer from "multer"; import { FindManyOptions, FindOperator, LessThan, MoreThan, MoreThanOrEqual } from "typeorm"; import { URL } from "url"; -import { isTextChannel, MessageCreateAttachment, MessageCreateCloudAttachment, MessageCreateSchema, Reaction } from "@spacebar/schemas"; +import { + AutomodCustomWordsRule, + AutomodRuleActionType, + AutomodRuleEventType, + isTextChannel, + MessageCreateAttachment, + MessageCreateCloudAttachment, + MessageCreateSchema, + Reaction, +} from "@spacebar/schemas"; const router: Router = Router({ mergeParams: true }); @@ -404,6 +415,72 @@ router.post( // eslint-disable-next-line @typescript-eslint/ban-ts-comment //@ts-ignore message.member.roles = message.member.roles.filter((x) => x.id != x.guild_id).map((x) => x.id); + + try { + if (message.content) + for (const rule of await AutomodRule.find({ where: { guild_id: message.guild_id, enabled: true, event_type: AutomodRuleEventType.MESSAGE_SEND } })) { + if (rule.exempt_channels.includes(channel_id)) continue; + if (message.member.roles.some((x) => rule.exempt_roles.includes(x.id))) continue; + if (rule.trigger_type == AutomodTriggerTypes.CUSTOM_WORDS) { + const triggerMeta = rule.trigger_metadata as AutomodCustomWordsRule; + const regexes = triggerMeta.regex_patterns + .map((x) => new RegExp(x, "i")) + .concat( + triggerMeta.keyword_filter + .map((k) => + k + // Convert simple wildcard patterns to regex + .replace(".", "\\.") + .replace("?", ".") + .replace("*", ".*"), + ) + .map((k) => new RegExp(k, "i")), + ); + const allowedRegexes = triggerMeta.allow_list + .map((k) => + k + // Convert simple wildcard patterns to regex + .replace(".", "\\.") + .replace("?", ".") + .replace("*", ".*"), + ) + .map((k) => new RegExp(k, "i")); + + const matches = regexes + .map((r) => message.content!.match(r)) + .filter((x) => x !== null && x.length > 0) + .filter((x) => !allowedRegexes.some((ar) => ar.test(x![0]))); + if (matches.length > 0) { + console.log("Automod triggered by message:", message.id, "matches:", matches); + if (rule.actions.some((x) => x.type == AutomodRuleActionType.SEND_ALERT_MESSAGE && x.metadata.channel_id)) { + const alertActions = rule.actions.filter((x) => x.type == AutomodRuleActionType.SEND_ALERT_MESSAGE); + for (const action of alertActions) { + const alertChannel = await Channel.findOne({ where: { id: action.metadata.channel_id } }); + if (!alertChannel) continue; + const msg = Message.create({ + channel_id: alertChannel.id, + content: `Automod Alert: Message ${message.id} by <@${message.author_id}> in <#${channel.id}> triggered automod rule "${rule.name}".\nMatched terms: ${matches + .map((x) => `\`${x![0]}\``) + .join(", ")}`, + author: message.author, + guild_id: message.channel.guild_id, + }); + await Promise.all([ + Message.insert(message), + emitEvent({ + event: "MESSAGE_CREATE", + channel_id: msg.channel_id, + data: msg.toJSON(), + } as MessageCreateEvent), + ]); + } + } + } + } + } + } catch (e) { + console.log("[Automod] failed to process message:", e); + } } let read_state = await ReadState.findOne({ diff --git a/src/cdn/Server.ts b/src/cdn/Server.ts index 321eba7c..f73d924a 100644 --- a/src/cdn/Server.ts +++ b/src/cdn/Server.ts @@ -129,7 +129,7 @@ export class CDNServer extends Server { } async cleanupSignaturesInDb() { - this.log("verbose", "[Server] Cleaning up signatures in database"); + this.log("verbose", "[CDN] Cleaning up signatures in database"); const attachmentsToFix = await Attachment.find({ where: { url: Like("%?ex=%") }, }); diff --git a/src/schemas/api/guilds/Automod.ts b/src/schemas/api/guilds/Automod.ts index 4195fa52..94398693 100644 --- a/src/schemas/api/guilds/Automod.ts +++ b/src/schemas/api/guilds/Automod.ts @@ -16,6 +16,8 @@ along with this program. If not, see . */ +import { Snowflake } from "../../Identifiers"; + export type AutomodRuleTriggerMetadata = AutomodMentionSpamRule | AutomodSuspectedSpamRule | AutomodCommonlyFlaggedWordsRule | AutomodCustomWordsRule; export class AutomodMentionSpamRule { @@ -26,12 +28,64 @@ export class AutomodMentionSpamRule { export class AutomodSuspectedSpamRule {} export class AutomodCommonlyFlaggedWordsRule { - allow_list: [string]; - presets: [number]; + allow_list: string[]; + presets: AutomodKeywordPresetType[]; } export class AutomodCustomWordsRule { - allow_list: [string]; - keyword_filter: [string]; - regex_patterns: [string]; + allow_list: string[]; + keyword_filter: string[]; + regex_patterns: string[]; } + +export enum AutomodRuleEventType { + MESSAGE_SEND = 1, + GUILD_MEMBER_EVENT = 2, +} +export enum AutomodRuleTriggerType { + KEYWORD = 1, + HARMFUL_LINK = 2, + SPAM = 3, + KEYWORD_PRESET = 4, + MENTION_SPAM = 5, + USER_PROFILE = 6, + GUILD_POLICY = 7, +} + +export enum AutomodKeywordPresetType { + PROFANITY = 1, + SEXUAL_CONTENT = 2, + SLURS = 3, +} + +export enum AutomodRuleActionType { + BLOCK_MESSAGE = 1, + SEND_ALERT_MESSAGE = 2, + TIMEOUT_USER = 3, + QUARANTINE_USER = 4 +} + +export type AutomodAction = { + type: AutomodRuleActionType.BLOCK_MESSAGE; + metadata: { + custom_message?: string; + } +} | { + type: AutomodRuleActionType.SEND_ALERT_MESSAGE; + metadata: { + channel_id: Snowflake; + }; +} | { + type: AutomodRuleActionType.TIMEOUT_USER; + metadata: { + duration_seconds: number; + }; +} | { + type: AutomodRuleActionType.QUARANTINE_USER; + metadata: { + duration_seconds: number; + }; +}; +export interface AutomodRuleActionMetadata { + +} \ No newline at end of file diff --git a/src/util/entities/AutomodRule.ts b/src/util/entities/AutomodRule.ts index 5765b662..ad8b8e06 100644 --- a/src/util/entities/AutomodRule.ts +++ b/src/util/entities/AutomodRule.ts @@ -19,7 +19,7 @@ import { BaseClass } from "./BaseClass"; import { Entity, JoinColumn, ManyToOne, Column } from "typeorm"; import { User } from "./User"; -import { AutomodRuleTriggerMetadata } from "@spacebar/schemas"; +import { AutomodAction, AutomodRuleActionType, AutomodRuleEventType, AutomodRuleTriggerMetadata, AutomodRuleTriggerType } from "@spacebar/schemas"; @Entity({ name: "automod_rules", @@ -33,13 +33,13 @@ export class AutomodRule extends BaseClass { enabled: boolean; @Column() - event_type: number; // No idea... + event_type: AutomodRuleEventType; @Column({ type: "simple-array" }) - exempt_channels: [string]; + exempt_channels: string[]; @Column({ type: "simple-array" }) - exempt_roles: [string]; + exempt_roles: string[]; @Column() guild_id: string; @@ -51,7 +51,7 @@ export class AutomodRule extends BaseClass { position: number; @Column() - trigger_type: number; + trigger_type: AutomodRuleTriggerType; @Column({ type: "simple-json", @@ -63,5 +63,5 @@ export class AutomodRule extends BaseClass { @Column({ type: "simple-json", }) - actions: { type: number; metadata: unknown }[]; + actions: AutomodAction[]; } diff --git a/src/util/util/Config.ts b/src/util/util/Config.ts index 1a97b198..2fd9c0fb 100644 --- a/src/util/util/Config.ts +++ b/src/util/util/Config.ts @@ -32,8 +32,8 @@ let pairs: ConfigEntity[]; // Config keys are separated with _ export class Config { - public static async init() { - if (config) return config; + public static async init(force: boolean = false) { + if (config && !force) return config; console.log("[Config] Loading configuration..."); if (!process.env.CONFIG_PATH) { pairs = await validateConfig(); @@ -59,7 +59,7 @@ export class Config { public static get() { if (!config) { // If we haven't initialised the config yet, return default config. - // Typeorm instantiates each entity once when initising database, + // Typeorm instantiates each entity once when initialising database, // which means when we use config values as default values in entity classes, // the config isn't initialised yet and would throw an error about the config being undefined. @@ -100,7 +100,8 @@ async function applyConfig(val: ConfigValue) { else console.log("[WARNING] JSON config file in use, and writing is disabled! Programmatic config changes will not be persisted, and your config will not get updated!"); else { const pairs = generatePairs(val); - await Promise.all(pairs.map((pair) => pair.save())); + // keys are sorted to try to influence database order... + await Promise.all(pairs.sort((x, y) => x.key > y.key ? 1 : -1).map((pair) => pair.save())); } return val; } @@ -134,9 +135,13 @@ function pairsToConfig(pairs: ConfigEntity[]) { const validateConfig = async () => { let hasErrored = false; + const totalStartTime = new Date(); const config = await ConfigEntity.find({ select: { key: true } }); for (const row in config) { + // extension methods... + if(typeof config[row] === "function") continue; + try { const found = await ConfigEntity.findOne({ where: { key: config[row].key }, @@ -153,9 +158,11 @@ const validateConfig = async () => { } } + console.log("[Config] Total config load time:", new Date().getTime() - totalStartTime.getTime(), "ms"); + if (hasErrored) { console.error( - "Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration", + "[Config] Your config has invalid values. Fix them first https://docs.spacebar.chat/setup/server/configuration", ); process.exit(1); }