From f20dff80eff002d39a662eedb538a486ec6be76b Mon Sep 17 00:00:00 2001 From: Rory& Date: Sun, 28 Sep 2025 20:09:38 +0200 Subject: [PATCH] Make cloud uploads work --- .../#channel_id/messages/#message_id/index.ts | 7 +- .../channels/#channel_id/messages/index.ts | 7 +- src/api/util/handlers/Message.ts | 222 ++++++++---------- src/cdn/routes/attachments.ts | 15 +- 4 files changed, 116 insertions(+), 135 deletions(-) diff --git a/src/api/routes/channels/#channel_id/messages/#message_id/index.ts b/src/api/routes/channels/#channel_id/messages/#message_id/index.ts index 6fa4cd38..3d1103e2 100644 --- a/src/api/routes/channels/#channel_id/messages/#message_id/index.ts +++ b/src/api/routes/channels/#channel_id/messages/#message_id/index.ts @@ -32,6 +32,8 @@ import { getRights, uploadFile, NewUrlUserSignatureData, + MessageCreateAttachment, + MessageCreateCloudAttachment, } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { HTTPError } from "lambert-server"; @@ -93,7 +95,8 @@ router.patch( } } else rights.hasThrow("SELF_EDIT_MESSAGES"); - //@ts-expect-error Something is wrong with message_reference here, TS complains since "channel_id" is optional in MessageCreateSchema + // no longer necessary, somehow resolved by updating the type of `attachments`...? + // //@ts-expect-error Something is wrong with message_reference here, TS complains since "channel_id" is optional in MessageCreateSchema const new_message = await handleMessage({ ...message, // TODO: should message_reference be overridable? @@ -172,7 +175,7 @@ router.put( async (req: Request, res: Response) => { const { channel_id, message_id } = req.params; const body = req.body as MessageCreateSchema; - const attachments: Attachment[] = []; + const attachments: (MessageCreateAttachment | MessageCreateCloudAttachment)[] = body.attachments ?? []; const rights = await getRights(req.user_id); rights.hasThrow("SEND_MESSAGES"); diff --git a/src/api/routes/channels/#channel_id/messages/index.ts b/src/api/routes/channels/#channel_id/messages/index.ts index d24aff05..8030e69d 100644 --- a/src/api/routes/channels/#channel_id/messages/index.ts +++ b/src/api/routes/channels/#channel_id/messages/index.ts @@ -39,6 +39,8 @@ import { uploadFile, NewUrlSignatureData, NewUrlUserSignatureData, + MessageCreateCloudAttachment, + MessageCreateAttachment, } from "@spacebar/util"; import { Request, Response, Router } from "express"; import { HTTPError } from "lambert-server"; @@ -51,6 +53,8 @@ import { MoreThanOrEqual, } from "typeorm"; import { URL } from "url"; +import fetch from "node-fetch-commonjs"; +import { CloudAttachment } from "../../../../../util/entities/CloudAttachment"; const router: Router = Router(); @@ -296,7 +300,7 @@ router.post( async (req: Request, res: Response) => { const { channel_id } = req.params; const body = req.body as MessageCreateSchema; - const attachments: Attachment[] = []; + const attachments: (Attachment | MessageCreateAttachment | MessageCreateCloudAttachment)[] = body.attachments ?? []; const channel = await Channel.findOneOrFail({ where: { id: channel_id }, @@ -364,6 +368,7 @@ router.post( const embeds = body.embeds || []; if (body.embed) embeds.push(body.embed); + console.log("messages/index.ts: attachments:", attachments); const message = await handleMessage({ ...body, type: 0, diff --git a/src/api/util/handlers/Message.ts b/src/api/util/handlers/Message.ts index e350fad7..7141ff17 100644 --- a/src/api/util/handlers/Message.ts +++ b/src/api/util/handlers/Message.ts @@ -47,28 +47,38 @@ import { Permissions, normalizeUrl, Reaction, + MessageCreateCloudAttachment, + MessageCreateAttachment, } from "@spacebar/util"; import { HTTPError } from "lambert-server"; import { In } from "typeorm"; import fetch from "node-fetch-commonjs"; +import { CloudAttachment } from "../../../util/entities/CloudAttachment"; const allow_empty = false; // TODO: check webhook, application, system author, stickers // TODO: embed gifs/videos/images -const LINK_REGEX = - /?/g; +const LINK_REGEX = /?/g; export async function handleMessage(opts: MessageOptions): Promise { const channel = await Channel.findOneOrFail({ where: { id: opts.channel_id }, relations: ["recipients"], }); - if (!channel || !opts.channel_id) - throw new HTTPError("Channel not found", 404); + if (!channel || !opts.channel_id) throw new HTTPError("Channel not found", 404); + + const stickers = opts.sticker_ids ? await Sticker.find({ where: { id: In(opts.sticker_ids) } }) : undefined; + // cloud attachments with indexes + const cloudAttachments = opts.attachments?.reduce( + (acc, att, index) => { + if ("uploaded_filename" in att) { + acc.push({ attachment: att, index }); + } + return acc; + }, + [] as { attachment: MessageCreateCloudAttachment; index: number }[], + ); - const stickers = opts.sticker_ids - ? await Sticker.find({ where: { id: In(opts.sticker_ids) } }) - : undefined; const message = Message.create({ ...opts, poll: opts.poll, @@ -82,10 +92,54 @@ export async function handleMessage(opts: MessageOptions): Promise { mentions: [], }); - if ( - message.content && - message.content.length > Config.get().limits.message.maxCharacters - ) { + if (cloudAttachments && cloudAttachments.length > 0) { + console.log("[Message] Processing attachments for message", message.id, ":", message.attachments); + const uploadedAttachments = await Promise.all( + cloudAttachments.map(async (att) => { + const cAtt = att.attachment; + const attEnt = await CloudAttachment.findOneOrFail({ + where: { + uploadFilename: cAtt.uploaded_filename, + }, + }); + + const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${attEnt.uploadFilename}/clone_to_message/${message.id}`, { + method: "POST", + headers: { + "signature": Config.get().security.requestSignature || "", + }, + }); + + if (!cloneResponse.ok) { + console.error(`[Message] Failed to clone attachment ${attEnt.userFilename} to message ${message.id}`); + throw new HTTPError("Failed to process attachment: " + (await cloneResponse.text()), 500); + } + + const cloneRespBody = (await cloneResponse.json()) as { success: boolean; new_path: string }; + + const realAtt = Attachment.create({ + filename: attEnt.userFilename, + url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`, + proxy_url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`, + size: attEnt.size, + height: attEnt.height, + width: attEnt.width, + content_type: attEnt.contentType || attEnt.userOriginalContentType, + }); + await realAtt.save(); + return { attachment: realAtt, index: att.index }; + }), + ); + console.log("[Message] Processed attachments for message", message.id, ":", message.attachments); + + for (const att of uploadedAttachments) { + message.attachments![att.index] = att.attachment; + } + } else console.log("[Message] No cloud attachments to process for message", message.id, ":", message.attachments); + + console.log("opts:", opts.attachments, "\nmessage:", message.attachments); + + if (message.content && message.content.length > Config.get().limits.message.maxCharacters) { throw new HTTPError("Content length over max character limit"); } @@ -138,28 +192,15 @@ export async function handleMessage(opts: MessageOptions): Promise { } if (opts.avatar_url) { const avatarData = await fetch(opts.avatar_url); - const base64 = await avatarData - .buffer() - .then((x) => x.toString("base64")); + const base64 = await avatarData.buffer().then((x) => x.toString("base64")); - const dataUri = - "data:" + - avatarData.headers.get("content-type") + - ";base64," + - base64; + const dataUri = "data:" + avatarData.headers.get("content-type") + ";base64," + base64; - message.avatar = await handleFile( - `/avatars/${opts.webhook_id}`, - dataUri as string, - ); + message.avatar = await handleFile(`/avatars/${opts.webhook_id}`, dataUri as string); message.author.avatar = message.avatar; } } else { - permission = await getPermission( - opts.author_id, - channel.guild_id, - opts.channel_id, - ); + permission = await getPermission(opts.author_id, channel.guild_id, opts.channel_id); permission.hasThrow("SEND_MESSAGES"); if (permission.cache.member) { message.member = permission.cache.member; @@ -173,20 +214,12 @@ export async function handleMessage(opts: MessageOptions): Promise { const guild = await Guild.findOneOrFail({ where: { id: channel.guild_id }, }); - if (!opts.message_reference.guild_id) - opts.message_reference.guild_id = channel.guild_id; - if (!opts.message_reference.channel_id) - opts.message_reference.channel_id = opts.channel_id; + if (!opts.message_reference.guild_id) opts.message_reference.guild_id = channel.guild_id; + if (!opts.message_reference.channel_id) opts.message_reference.channel_id = opts.channel_id; if (!guild.features.includes("CROSS_CHANNEL_REPLIES")) { - if (opts.message_reference.guild_id !== channel.guild_id) - throw new HTTPError( - "You can only reference messages from this guild", - ); - if (opts.message_reference.channel_id !== opts.channel_id) - throw new HTTPError( - "You can only reference messages from this channel", - ); + if (opts.message_reference.guild_id !== channel.guild_id) throw new HTTPError("You can only reference messages from this guild"); + if (opts.message_reference.channel_id !== opts.channel_id) throw new HTTPError("You can only reference messages from this channel"); } message.message_reference = opts.message_reference; @@ -198,15 +231,7 @@ export async function handleMessage(opts: MessageOptions): Promise { } // TODO: stickers/activity - if ( - !allow_empty && - !opts.content && - !opts.embeds?.length && - !opts.attachments?.length && - !opts.sticker_ids?.length && - !opts.poll && - !opts.components?.length - ) { + if (!allow_empty && !opts.content && !opts.embeds?.length && !opts.attachments?.length && !opts.sticker_ids?.length && !opts.poll && !opts.components?.length) { console.log("[Message] Rejecting empty message:", opts, message); throw new HTTPError("Empty messages are not allowed", 50006); } @@ -230,31 +255,22 @@ export async function handleMessage(opts: MessageOptions): Promise { }*/ for (const [, mention] of content.matchAll(USER_MENTION)) { - if (!mention_user_ids.includes(mention)) - mention_user_ids.push(mention); + if (!mention_user_ids.includes(mention)) mention_user_ids.push(mention); } await Promise.all( - Array.from(content.matchAll(ROLE_MENTION)).map( - async ([, mention]) => { - const role = await Role.findOneOrFail({ - where: { id: mention, guild_id: channel.guild_id }, - }); - if ( - role.mentionable || - opts.webhook_id || - permission?.has("MANAGE_ROLES") - ) { - mention_role_ids.push(mention); - } - }, - ), + Array.from(content.matchAll(ROLE_MENTION)).map(async ([, mention]) => { + const role = await Role.findOneOrFail({ + where: { id: mention, guild_id: channel.guild_id }, + }); + if (role.mentionable || opts.webhook_id || permission?.has("MANAGE_ROLES")) { + mention_role_ids.push(mention); + } + }), ); if (opts.webhook_id || permission?.has("MENTION_EVERYONE")) { - mention_everyone = - !!content.match(EVERYONE_MENTION) || - !!content.match(HERE_MENTION); + mention_everyone = !!content.match(EVERYONE_MENTION) || !!content.match(HERE_MENTION); } } @@ -265,10 +281,7 @@ export async function handleMessage(opts: MessageOptions): Promise { channel_id: message.channel_id, }, }); - if ( - referencedMessage && - referencedMessage.author_id !== message.author_id - ) { + if (referencedMessage && referencedMessage.author_id !== message.author_id) { message.mentions.push( User.create({ id: referencedMessage.author_id, @@ -282,46 +295,12 @@ export async function handleMessage(opts: MessageOptions): Promise { Channel.create({ id: x }), );*/ message.mention_roles = mention_role_ids.map((x) => Role.create({ id: x })); - message.mentions = [ - ...message.mentions, - ...mention_user_ids.map((x) => User.create({ id: x })), - ]; + message.mentions = [...message.mentions, ...mention_user_ids.map((x) => User.create({ id: x }))]; message.mention_everyone = mention_everyone; // TODO: check and put it all in the body - if(message.attachments?.some(att => "uploaded_filename" in att)) { - message.attachments = await Promise.all(message.attachments.map(async att => { - if("uploaded_filename" in att) { - const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${att.uploaded_filename}/clone_to_message/${message.id}`, { - method: "POST", - headers: { - "X-Signature": Config.get().security.requestSignature || "", - }, - }); - if(!cloneResponse.ok) { - console.error(`[Message] Failed to clone attachment ${att.uploaded_filename} to message ${message.id}`); - throw new HTTPError("Failed to process attachment: " + await cloneResponse.text(), 500); - } - - const cloneRespBody = await cloneResponse.json() as { success: boolean, new_path: string }; - - return Attachment.create({ - id: att.id, - filename: att.filename, - url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`, - proxy_url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`, - size: att.size, - height: att.height, - width: att.width, - content_type: att.content_type, - }); - } - return att; - })); - } - return message; } @@ -383,10 +362,7 @@ export async function postHandleMessage(message: Message) { channel_id: message.channel_id, data, } as MessageUpdateEvent), - Message.update( - { id: message.id, channel_id: message.channel_id }, - { embeds: data.embeds }, - ), + Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds }), ]); return; } @@ -415,12 +391,8 @@ export async function postHandleMessage(message: Message) { } // bit gross, but whatever! - const endpointPublic = - Config.get().cdn.endpointPublic || "http://127.0.0.1"; // lol - const handler = - url.hostname === new URL(endpointPublic).hostname - ? EmbedHandlers["self"] - : EmbedHandlers[url.hostname] || EmbedHandlers["default"]; + const endpointPublic = Config.get().cdn.endpointPublic || "http://127.0.0.1"; // lol + const handler = url.hostname === new URL(endpointPublic).hostname ? EmbedHandlers["self"] : EmbedHandlers[url.hostname] || EmbedHandlers["default"]; try { let res = await handler(url); @@ -438,10 +410,7 @@ export async function postHandleMessage(message: Message) { data.embeds.push(embed); } } catch (e) { - console.error( - `[Embeds] Error while generating embed for ${link}`, - e, - ); + console.error(`[Embeds] Error while generating embed for ${link}`, e); Sentry.captureException(e, (scope) => { scope.clear(); scope.setContext("request", { url: link }); @@ -457,10 +426,7 @@ export async function postHandleMessage(message: Message) { channel_id: message.channel_id, data, } as MessageUpdateEvent), - Message.update( - { id: message.id, channel_id: message.channel_id }, - { embeds: data.embeds }, - ), + Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds }), ...cachePromises, ]); } @@ -478,9 +444,7 @@ export async function sendMessage(opts: MessageOptions) { ]); // no await as it should catch error non-blockingly - postHandleMessage(message).catch((e) => - console.error("[Message] post-message handler failed", e), - ); + postHandleMessage(message).catch((e) => console.error("[Message] post-message handler failed", e)); return message; } @@ -495,7 +459,7 @@ interface MessageOptions extends MessageCreateSchema { embeds?: Embed[]; reactions?: Reaction[]; channel_id?: string; - attachments?: Attachment[]; // why are we masking this? + attachments?: (MessageCreateAttachment | MessageCreateCloudAttachment | Attachment)[]; // why are we masking this? edited_timestamp?: Date; timestamp?: Date; username?: string; diff --git a/src/cdn/routes/attachments.ts b/src/cdn/routes/attachments.ts index 23ca141a..4c65bbb2 100644 --- a/src/cdn/routes/attachments.ts +++ b/src/cdn/routes/attachments.ts @@ -150,7 +150,14 @@ router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("fil const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; await storage.set(path, buffer); - if (att.userOriginalContentType?.includes("image")) { + + let mimeType = att.userOriginalContentType; + if (att.userOriginalContentType === null) { + const ft = await FileType.fromBuffer(buffer); + mimeType = att.contentType = ft?.mime || "application/octet-stream"; + } + + if (mimeType?.includes("image")) { const dimensions = imageSize(buffer); if (dimensions) { att.width = dimensions.width; @@ -168,6 +175,7 @@ router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("fil router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + console.log("[Cloud Delete] Deleting attachment", req.params); const { channel_id, batch_id, attachment_id, filename } = req.params; const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; @@ -191,10 +199,11 @@ router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Req router.post("/:channel_id/:batch_id/:attachment_id/:filename/clone_to_message/:message_id", async (req: Request, res: Response) => { if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature"); + console.log("[Cloud Clone] Cloning attachment to message", req.params); - const { channel_id, batch_id, attachment_id, filename } = req.params; + const { channel_id, batch_id, attachment_id, filename, message_id } = req.params; const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`; - const newPath = `attachments/${channel_id}/${filename}`; + const newPath = `attachments/${channel_id}/${message_id}/${filename}`; const att = await CloudAttachment.findOne({ where: {