Make cloud uploads work

src/api/routes/channels/#channel_id/messages/#message_id/index.ts

@@ -32,6 +32,8 @@ import {
 	getRights,
 	uploadFile,
 	NewUrlUserSignatureData,
+	MessageCreateAttachment,
+	MessageCreateCloudAttachment,
 } from "@spacebar/util";
 import { Request, Response, Router } from "express";
 import { HTTPError } from "lambert-server";
@@ -93,7 +95,8 @@ router.patch(
 			}
 		} else rights.hasThrow("SELF_EDIT_MESSAGES");
 
-		//@ts-expect-error Something is wrong with message_reference here, TS complains since "channel_id" is optional in MessageCreateSchema
+		// no longer necessary, somehow resolved by updating the type of `attachments`...?
+		// //@ts-expect-error Something is wrong with message_reference here, TS complains since "channel_id" is optional in MessageCreateSchema
 		const new_message = await handleMessage({
 			...message,
 			// TODO: should message_reference be overridable?
@@ -172,7 +175,7 @@ router.put(
 	async (req: Request, res: Response) => {
 		const { channel_id, message_id } = req.params;
 		const body = req.body as MessageCreateSchema;
-		const attachments: Attachment[] = [];
+		const attachments: (MessageCreateAttachment | MessageCreateCloudAttachment)[] = body.attachments ?? [];
 
 		const rights = await getRights(req.user_id);
 		rights.hasThrow("SEND_MESSAGES");

src/api/routes/channels/#channel_id/messages/index.ts

@@ -39,6 +39,8 @@ import {
 	uploadFile,
 	NewUrlSignatureData,
 	NewUrlUserSignatureData,
+	MessageCreateCloudAttachment,
+	MessageCreateAttachment,
 } from "@spacebar/util";
 import { Request, Response, Router } from "express";
 import { HTTPError } from "lambert-server";
@@ -51,6 +53,8 @@ import {
 	MoreThanOrEqual,
 } from "typeorm";
 import { URL } from "url";
+import fetch from "node-fetch-commonjs";
+import { CloudAttachment } from "../../../../../util/entities/CloudAttachment";
 
 const router: Router = Router();
 
@@ -296,7 +300,7 @@ router.post(
 	async (req: Request, res: Response) => {
 		const { channel_id } = req.params;
 		const body = req.body as MessageCreateSchema;
-		const attachments: Attachment[] = [];
+		const attachments: (Attachment | MessageCreateAttachment | MessageCreateCloudAttachment)[] = body.attachments ?? [];
 
 		const channel = await Channel.findOneOrFail({
 			where: { id: channel_id },
@@ -364,6 +368,7 @@ router.post(
 
 		const embeds = body.embeds || [];
 		if (body.embed) embeds.push(body.embed);
+		console.log("messages/index.ts: attachments:", attachments);
 		const message = await handleMessage({
 			...body,
 			type: 0,

src/api/util/handlers/Message.ts

@@ -47,28 +47,38 @@ import {
 	Permissions,
 	normalizeUrl,
 	Reaction,
+	MessageCreateCloudAttachment,
+	MessageCreateAttachment,
 } from "@spacebar/util";
 import { HTTPError } from "lambert-server";
 import { In } from "typeorm";
 import fetch from "node-fetch-commonjs";
+import { CloudAttachment } from "../../../util/entities/CloudAttachment";
 const allow_empty = false;
 // TODO: check webhook, application, system author, stickers
 // TODO: embed gifs/videos/images
 
-const LINK_REGEX =
+const LINK_REGEX = /<?https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)>?/g;
-	/<?https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)>?/g;
 
 export async function handleMessage(opts: MessageOptions): Promise<Message> {
 	const channel = await Channel.findOneOrFail({
 		where: { id: opts.channel_id },
 		relations: ["recipients"],
 	});
-	if (!channel || !opts.channel_id)
+	if (!channel || !opts.channel_id) throw new HTTPError("Channel not found", 404);
-		throw new HTTPError("Channel not found", 404);
+
+	const stickers = opts.sticker_ids ? await Sticker.find({ where: { id: In(opts.sticker_ids) } }) : undefined;
+	// cloud attachments with indexes
+	const cloudAttachments = opts.attachments?.reduce(
+		(acc, att, index) => {
+			if ("uploaded_filename" in att) {
+				acc.push({ attachment: att, index });
+			}
+			return acc;
+		},
+		[] as { attachment: MessageCreateCloudAttachment; index: number }[],
+	);
 
-	const stickers = opts.sticker_ids
-		? await Sticker.find({ where: { id: In(opts.sticker_ids) } })
-		: undefined;
 	const message = Message.create({
 		...opts,
 		poll: opts.poll,
@@ -82,10 +92,54 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 		mentions: [],
 	});
 
-	if (
+	if (cloudAttachments && cloudAttachments.length > 0) {
-		message.content &&
+		console.log("[Message] Processing attachments for message", message.id, ":", message.attachments);
-		message.content.length > Config.get().limits.message.maxCharacters
+		const uploadedAttachments = await Promise.all(
-	) {
+			cloudAttachments.map(async (att) => {
+				const cAtt = att.attachment;
+				const attEnt = await CloudAttachment.findOneOrFail({
+					where: {
+						uploadFilename: cAtt.uploaded_filename,
+					},
+				});
+
+				const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${attEnt.uploadFilename}/clone_to_message/${message.id}`, {
+					method: "POST",
+					headers: {
+						"signature": Config.get().security.requestSignature || "",
+					},
+				});
+
+				if (!cloneResponse.ok) {
+					console.error(`[Message] Failed to clone attachment ${attEnt.userFilename} to message ${message.id}`);
+					throw new HTTPError("Failed to process attachment: " + (await cloneResponse.text()), 500);
+				}
+
+				const cloneRespBody = (await cloneResponse.json()) as { success: boolean; new_path: string };
+
+				const realAtt = Attachment.create({
+					filename: attEnt.userFilename,
+					url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`,
+					proxy_url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`,
+					size: attEnt.size,
+					height: attEnt.height,
+					width: attEnt.width,
+					content_type: attEnt.contentType || attEnt.userOriginalContentType,
+				});
+				await realAtt.save();
+				return { attachment: realAtt, index: att.index };
+			}),
+		);
+		console.log("[Message] Processed attachments for message", message.id, ":", message.attachments);
+
+		for (const att of uploadedAttachments) {
+			message.attachments![att.index] = att.attachment;
+		}
+	} else console.log("[Message] No cloud attachments to process for message", message.id, ":", message.attachments);
+
+	console.log("opts:", opts.attachments, "\nmessage:", message.attachments);
+
+	if (message.content && message.content.length > Config.get().limits.message.maxCharacters) {
 		throw new HTTPError("Content length over max character limit");
 	}
 
@@ -138,28 +192,15 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 		}
 		if (opts.avatar_url) {
 			const avatarData = await fetch(opts.avatar_url);
-			const base64 = await avatarData
+			const base64 = await avatarData.buffer().then((x) => x.toString("base64"));
-				.buffer()
-				.then((x) => x.toString("base64"));
 
-			const dataUri =
+			const dataUri = "data:" + avatarData.headers.get("content-type") + ";base64," + base64;
-				"data:" +
-				avatarData.headers.get("content-type") +
-				";base64," +
-				base64;
 
-			message.avatar = await handleFile(
+			message.avatar = await handleFile(`/avatars/${opts.webhook_id}`, dataUri as string);
-				`/avatars/${opts.webhook_id}`,
-				dataUri as string,
-			);
 			message.author.avatar = message.avatar;
 		}
 	} else {
-		permission = await getPermission(
+		permission = await getPermission(opts.author_id, channel.guild_id, opts.channel_id);
-			opts.author_id,
-			channel.guild_id,
-			opts.channel_id,
-		);
 		permission.hasThrow("SEND_MESSAGES");
 		if (permission.cache.member) {
 			message.member = permission.cache.member;
@@ -173,20 +214,12 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 				const guild = await Guild.findOneOrFail({
 					where: { id: channel.guild_id },
 				});
-				if (!opts.message_reference.guild_id)
+				if (!opts.message_reference.guild_id) opts.message_reference.guild_id = channel.guild_id;
-					opts.message_reference.guild_id = channel.guild_id;
+				if (!opts.message_reference.channel_id) opts.message_reference.channel_id = opts.channel_id;
-				if (!opts.message_reference.channel_id)
-					opts.message_reference.channel_id = opts.channel_id;
 
 				if (!guild.features.includes("CROSS_CHANNEL_REPLIES")) {
-					if (opts.message_reference.guild_id !== channel.guild_id)
+					if (opts.message_reference.guild_id !== channel.guild_id) throw new HTTPError("You can only reference messages from this guild");
-						throw new HTTPError(
+					if (opts.message_reference.channel_id !== opts.channel_id) throw new HTTPError("You can only reference messages from this channel");
-							"You can only reference messages from this guild",
-						);
-					if (opts.message_reference.channel_id !== opts.channel_id)
-						throw new HTTPError(
-							"You can only reference messages from this channel",
-						);
 				}
 
 				message.message_reference = opts.message_reference;
@@ -198,15 +231,7 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 	}
 
 	// TODO: stickers/activity
-	if (
+	if (!allow_empty && !opts.content && !opts.embeds?.length && !opts.attachments?.length && !opts.sticker_ids?.length && !opts.poll && !opts.components?.length) {
-		!allow_empty &&
-		!opts.content &&
-		!opts.embeds?.length &&
-		!opts.attachments?.length &&
-		!opts.sticker_ids?.length &&
-		!opts.poll &&
-		!opts.components?.length
-	) {
 		console.log("[Message] Rejecting empty message:", opts, message);
 		throw new HTTPError("Empty messages are not allowed", 50006);
 	}
@@ -230,31 +255,22 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 		}*/
 
 		for (const [, mention] of content.matchAll(USER_MENTION)) {
-			if (!mention_user_ids.includes(mention))
+			if (!mention_user_ids.includes(mention)) mention_user_ids.push(mention);
-				mention_user_ids.push(mention);
 		}
 
 		await Promise.all(
-			Array.from(content.matchAll(ROLE_MENTION)).map(
+			Array.from(content.matchAll(ROLE_MENTION)).map(async ([, mention]) => {
-				async ([, mention]) => {
+				const role = await Role.findOneOrFail({
-					const role = await Role.findOneOrFail({
+					where: { id: mention, guild_id: channel.guild_id },
-						where: { id: mention, guild_id: channel.guild_id },
+				});
-					});
+				if (role.mentionable || opts.webhook_id || permission?.has("MANAGE_ROLES")) {
-					if (
+					mention_role_ids.push(mention);
-						role.mentionable ||
+				}
-						opts.webhook_id ||
+			}),
-						permission?.has("MANAGE_ROLES")
-					) {
-						mention_role_ids.push(mention);
-					}
-				},
-			),
 		);
 
 		if (opts.webhook_id || permission?.has("MENTION_EVERYONE")) {
-			mention_everyone =
+			mention_everyone = !!content.match(EVERYONE_MENTION) || !!content.match(HERE_MENTION);
-				!!content.match(EVERYONE_MENTION) ||
-				!!content.match(HERE_MENTION);
 		}
 	}
 
@@ -265,10 +281,7 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 				channel_id: message.channel_id,
 			},
 		});
-		if (
+		if (referencedMessage && referencedMessage.author_id !== message.author_id) {
-			referencedMessage &&
-			referencedMessage.author_id !== message.author_id
-		) {
 			message.mentions.push(
 				User.create({
 					id: referencedMessage.author_id,
@@ -282,46 +295,12 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 		Channel.create({ id: x }),
 	);*/
 	message.mention_roles = mention_role_ids.map((x) => Role.create({ id: x }));
-	message.mentions = [
+	message.mentions = [...message.mentions, ...mention_user_ids.map((x) => User.create({ id: x }))];
-		...message.mentions,
-		...mention_user_ids.map((x) => User.create({ id: x })),
-	];
 
 	message.mention_everyone = mention_everyone;
 
 	// TODO: check and put it all in the body
 
-	if(message.attachments?.some(att => "uploaded_filename" in att)) {
-		message.attachments = await Promise.all(message.attachments.map(async att => {
-			if("uploaded_filename" in att) {
-				const cloneResponse = await fetch(`${Config.get().cdn.endpointPrivate}/attachments/${att.uploaded_filename}/clone_to_message/${message.id}`, {
-					method: "POST",
-					headers: {
-						"X-Signature": Config.get().security.requestSignature || "",
-					},
-				});
-				if(!cloneResponse.ok) {
-					console.error(`[Message] Failed to clone attachment ${att.uploaded_filename} to message ${message.id}`);
-					throw new HTTPError("Failed to process attachment: " + await cloneResponse.text(), 500);
-				}
-
-				const cloneRespBody = await cloneResponse.json() as { success: boolean, new_path: string };
-
-				return Attachment.create({
-					id: att.id,
-					filename: att.filename,
-					url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`,
-					proxy_url: `${Config.get().cdn.endpointPublic}/${cloneRespBody.new_path}`,
-					size: att.size,
-					height: att.height,
-					width: att.width,
-					content_type: att.content_type,
-				});
-			}
-			return att;
-		}));
-	}
-
 	return message;
 }
 
@@ -383,10 +362,7 @@ export async function postHandleMessage(message: Message) {
 				channel_id: message.channel_id,
 				data,
 			} as MessageUpdateEvent),
-			Message.update(
+			Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds }),
-				{ id: message.id, channel_id: message.channel_id },
-				{ embeds: data.embeds },
-			),
 		]);
 		return;
 	}
@@ -415,12 +391,8 @@ export async function postHandleMessage(message: Message) {
 		}
 
 		// bit gross, but whatever!
-		const endpointPublic =
+		const endpointPublic = Config.get().cdn.endpointPublic || "http://127.0.0.1"; // lol
-			Config.get().cdn.endpointPublic || "http://127.0.0.1"; // lol
+		const handler = url.hostname === new URL(endpointPublic).hostname ? EmbedHandlers["self"] : EmbedHandlers[url.hostname] || EmbedHandlers["default"];
-		const handler =
-			url.hostname === new URL(endpointPublic).hostname
-				? EmbedHandlers["self"]
-				: EmbedHandlers[url.hostname] || EmbedHandlers["default"];
 
 		try {
 			let res = await handler(url);
@@ -438,10 +410,7 @@ export async function postHandleMessage(message: Message) {
 				data.embeds.push(embed);
 			}
 		} catch (e) {
-			console.error(
+			console.error(`[Embeds] Error while generating embed for ${link}`, e);
-				`[Embeds] Error while generating embed for ${link}`,
-				e,
-			);
 			Sentry.captureException(e, (scope) => {
 				scope.clear();
 				scope.setContext("request", { url: link });
@@ -457,10 +426,7 @@ export async function postHandleMessage(message: Message) {
 			channel_id: message.channel_id,
 			data,
 		} as MessageUpdateEvent),
-		Message.update(
+		Message.update({ id: message.id, channel_id: message.channel_id }, { embeds: data.embeds }),
-			{ id: message.id, channel_id: message.channel_id },
-			{ embeds: data.embeds },
-		),
 		...cachePromises,
 	]);
 }
@@ -478,9 +444,7 @@ export async function sendMessage(opts: MessageOptions) {
 	]);
 
 	// no await as it should catch error non-blockingly
-	postHandleMessage(message).catch((e) =>
+	postHandleMessage(message).catch((e) => console.error("[Message] post-message handler failed", e));
-		console.error("[Message] post-message handler failed", e),
-	);
 
 	return message;
 }
@@ -495,7 +459,7 @@ interface MessageOptions extends MessageCreateSchema {
 	embeds?: Embed[];
 	reactions?: Reaction[];
 	channel_id?: string;
-	attachments?: Attachment[]; // why are we masking this?
+	attachments?: (MessageCreateAttachment | MessageCreateCloudAttachment | Attachment)[]; // why are we masking this?
 	edited_timestamp?: Date;
 	timestamp?: Date;
 	username?: string;

src/cdn/routes/attachments.ts

@@ -150,7 +150,14 @@ router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("fil
 		const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`;
 
 		await storage.set(path, buffer);
-		if (att.userOriginalContentType?.includes("image")) {
+
+		let mimeType = att.userOriginalContentType;
+		if (att.userOriginalContentType === null) {
+			const ft = await FileType.fromBuffer(buffer);
+			mimeType = att.contentType = ft?.mime || "application/octet-stream";
+		}
+
+		if (mimeType?.includes("image")) {
 			const dimensions = imageSize(buffer);
 			if (dimensions) {
 				att.width = dimensions.width;
@@ -168,6 +175,7 @@ router.put("/:channel_id/:batch_id/:attachment_id/:filename", multer.single("fil
 
 router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Request, res: Response) => {
 	if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature");
+	console.log("[Cloud Delete] Deleting attachment", req.params);
 
 	const { channel_id, batch_id, attachment_id, filename } = req.params;
 	const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`;
@@ -191,10 +199,11 @@ router.delete("/:channel_id/:batch_id/:attachment_id/:filename", async (req: Req
 
 router.post("/:channel_id/:batch_id/:attachment_id/:filename/clone_to_message/:message_id", async (req: Request, res: Response) => {
 	if (req.headers.signature !== Config.get().security.requestSignature) throw new HTTPError("Invalid request signature");
+	console.log("[Cloud Clone] Cloning attachment to message", req.params);
 
-	const { channel_id, batch_id, attachment_id, filename } = req.params;
+	const { channel_id, batch_id, attachment_id, filename, message_id } = req.params;
 	const path = `attachments/${channel_id}/${batch_id}/${attachment_id}/${filename}`;
-	const newPath = `attachments/${channel_id}/${filename}`;
+	const newPath = `attachments/${channel_id}/${message_id}/${filename}`;
 
 	const att = await CloudAttachment.findOne({
 		where: {