Merge branch 'fosscord:master' into master

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '25 10 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

package.json

@@ -29,7 +29,7 @@
 	},
 	"homepage": "https://github.com/fosscord/fosscord-api#readme",
 	"dependencies": {
-		"@fosscord/server-util": "^1.3.5",
+		"@fosscord/server-util": "^1.3.9",
 		"@types/jest": "^26.0.22",
 		"@types/json-schema": "^7.0.7",
 		"ajv": "^8.4.0",
@@ -38,6 +38,7 @@
 		"atomically": "^1.7.0",
 		"bcrypt": "^5.0.1",
 		"body-parser": "^1.19.0",
+		"cheerio": "^1.0.0-rc.9",
 		"dot-prop": "^6.0.1",
 		"dotenv": "^8.2.0",
 		"env-paths": "^2.2.1",

src/routes/channels/#channel_id/messages/#message_id/index.ts

@@ -4,7 +4,7 @@ import { HTTPError } from "lambert-server";
 import { MessageCreateSchema } from "../../../../../schema/Message";
 import { emitEvent } from "../../../../../util/Event";
 import { check } from "../../../../../util/instanceOf";
-import { handleMessage } from "../../../../../util/Message";
+import { handleMessage, postHandleMessage } from "../../../../../util/Message";
 
 const router = Router();
 
@@ -40,6 +40,8 @@ router.patch("/", check(MessageCreateSchema), async (req, res) => {
 		data: { ...toObject(message), nonce: undefined }
 	} as MessageUpdateEvent);
 
+	postHandleMessage(message);
+
 	return res.json(toObject(message));
 });
 

src/util/Message.ts

@@ -1,14 +1,28 @@
-import { ChannelModel, MessageCreateEvent } from "@fosscord/server-util";
+import { ChannelModel, Embed, Message, MessageCreateEvent, MessageUpdateEvent } from "@fosscord/server-util";
 import { Snowflake } from "@fosscord/server-util";
 import { MessageModel } from "@fosscord/server-util";
 import { PublicMemberProjection } from "@fosscord/server-util";
 import { toObject } from "@fosscord/server-util";
 import { getPermission } from "@fosscord/server-util";
-import { Message } from "@fosscord/server-util";
 import { HTTPError } from "lambert-server";
+import fetch from "node-fetch";
+import cheerio from "cheerio";
 import { emitEvent } from "./Event";
 // TODO: check webhook, application, system author
 
+const LINK_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/g;
+
+const DEFAULT_FETCH_OPTIONS: any = {
+	redirect: "follow",
+	follow: 1,
+	headers: {
+		"user-agent": "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"
+	},
+	size: 1024 * 1024 * 1,
+	compress: true,
+	method: "GET"
+};
+
 export async function handleMessage(opts: Partial<Message>) {
 	const channel = await ChannelModel.findOne({ id: opts.channel_id }, { guild_id: true, type: true, permission_overwrites: true }).exec();
 	if (!channel || !opts.channel_id) throw new HTTPError("Channel not found", 404);
@@ -43,6 +57,60 @@ export async function handleMessage(opts: Partial<Message>) {
 	};
 }
 
+// TODO: cache link result in db
+export async function postHandleMessage(message: Message) {
+	var links = message.content?.match(LINK_REGEX);
+	if (!links) return;
+
+	const data = { ...message };
+	data.embeds = data.embeds.filter((x) => x.type !== "link");
+
+	links = links.slice(0, 5); // embed max 5 links
+
+	for (const link of links) {
+		try {
+			const request = await fetch(link, DEFAULT_FETCH_OPTIONS);
+
+			const text = await request.text();
+			const $ = cheerio.load(text);
+
+			const title = $('meta[property="og:title"]').attr("content");
+			const provider_name = $('meta[property="og:site_name"]').text();
+			const author_name = $('meta[property="article:author"]').attr("content");
+			const description = $('meta[property="og:description"]').attr("content") || $('meta[property="description"]').attr("content");
+			const image = $('meta[property="og:image"]').attr("content");
+			const url = $('meta[property="og:url"]').attr("content");
+			// TODO: color
+			const embed: Embed = {
+				provider: {
+					url: link,
+					name: provider_name
+				}
+			};
+
+			if (author_name) embed.author = { name: author_name };
+			if (image) embed.thumbnail = { proxy_url: image, url: image };
+			if (title) embed.title = title;
+			if (url) embed.url = url;
+			if (description) embed.description = description;
+
+			if (title || description) {
+				data.embeds.push(embed);
+			}
+		} catch (error) {}
+	}
+
+	await Promise.all([
+		emitEvent({
+			event: "MESSAGE_UPDATE",
+			guild_id: message.guild_id,
+			channel_id: message.channel_id,
+			data
+		} as MessageUpdateEvent),
+		MessageModel.updateOne({ id: message.id, channel_id: message.channel_id }, data).exec()
+	]);
+}
+
 export async function sendMessage(opts: Partial<Message>) {
 	const message = await handleMessage({ ...opts, id: Snowflake.generate(), timestamp: new Date() });
 
@@ -50,5 +118,7 @@ export async function sendMessage(opts: Partial<Message>) {
 
 	await emitEvent({ event: "MESSAGE_CREATE", channel_id: opts.channel_id, data, guild_id: message.guild_id } as MessageCreateEvent);
 
+	postHandleMessage(data); // no await as it shouldnt block the message send function
+
 	return data;
 }