Merge pull request #553 from TheArcaneBrony/master

Remove fs-extras, add Sentry support, fix compile errors, fix some typos
2021-12-26 13:38:25 -05:00 · 2021-12-26 13:38:25 -05:00 · 78b33a8b7a
commit 78b33a8b7a
parent 6136cd85f2 07768ecb68
22 changed files with 222 additions and 38 deletions
--- a/.gitignore
+++ b/.gitignore
@ -2,8 +2,6 @@
 db/
 dist/
 node_modules
-api/assets/*.js
-api/assets/*.css
 database.db
 tsconfig.tsbuildinfo
 files/
@ -11,3 +9,4 @@ files/
 config.json

 .vscode/settings.json
+api/assets/plugins/*.js
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -1,4 +0,0 @@
-{
-    "awooga.originalColorCustomizations": {},
-    "workbench.colorCustomizations": {}
-}
--- a/api/assets/fosscord-login.css
+++ b/api/assets/fosscord-login.css
@ -0,0 +1,69 @@
+/* replace tos acceptance popup */
+#app-mount > div:nth-child(7) > div > div > div.tooltipContent-bqVLWK {
+    visibility: hidden;
+}
+#app-mount > div:nth-child(7) > div > div > div.tooltipContent-bqVLWK::after{
+    visibility: visible;
+    display: block;
+    content: "You need to agree to this instance's rules to continue";
+    margin-top: -32px;
+}
+/* replace login header */
+#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.mainLoginContainer-1ddwnR > h3 {
+	visibility: hidden;
+}
+h3.title-jXR8lp.marginBottom8-AtZOdT.base-1x0h_U.size24-RIRrxO::after {
+	margin-top: -32px;
+	content: "Welcome to Fosscord!";
+	visibility: visible;
+	display: block;
+}
+
+/* Logo in top left when bg removed */
+#app-mount > div.app-1q1i1E > div > a {
+	/* replace me: original dimensions: 130x36 */
+	background: url(https://raw.githubusercontent.com/fosscord/fosscord/9900329e5ef2c17bdeb6893e04c0511f72027f97/assets/logo/temp.svg);
+	background-size: contain;
+	width: 128px;
+	height: 128px;
+	border-radius: 50%;
+}
+
+/* replace TOS text */
+
+#app-mount
+	> div.app-1q1i1E
+	> div
+	> div
+	> div
+	> form
+	> div
+	> div
+	> div.flex-1xMQg5.flex-1O1GKY.horizontal-1ae9ci.horizontal-2EEEnY.flex-1O1GKY.directionRow-3v3tfG.justifyStart-2NDFzi.alignCenter-1dQNNs.noWrap-3jynv6.marginTop20-3TxNs6
+	> label
+	> div.label-cywgfr.labelClickable-11AuB8.labelForward-1wfipV
+	> * {
+	visibility: hidden;
+}
+
+#app-mount
+	> div.app-1q1i1E
+	> div
+	> div
+	> div
+	> form
+	> div
+	> div
+	> div.flex-1xMQg5.flex-1O1GKY.horizontal-1ae9ci.horizontal-2EEEnY.flex-1O1GKY.directionRow-3v3tfG.justifyStart-2NDFzi.alignCenter-1dQNNs.noWrap-3jynv6.marginTop20-3TxNs6
+	> label
+	> div.label-cywgfr.labelClickable-11AuB8.labelForward-1wfipV::after {
+	visibility: visible;
+	content: "I have read and agree with the rules set by this instance.";
+	display: block;
+	margin-top: -16px;
+}
+
+/* shrink login box to same size as register */
+.authBoxExpanded-2jqaBe {
+	width: 480px !important;
+}
--- a/api/assets/fosscord.css
+++ b/api/assets/fosscord.css
@ -0,0 +1,42 @@
+
+/* loading spinner */
+#app-mount > div.app-1q1i1E > div.container-16j22k.fixClipping-3qAKRb > div.content-1-zrf2 > video {
+	filter: opacity(1);
+	background: url("http://www.clipartbest.com/cliparts/7ca/6Rr/7ca6RrLAi.gif");
+	background-size: contain;
+	/* width: 64px;
+    height: 64px; */
+	padding-bottom: 64px;
+	background-repeat: no-repeat;
+}
+
+/* home button icon */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div.tutorialContainer-2sGCg9 > div > div.listItemWrapper-KhRmzM > div > svg > foreignObject > div > div
+{
+    background-image: url(https://raw.githubusercontent.com/fosscord/fosscord/9900329e5ef2c17bdeb6893e04c0511f72027f97/assets/logo/temp.svg);
+    background-size: contain;
+    border-radius: 50%;
+}
+/* Login QR */
+#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.transitionGroup-aR7y1d.qrLogin-1AOZMt,
+#app-mount > div.app-1q1i1E > div > div > div > div > form > div > div > div.verticalSeparator-3huAjp,
+/* Remove login bg */
+#app-mount > div.app-1q1i1E > div > svg,
+/* Download bar */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.notice-3bPHh-.colorDefault-22HBa0,
+/* Connection problem links */
+#app-mount > div.app-1q1i1E > div.container-16j22k.fixClipping-3qAKRb > div.problems-3mgf6w.slideIn-sCvzGz > div:nth-child(2),
+/* Downloads button */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div:nth-child(7) > div.listItemWrapper-KhRmzM > div > svg > foreignObject > div,
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div:nth-child(6) > div,
+/* help button */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.content-98HsJk > div.chat-3bRxxu > section > div.toolbar-1t6TWx > a,
+/* download button start of guild */
+#chat-messages-899316648933185083 > div > div > div:nth-child(5),
+/* Thread permissions etc popups */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > div > div.content-98HsJk > div.sidebar-2K8pFh.hasNotice-1XRy4h > nav > div.container-3O_wAf,
+/* home button icon */
+#app-mount > div.app-1q1i1E > div > div.layers-3iHuyZ.layers-3q14ss > div > div > nav > ul > div.scroller-1Bvpku.none-2Eo-qx.scrollerBase-289Jih > div.tutorialContainer-2sGCg9 > div > div.listItemWrapper-KhRmzM > div > svg > foreignObject > div > div > svg
+{
+    display: none;
+}
--- a/api/package.json
+++ b/api/package.json
@ -65,6 +65,8 @@
 		"@babel/preset-env": "^7.15.8",
 		"@babel/preset-typescript": "^7.15.0",
 		"@fosscord/util": "file:../util",
+		"@sentry/node": "^6.16.1",
+		"@sentry/tracing": "^6.16.1",
 		"ajv": "8.6.2",
 		"ajv-formats": "^2.1.1",
 		"amqplib": "^0.8.0",
@ -84,9 +86,9 @@
 		"missing-native-js-functions": "^1.2.18",
 		"morgan": "^1.10.0",
 		"multer": "^1.4.2",
-		"nanocolors": "^0.2.13",
 		"node-fetch": "^2.6.1",
 		"patch-package": "^6.4.7",
+		"picocolors": "^1.0.0",
 		"proxy-agent": "^5.0.0",
 		"supertest": "^6.1.6",
 		"typeorm": "^0.2.37"
--- a/api/src/Server.ts
+++ b/api/src/Server.ts
@ -12,7 +12,7 @@ import { initTranslation } from "./middlewares/Translation";
 import morgan from "morgan";
 import { initInstance } from "./util/Instance";
 import { registerRoutes } from "@fosscord/util";
-import { red } from "nanocolors"
+import { red } from "picocolors"

 export interface FosscordServerOptions extends ServerOptions {}

--- a/api/src/routes/channels/#channel_id/messages/index.ts
+++ b/api/src/routes/channels/#channel_id/messages/index.ts
@ -107,7 +107,7 @@ router.get("/", async (req: Request, res: Response) => {
 	const endpoint = Config.get().cdn.endpointPublic;

 	return res.json(
-		messages.map((x) => {
+		messages.map((x: any) => {
 			(x.reactions || []).forEach((x: any) => {
 				// @ts-ignore
 				if ((x.user_ids || []).includes(req.user_id)) x.me = true;
@ -116,10 +116,10 @@ router.get("/", async (req: Request, res: Response) => {
 			});
 			// @ts-ignore
 			if (!x.author) x.author = { discriminator: "0000", username: "Deleted User", public_flags: "0", avatar: null };
-			x.attachments?.forEach((x) => {
+			x.attachments?.forEach((y: any) => {
 				// dynamically set attachment proxy_url in case the endpoint changed
-				const uri = x.proxy_url.startsWith("http") ? x.proxy_url : `https://example.org${x.proxy_url}`;
-				x.proxy_url = `${endpoint == null ? "" : endpoint}${new URL(uri).pathname}`;
+				const uri = y.proxy_url.startsWith("http") ? y.proxy_url : `https://example.org${y.proxy_url}`;
+				y.proxy_url = `${endpoint == null ? "" : endpoint}${new URL(uri).pathname}`;
 			});

 			return x;
--- a/bundle/package-lock.json
+++ b/bundle/package-lock.json
--- a/bundle/package.json
+++ b/bundle/package.json
@ -59,6 +59,11 @@
 		"@aws-sdk/node-http-handler": "^3.36.0",
 		"@babel/preset-env": "^7.15.8",
 		"@babel/preset-typescript": "^7.15.0",
+		"@fosscord/api": "file:../api",
+		"@fosscord/cdn": "file:../cdn",
+		"@fosscord/gateway": "file:../gateway",
+		"@sentry/node": "^6.16.1",
+		"@sentry/tracing": "^6.16.1",
 		"ajv": "8.6.2",
 		"ajv-formats": "^2.1.1",
 		"amqplib": "^0.8.0",
@ -86,18 +91,20 @@
 		"missing-native-js-functions": "^1.2.18",
 		"morgan": "^1.10.0",
 		"multer": "^1.4.2",
-		"nanocolors": "^0.2.12",
 		"node-fetch": "^2.6.1",
 		"node-os-utils": "^1.3.5",
 		"patch-package": "^6.4.7",
 		"pg": "^8.7.1",
+		"picocolors": "^1.0.0",
 		"proxy-agent": "^5.0.0",
 		"reflect-metadata": "^0.1.13",
 		"sqlite3": "^5.0.2",
 		"supertest": "^6.1.6",
+		"tslib": "^2.3.1",
 		"typeorm": "^0.2.37",
 		"typescript": "^4.1.2",
 		"typescript-json-schema": "^0.50.1",
-		"ws": "^7.4.2"
+		"ws": "^7.4.2",
+		"nanocolors": "^0.2.12"
 	}
 }
--- a/bundle/scripts/build.js
+++ b/bundle/scripts/build.js
@ -1,34 +1,58 @@
 const { execSync } = require("child_process");
 const path = require("path");
-const fse = require("fs-extra");
+const fs = require("fs");
 const { getSystemErrorMap } = require("util");
 const { argv } = require("process");

+var steps = 2, i = 0;
+if (argv.includes("clean")) steps++;
+if (argv.includes("copyonly")) steps--;
 const dirs = ["api", "util", "cdn", "gateway", "bundle"];

 const verbose = argv.includes("verbose") || argv.includes("v");

+var copyRecursiveSync = function(src, dest) {
+	if(verbose) console.log(`cpsync: ${src} -> ${dest}`);
+	var exists = fs.existsSync(src);
+	if(!exists){
+		console.log(src + " doesn't exist, not copying!");
+		return;
+	}
+	var stats = exists && fs.statSync(src);
+	var isDirectory = exists && stats.isDirectory();
+	if (isDirectory) {
+	  fs.mkdirSync(dest, {recursive: true});
+	  fs.readdirSync(src).forEach(function(childItemName) {
+		copyRecursiveSync(path.join(src, childItemName),
+						  path.join(dest, childItemName));
+	  });
+	} else {
+	  fs.copyFileSync(src, dest);
+	}
+  };
+
 if (argv.includes("clean")) {
+	console.log(`[${++i}/${steps}] Cleaning...`);
 	dirs.forEach((a) => {
 		var d = "../" + a + "/dist";
-		if (fse.existsSync(d)) {
-			fse.rmSync(d, { recursive: true });
+		if (fs.existsSync(d)) {
+			fs.rmSync(d, { recursive: true });
 			if (verbose) console.log(`Deleted ${d}!`);
 		}
 	});
 }

-fse.copySync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
-fse.copySync(path.join(__dirname, "..", "..", "api", "client_test"), path.join(__dirname, "..", "dist", "api", "client_test"));
-fse.copySync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
+console.log(`[${++i}/${steps}] Copying src files...`);
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "client_test"), path.join(__dirname, "..", "dist", "api", "client_test"));
+copyRecursiveSync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
 dirs.forEach((a) => {
-	fse.copySync("../" + a + "/src", "dist/" + a + "/src");
+	copyRecursiveSync("../" + a + "/src", "dist/" + a + "/src");
 	if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`);
 });

-console.log("[1/2] Copying src files done");
 if (!argv.includes("copyonly")) {
-	console.log("[2/2] Compiling src files ...");
+	console.log(`[${++i}/${steps}] Compiling src files ...`);

 	console.log(
 		execSync(
@ -46,3 +70,4 @@ if (!argv.includes("copyonly")) {
 		)
 	);
 }
+
--- a/bundle/src/Server.ts
+++ b/bundle/src/Server.ts
@ -6,8 +6,10 @@ import * as Api from "@fosscord/api";
 import * as Gateway from "@fosscord/gateway";
 import { CDNServer } from "@fosscord/cdn";
 import express from "express";
-import { green, bold } from "nanocolors";
+import { green, bold, yellow } from "picocolors";
 import { Config, initDatabase } from "@fosscord/util";
+import * as Sentry from "@sentry/node";
+import * as Tracing from "@sentry/tracing";

 const app = express();
 const server = http.createServer();
@ -64,7 +66,32 @@ async function main() {
 		// },
 	} as any);

+	//Sentry
+	if (Config.get().sentry.enabled) {
+		console.log(
+			`[Bundle] ${yellow("You are using Sentry! This may slightly impact performance on large loads!")}`
+		);
+		Sentry.init({
+			dsn: Config.get().sentry.endpoint,
+			integrations: [
+				new Sentry.Integrations.Http({ tracing: true }),
+				new Tracing.Integrations.Express({ app }),
+			],
+			tracesSampleRate: Config.get().sentry.traceSampleRate,
+			environment: Config.get().sentry.environment
+		});
+
+		app.use(Sentry.Handlers.requestHandler());
+		app.use(Sentry.Handlers.tracingHandler());
+	}
 	await Promise.all([api.start(), cdn.start(), gateway.start()]);
+	if (Config.get().sentry.enabled) {
+		app.use(Sentry.Handlers.errorHandler());
+		app.use(function onError(err: any, req: any, res: any, next: any) {
+			res.statusCode = 500;
+			res.end(res.sentry + "\n");
+		});
+	}
 	console.log(`[Server] ${green(`listening on port ${bold(port)}`)}`);
 }

--- a/bundle/src/start.ts
+++ b/bundle/src/start.ts
@ -2,7 +2,7 @@
 import "reflect-metadata";
 import cluster, { Worker } from "cluster";
 import os from "os";
-import { red, bold, yellow, cyan } from "nanocolors";
+import { red, bold, yellow, cyan } from "picocolors";
 import { initStats } from "./stats";
 import { config } from "dotenv";
 config();
@ -43,7 +43,7 @@ Commit Hash: ${
 				? `${cyan(commit)} (${yellow(commit.slice(0, 7))})`
 				: "Unknown (Git cannot be found)"
 		}
-Cores: ${cyan(cores)}
+Cores: ${cyan(os.cpus().length)} (Using ${cores} thread(s).)
 `)
 	);

--- a/bundle/src/stats.ts
+++ b/bundle/src/stats.ts
@ -1,6 +1,6 @@
 import os from "os";
 import osu from "node-os-utils";
-import { red } from "nanocolors";
+import { red } from "picocolors";

 export function initStats() {
 	console.log(`[Path] running in ${__dirname}`);
--- a/bundle/tsconfig.json
+++ b/bundle/tsconfig.json
@ -45,7 +45,7 @@
 		// "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */

 		/* Module Resolution Options */
-		// "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+		"moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
 		// "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
 		// "typeRoots": [],                       /* List of folders to include type definitions from. */
 		"types": [
@ -79,6 +79,8 @@
 			"@fosscord/cdn": ["cdn/src/index"],
 			"@fosscord/util": ["util/src/index"]
 		},
-		"plugins": [{ "transform": "@zerollup/ts-transform-paths" }]
+		"plugins": [{ "transform": "@zerollup/ts-transform-paths" }],
+		"noEmitHelpers": true,
+		"importHelpers": true
 	}
 }
--- a/cdn/package-lock.json
+++ b/cdn/package-lock.json
--- a/cdn/src/util/Storage.ts
+++ b/cdn/src/util/Storage.ts
@ -1,7 +1,7 @@
 import { FileStorage } from "./FileStorage";
 import path from "path";
 import fse from "fs-extra";
-import { bgCyan, black } from "nanocolors";
+import { bgCyan, black } from "picocolors";
 import { S3 } from "@aws-sdk/client-s3";
 import { S3Storage } from "./S3Storage";
 process.cwd();
--- a/cdn/tests/cdn_endpoints.test.js
+++ b/cdn/tests/cdn_endpoints.test.js
@ -1,6 +1,6 @@
 const dotenv = require("dotenv");
 const path = require("path");
-const fse = require("fs-extra");
+const fs = require("fs");
 dotenv.config();

 // TODO: write unittest to check if FileStorage.ts is working
@ -21,7 +21,7 @@ if (process.env.STORAGE_PROVIDER === "file") {
 	} else {
 		process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/");
 	}
-	fse.ensureDirSync(process.env.STORAGE_LOCATION);
+	if(!fs.existsSync(process.env.STORAGE_LOCATION)) fs.mkdirSync(process.env.STORAGE_LOCATION, {recursive:true});
 }
 const { CDNServer } = require("../dist/Server");
 const { Config } = require("@fosscord/util");
--- a/util/package.json
+++ b/util/package.json
@ -39,16 +39,15 @@
 	},
 	"dependencies": {
 		"amqplib": "^0.8.0",
-		"better-sqlite3": "^7.4.3",
 		"form-data": "^4.0.0",
 		"jsonwebtoken": "^8.5.1",
 		"lambert-server": "^1.2.12",
 		"missing-native-js-functions": "^1.2.18",
 		"multer": "^1.4.3",
-		"nanocolors": "^0.2.12",
 		"node-fetch": "^2.6.1",
 		"patch-package": "^6.4.7",
 		"pg": "^8.7.1",
+		"picocolors": "^1.0.0",
 		"proxy-agent": "^5.0.0",
 		"reflect-metadata": "^0.1.13",
 		"typeorm": "^0.2.38",
--- a/util/src/entities/BaseClass.ts
+++ b/util/src/entities/BaseClass.ts
@ -52,12 +52,12 @@ export class BaseClassWithoutId extends BaseEntity {

 	static increment<T extends BaseClass>(conditions: FindConditions<T>, propertyPath: string, value: number | string) {
 		const repository = this.getRepository();
-		return repository.increment(conditions, propertyPath, value);
+		return repository.increment(conditions as T, propertyPath, value);
 	}

 	static decrement<T extends BaseClass>(conditions: FindConditions<T>, propertyPath: string, value: number | string) {
 		const repository = this.getRepository();
-		return repository.decrement(conditions, propertyPath, value);
+		return repository.decrement(conditions as T, propertyPath, value);
 	}
 }

--- a/util/src/entities/Channel.ts
+++ b/util/src/entities/Channel.ts
@ -203,7 +203,7 @@ export class Channel extends BaseClass {

 	static async createDMChannel(recipients: string[], creator_user_id: string, name?: string) {
 		recipients = recipients.unique().filter((x) => x !== creator_user_id);
-		const otherRecipientsUsers = await User.find({ where: recipients.map((x) => ({ id: x })), select: ["id"] });
+		const otherRecipientsUsers = await User.find({ where: recipients.map((x) => ({ id: x })) });

 		// TODO: check config for max number of recipients
 		if (otherRecipientsUsers.length !== recipients.length) {
--- a/util/src/entities/Config.ts
+++ b/util/src/entities/Config.ts
@ -3,6 +3,7 @@ import { BaseClassWithoutId, PrimaryIdColumn } from "./BaseClass";
 import crypto from "crypto";
 import { Snowflake } from "../util/Snowflake";
 import { SessionsReplace } from "..";
+import { hostname } from "os";

@Entity("config")
 export class ConfigEntity extends BaseClassWithoutId {
@ -188,6 +189,12 @@ export interface ConfigValue {
 	},
 	metrics: {
 		timeout: number;
+	},
+	sentry: {
+		enabled: boolean;
+		endpoint: string;
+		traceSampleRate: number;
+		environment: string;
 	}
 }

@ -377,5 +384,11 @@ export const DefaultConfigOptions: ConfigValue = {
 	},
 	metrics: {
 		timeout: 30000
+	},
+	sentry: {
+		enabled: false,
+		endpoint: "https://05e8e3d005f34b7d97e920ae5870a5e5@sentry.thearcanebrony.net/6",
+		traceSampleRate: 1.0,
+		environment: hostname()
 	}
 };
--- a/util/src/util/Database.ts
+++ b/util/src/util/Database.ts
@ -3,7 +3,7 @@ import "reflect-metadata";
 import { Connection, createConnection } from "typeorm";
 import * as Models from "../entities";
 import { Migration } from "../entities/Migration";
-import { yellow, green } from "nanocolors";
+import { yellow, green, red } from "picocolors";

 // UUID extension option is only supported with postgres
 // We want to generate all id's with Snowflakes that's why we have our own BaseEntity class
@ -19,6 +19,9 @@ export function initDatabase(): Promise<Connection> {
 	const isSqlite = type.includes("sqlite");

 	console.log(`[Database] ${yellow(`connecting to ${type} db`)}`);
+	if(isSqlite) {
+		console.log(`[Database] ${red(`You are running sqlite! Please keep in mind that we recommend setting up a dedicated database!`)}`);
+	}
 	// @ts-ignore
 	promise = createConnection({
 		type,