diff --git a/.gitignore b/.gitignore index f67723af..a09d215a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ node_modules api/assets/*.js api/assets/*.css database.db -tsconfig.tsbuildinfo \ No newline at end of file +tsconfig.tsbuildinfo +files/ +.env \ No newline at end of file diff --git a/api/package-lock.json b/api/package-lock.json index f4e7506c..265e70bb 100644 Binary files a/api/package-lock.json and b/api/package-lock.json differ diff --git a/api/package.json b/api/package.json index 3f0315ae..aae31e2d 100644 --- a/api/package.json +++ b/api/package.json @@ -10,7 +10,7 @@ "test": "npm run build && npm run test:only", "test:watch": "jest --watch", "start": "npm run build && node dist/start", - "build": "npx tsc -b .", + "build": "npx tsc -p .", "build-docker": "tsc -p tsconfig-docker.json", "dev": "tsnd --respawn src/start.ts", "patch": "ts-patch install -s && npx patch-package", @@ -38,10 +38,6 @@ "homepage": "https://fosscord.com", "devDependencies": { "@babel/core": "^7.15.5", - "@babel/preset-env": "^7.15.6", - "@babel/preset-typescript": "^7.15.0", - "@swc/cli": "^0.1.51", - "@swc/core": "^1.2.93", "@types/amqplib": "^0.8.1", "@types/bcrypt": "^5.0.0", "@types/express": "^4.17.9", @@ -49,65 +45,46 @@ "@types/jest": "^27.0.1", "@types/jest-expect-message": "^1.0.3", "@types/jsonwebtoken": "^8.5.0", - "@types/mongodb": "^3.6.9", - "@types/mongoose": "^5.10.5", - "@types/mongoose-autopopulate": "^0.10.1", - "@types/mongoose-lean-virtuals": "^0.5.1", "@types/multer": "^1.4.5", "@types/node": "^14.17.9", "@types/node-fetch": "^2.5.7", "@types/supertest": "^2.0.11", "@zerollup/ts-transform-paths": "^1.7.18", - "0x": "^4.10.2", - "babel-jest": "^27.2.0", - "caxa": "^2.1.0", - "image-size": "^1.0.0", "jest": "^26.6.3", "jest-expect-message": "^1.0.2", "jest-runtime": "^27.2.1", - "saslprep": "^1.0.3", "ts-node": "^9.1.1", "ts-node-dev": "^1.1.6", "ts-patch": "^1.4.4", - "tsup": "^5.4.0", "typescript": "^4.4.2", - "typescript-json-schema": "0.50.1" + "typescript-json-schema": "0.50.1", + "@types/morgan": "^1.9.3" }, "dependencies": { "@fosscord/util": "file:../util", - "@types/morgan": "^1.9.3", "ajv": "8.6.2", "ajv-formats": "^2.1.1", "amqplib": "^0.8.0", "assert": "^1.5.0", - "atomically": "^1.7.0", "bcrypt": "^5.0.1", "body-parser": "^1.19.0", - "cheerio": "^1.0.0-rc.9", - "dot-prop": "^6.0.1", + "cheerio": "^1.0.0-rc.10", "dotenv": "^8.2.0", - "env-paths": "^2.2.1", - "esbuild": "^0.13.4", "express": "^4.17.1", - "express-validator": "^6.9.2", "form-data": "^3.0.0", "i18next": "^19.9.2", "i18next-http-middleware": "^3.1.3", "i18next-node-fs-backend": "^2.1.3", + "image-size": "^1.0.0", "jsonwebtoken": "^8.5.1", "lambert-server": "^1.2.11", "missing-native-js-functions": "^1.2.17", - "mongoose": "^5.12.3", - "mongoose-autopopulate": "^0.12.3", - "mongoose-long": "^0.3.2", "morgan": "^1.10.0", "multer": "^1.4.2", "node-fetch": "^2.6.1", "patch-package": "^6.4.7", "supertest": "^6.1.6", - "tsconfig-paths": "^3.11.0", - "typeorm": "^0.2.37", - "wsc": "^0.3.0" + "typeorm": "^0.2.37" }, "jest": { "setupFiles": [ diff --git a/api/src/Server.ts b/api/src/Server.ts index 12c1d6b4..1f11a295 100644 --- a/api/src/Server.ts +++ b/api/src/Server.ts @@ -1,19 +1,17 @@ -import { OptionsJson } from "body-parser"; import "missing-native-js-functions"; -import { Connection } from "mongoose"; import { Server, ServerOptions } from "lambert-server"; import { Authentication, CORS } from "./middlewares/"; import { Config, initDatabase, initEvent } from "@fosscord/util"; import { ErrorHandler } from "./middlewares/ErrorHandler"; import { BodyParser } from "./middlewares/BodyParser"; import { Router, Request, Response, NextFunction } from "express"; -import mongoose from "mongoose"; import path from "path"; import { initRateLimits } from "./middlewares/RateLimit"; import TestClient from "./middlewares/TestClient"; import { initTranslation } from "./middlewares/Translation"; import morgan from "morgan"; import { initInstance } from "./util/Instance"; +import { registerRoutes } from "@fosscord/util"; export interface FosscordServerOptions extends ServerOptions {} @@ -75,7 +73,7 @@ export class FosscordServer extends Server { await initRateLimits(api); await initTranslation(api); - this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/")); + this.routes = await registerRoutes(this, path.join(__dirname, "routes", "/")); api.use("*", (error: any, req: Request, res: Response, next: NextFunction) => { if (error) return next(error); diff --git a/api/src/routes/channels/#channel_id/invites.ts b/api/src/routes/channels/#channel_id/invites.ts index 22420983..6d2c625d 100644 --- a/api/src/routes/channels/#channel_id/invites.ts +++ b/api/src/routes/channels/#channel_id/invites.ts @@ -2,7 +2,7 @@ import { Router, Request, Response } from "express"; import { HTTPError } from "lambert-server"; import { route } from "@fosscord/api"; import { random } from "@fosscord/api"; -import { getPermission, Channel, Invite, InviteCreateEvent, emitEvent, User, Guild, PublicInviteRelation } from "@fosscord/util"; +import { Channel, Invite, InviteCreateEvent, emitEvent, User, Guild, PublicInviteRelation } from "@fosscord/util"; import { isTextChannel } from "./messages"; const router: Router = Router(); diff --git a/api/src/routes/channels/#channel_id/messages/index.ts b/api/src/routes/channels/#channel_id/messages/index.ts index 4ec31417..26bb9e5d 100644 --- a/api/src/routes/channels/#channel_id/messages/index.ts +++ b/api/src/routes/channels/#channel_id/messages/index.ts @@ -22,7 +22,7 @@ const router: Router = Router(); export default router; -function isTextChannel(type: ChannelType): boolean { +export function isTextChannel(type: ChannelType): boolean { switch (type) { case ChannelType.GUILD_STORE: case ChannelType.GUILD_VOICE: @@ -39,7 +39,6 @@ function isTextChannel(type: ChannelType): boolean { return true; } } -module.exports.isTextChannel = isTextChannel; export interface MessageCreateSchema { content?: string; diff --git a/api/src/routes/channels/#channel_id/permissions.ts b/api/src/routes/channels/#channel_id/permissions.ts index 6ebf721a..2eded853 100644 --- a/api/src/routes/channels/#channel_id/permissions.ts +++ b/api/src/routes/channels/#channel_id/permissions.ts @@ -44,8 +44,8 @@ router.put( }; channel.permission_overwrites!.push(overwrite); } - overwrite.allow = String(req.permission!.bitfield & (BigInt(body.allow) || 0n)); - overwrite.deny = String(req.permission!.bitfield & (BigInt(body.deny) || 0n)); + overwrite.allow = String(req.permission!.bitfield & (BigInt(body.allow) || BigInt("0"))); + overwrite.deny = String(req.permission!.bitfield & (BigInt(body.deny) || BigInt("0"))); await Promise.all([ channel.save(), diff --git a/api/src/routes/guilds/#guild_id/roles.ts b/api/src/routes/guilds/#guild_id/roles.ts index d1d60906..0a57c6a2 100644 --- a/api/src/routes/guilds/#guild_id/roles.ts +++ b/api/src/routes/guilds/#guild_id/roles.ts @@ -57,7 +57,7 @@ router.post("/", route({ body: "RoleModifySchema", permission: "MANAGE_ROLES" }) ...body, guild_id: guild_id, managed: false, - permissions: String(req.permission!.bitfield & (body.permissions || 0n)), + permissions: String(req.permission!.bitfield & (body.permissions || BigInt("0"))), tags: undefined }); @@ -105,7 +105,12 @@ router.patch("/:role_id", route({ body: "RoleModifySchema", permission: "MANAGE_ const { role_id, guild_id } = req.params; const body = req.body as RoleModifySchema; - const role = new Role({ ...body, id: role_id, guild_id, permissions: String(req.permission!.bitfield & (body.permissions || 0n)) }); + const role = new Role({ + ...body, + id: role_id, + guild_id, + permissions: String(req.permission!.bitfield & (body.permissions || BigInt("0"))) + }); await Promise.all([ role.save(), diff --git a/api/src/routes/guilds/templates/index.ts b/api/src/routes/guilds/templates/index.ts index b5e243e9..86316d23 100644 --- a/api/src/routes/guilds/templates/index.ts +++ b/api/src/routes/guilds/templates/index.ts @@ -47,7 +47,7 @@ router.post("/:code", route({ body: "GuildTemplateCreateSchema" }), async (req: managed: true, mentionable: true, name: "@everyone", - permissions: 2251804225n, + permissions: BigInt("2251804225"), position: 0, tags: null }).save() diff --git a/api/src/test/jwt.ts b/api/src/test/jwt.ts deleted file mode 100644 index bdad513b..00000000 --- a/api/src/test/jwt.ts +++ /dev/null @@ -1,37 +0,0 @@ -const jwa = require("jwa"); - -var STR64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".split(""); - -function base64url(string: string, encoding: string) { - // @ts-ignore - return Buffer.from(string, encoding).toString("base64").replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_"); -} - -function to64String(input: number, current = ""): string { - if (input < 0 && current.length == 0) { - input = input * -1; - } - var modify = input % 64; - var remain = Math.floor(input / 64); - var result = STR64[modify] + current; - return remain <= 0 ? result : to64String(remain, result); -} - -function to64Parse(input: string) { - var result = 0; - var toProc = input.split(""); - var e; - for (e in toProc) { - result = result * 64 + STR64.indexOf(toProc[e]); - } - return result; -} - -// @ts-ignore -const start = `${base64url("311129357362135041")}.${to64String(Date.now())}`; -const signature = jwa("HS256").sign(start, `test`); -const token = `${start}.${signature}`; -console.log(token); - -// MzExMTI5MzU3MzYyMTM1MDQx.XdQb_rA.907VgF60kocnOTl32MSUWGSSzbAytQ0jbt36KjLaxuY -// MzExMTI5MzU3MzYyMTM1MDQx.XdQbaPy.4vGx4L7IuFJGsRe6IL3BeybLIvbx4Vauvx12pwNsy2U diff --git a/api/src/test/jwt2.ts b/api/src/test/jwt2.ts deleted file mode 100644 index e231233d..00000000 --- a/api/src/test/jwt2.ts +++ /dev/null @@ -1,13 +0,0 @@ -import jwt from "jsonwebtoken"; - -const algorithm = "HS256"; -const iat = Math.floor(Date.now() / 1000); - -// @ts-ignore -const token = jwt.sign({ id: "311129357362135041" }, "secret", { - algorithm, -}); -console.log(token); - -const decoded = jwt.verify(token, "secret", { algorithms: [algorithm] }); -console.log(decoded); diff --git a/api/src/test/password_test.ts b/api/src/test/password_test.ts deleted file mode 100644 index 983b18ae..00000000 --- a/api/src/test/password_test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { checkPassword } from "@fosscord/api"; - -console.log(checkPassword("123456789012345")); -// -> 0.25 -console.log(checkPassword("ABCDEFGHIJKLMOPQ")); -// -> 0.25 -console.log(checkPassword("ABC123___...123")); -// -> -console.log(checkPassword("")); -// -> -// console.log(checkPassword("")); -// // -> diff --git a/api/tsconfig.json b/api/tsconfig.json index 0bbd615a..2cf4e4c1 100644 --- a/api/tsconfig.json +++ b/api/tsconfig.json @@ -1,10 +1,11 @@ { + "exclude": ["node_modules"], "include": ["src/**/*.ts"], "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */ /* Basic Options */ - // "incremental": true, /* Enable incremental compilation */ + "incremental": true /* Enable incremental compilation */, "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, "lib": ["ES2021"] /* Specify library files to be included in the compilation. */, @@ -69,6 +70,7 @@ "@fosscord/api": ["src/index"], "@fosscord/api/*": ["src/*"] }, - "plugins": [{ "transform": "@zerollup/ts-transform-paths" }] + "plugins": [{ "transform": "@zerollup/ts-transform-paths" }], + "experimentalDecorators": true } } diff --git a/bundle/.gitignore b/bundle/.gitignore deleted file mode 100644 index cf073d9c..00000000 --- a/bundle/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -files/ -.env \ No newline at end of file diff --git a/bundle/.vscode/launch.json b/bundle/.vscode/launch.json index 917f2a93..aa4e743a 100644 --- a/bundle/.vscode/launch.json +++ b/bundle/.vscode/launch.json @@ -8,13 +8,11 @@ "sourceMaps": true, "type": "node", "request": "launch", - "name": "Launch server bundle", - "program": "${workspaceFolder}/dist/start.js", - "runtimeArgs": ["-r", "./tsconfig-paths-bootstrap.js"], + "name": "Launch Server", + "program": "${workspaceFolder}/dist/bundle/src/start.js", "preLaunchTask": "tsc: build - tsconfig.json", - "outFiles": ["${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/node_modules/@fosscord/**/*.js"], - "envFile": "${workspaceFolder}/.env", - "outDir": "${workspaceFolder}/dist" + "outFiles": ["${workspaceFolder}/dist/**/*.js"], + "envFile": "${workspaceFolder}/.env" } ] } diff --git a/bundle/package-lock.json b/bundle/package-lock.json index cb8078de..a967e97f 100644 Binary files a/bundle/package-lock.json and b/bundle/package-lock.json differ diff --git a/bundle/package.json b/bundle/package.json index a7f5358b..3bed5b07 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -4,11 +4,10 @@ "description": "", "main": "src/start.js", "scripts": { - "setup": "cd ../util && npm --production=false i && cd ../api && npm --production=false i && cd ../cdn && npm --production=false i && cd ../gateway && npm --production=false i && cd ../bundle/ && npm --production=false i && npm run build", + "setup": "node scripts/install.js && npm install && ts-patch install -s && patch-package --patch-dir ../api/patches/ && npm run build", "build": "node scripts/build.js", - "build:bundle": "npx tsc -b .", - "start": "node scripts/build.js && node -r tsconfig-paths/register dist/start.js", - "start:bundle": "node -r tsconfig-paths/register dist/start.js", + "start": "node scripts/build.js && node dist/bundle/src/start.js", + "start:bundle": "node dist/bundle/src/start.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -23,42 +22,76 @@ }, "homepage": "https://fosscord.com", "devDependencies": { - "@swc/cli": "^0.1.51", - "@swc/core": "^1.2.93", + "@babel/core": "^7.15.5", + "@babel/preset-env": "^7.15.6", + "@babel/preset-typescript": "^7.15.0", "@types/amqplib": "^0.8.1", - "@types/async-exit-hook": "^2.0.0", "@types/bcrypt": "^5.0.0", - "@types/express": "^4.17.9", + "@types/body-parser": "^1.19.0", + "@types/btoa": "^1.2.3", + "@types/dotenv": "^8.2.0", + "@types/express": "^4.17.12", + "@types/fs-extra": "^9.0.12", "@types/i18next-node-fs-backend": "^2.1.0", + "@types/jest": "^27.0.1", + "@types/jest-expect-message": "^1.0.3", "@types/jsonwebtoken": "^8.5.0", - "@types/mongodb": "^3.6.9", - "@types/mongoose-autopopulate": "^0.10.1", - "@types/mongoose-lean-virtuals": "^0.5.1", - "@types/multer": "^1.4.5", - "@types/node": "^14.17.20", - "@types/node-fetch": "^2.5.7", + "@types/multer": "^1.4.7", + "@types/node": "^14.17.9", + "@types/node-fetch": "^2.5.12", "@types/node-os-utils": "^1.2.0", - "@types/uuid": "^8.3.0", + "@types/supertest": "^2.0.11", "@types/ws": "^7.4.0", "@zerollup/ts-transform-paths": "^1.7.18", - "esbuild": "^0.13.4", - "esbuild-plugin-tsc": "^0.3.0", - "ts-node": "^10.2.1", + "jest": "^27.0.6", + "jest-expect-message": "^1.0.2", + "jest-runtime": "^27.2.1", + "ts-node": "^9.1.1", + "ts-node-dev": "^1.1.6", "ts-patch": "^1.4.4", - "tsconfig-paths": "^3.11.0", - "typescript": "^4.4.3" + "typescript": "^4.2.3", + "typescript-json-schema": "0.50.1", + "@types/morgan": "^1.9.3" }, "dependencies": { - "@fosscord/api": "file:../api", - "@fosscord/cdn": "file:../cdn", - "@fosscord/gateway": "file:../gateway", - "@fosscord/util": "file:../util", + "ajv": "8.6.2", + "ajv-formats": "^2.1.1", + "amqplib": "^0.8.0", + "assert": "^1.5.0", "async-exit-hook": "^2.0.1", - "dotenv": "^10.0.0", + "bcrypt": "^5.0.1", + "body-parser": "^1.19.0", + "btoa": "^1.2.1", + "dotenv": "^8.2.0", + "exif-be-gone": "^1.2.0", "express": "^4.17.1", + "express-async-errors": "^3.1.1", + "file-type": "^16.5.0", + "form-data": "^3.0.0", + "fs-extra": "^10.0.0", + "i18next": "^19.9.2", + "i18next-http-middleware": "^3.1.3", + "i18next-node-fs-backend": "^2.1.3", + "image-size": "^1.0.0", + "jest": "^27.0.6", + "jsonwebtoken": "^8.5.1", + "lambert-db": "^1.2.3", + "lambert-server": "^1.2.11", "missing-native-js-functions": "^1.2.17", + "morgan": "^1.10.0", + "multer": "^1.4.2", "nanocolors": "^0.2.12", + "node-fetch": "^2.6.1", "node-os-utils": "^1.3.5", - "reflect-metadata": "^0.1.13" + "patch-package": "^6.4.7", + "pg": "^8.7.1", + "reflect-metadata": "^0.1.13", + "sqlite3": "^5.0.2", + "supertest": "^6.1.6", + "typeorm": "^0.2.37", + "typescript": "^4.1.2", + "typescript-json-schema": "^0.50.1", + "ws": "^7.4.2", + "cheerio": "^1.0.0-rc.10" } -} +} \ No newline at end of file diff --git a/bundle/scripts/build.js b/bundle/scripts/build.js index 4266942f..69111c88 100644 --- a/bundle/scripts/build.js +++ b/bundle/scripts/build.js @@ -1,103 +1,33 @@ -const { spawn } = require("child_process"); +const { execSync } = require("child_process"); const path = require("path"); -const fs = require("fs"); -const { performance } = require("perf_hooks"); +const fse = require("fs-extra"); -let parts = "api,cdn,gateway,bundle".split(","); -const tscBin = path.join(__dirname, "..", "..", "util", "node_modules", "typescript", "bin", "tsc"); -const swcBin = path.join(__dirname, "..", "..", "util", "node_modules", "@swc", "cli", "bin", "swc"); +fse.copySync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets")); +fse.copySync( + path.join(__dirname, "..", "..", "api", "client_test"), + path.join(__dirname, "..", "dist", "api", "client_test") +); +fse.copySync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales")); +fse.copySync(path.join(__dirname, "..", "..", "api", "src"), path.join(__dirname, "..", "dist", "api", "src")); +fse.copySync(path.join(__dirname, "..", "..", "util", "src"), path.join(__dirname, "..", "dist", "util", "src")); +fse.copySync(path.join(__dirname, "..", "..", "cdn", "src"), path.join(__dirname, "..", "dist", "cdn", "src")); +fse.copySync(path.join(__dirname, "..", "..", "gateway", "src"), path.join(__dirname, "..", "dist", "gateway", "src")); +fse.copySync(path.join(__dirname, "..", "..", "bundle", "src"), path.join(__dirname, "..", "dist", "bundle", "src")); -// because npm run is slow we directly get the build script of the package.json script +console.log("Copying src files done"); +console.log("Compiling src files ..."); -function buildPackage(dir) { - const element = path.basename(dir); - - return require("esbuild").build({ - entryPoints: walk(path.join(dir, "src")), - bundle: false, - outdir: path.join(dir, "dist"), - target: "es2021", - // plugins don't really work because bundle is false - keepNames: false, - tsconfig: path.join(dir, "tsconfig.json"), - }); -} - -const importPart = /import (\* as )?(({[^}]+})|(\w+)) from ("[.\w-/@q]+")/g; -const importMod = /import ("[\w-/@q.]+")/g; -const exportDefault = /export default/g; -const exportAllAs = /export \* from (".+")/g; -const exportMod = /export ({[\w, ]+})/g; -const exportConst = /export (const|var|let) (\w+)/g; -const exportPart = /export ((async )?\w+) (\w+)/g; - -// resolves tsconfig paths + rewrites es6 imports/exports to require (because esbuild/swc doesn't work properly) -function transpileFiles() { - for (const part of ["gateway", "api", "cdn", "bundle"]) { - const files = walk(path.join(__dirname, "..", "..", part, "dist")); - for (const file of files) { - let content = fs.readFileSync(file, { encoding: "utf8" }); - content = content - .replace( - new RegExp(`@fosscord/${part}`), - path.relative(file, path.join(__dirname, "..", "..", part, "dist")).slice(3) - ) - .replace(importPart, `const $2 = require($5)`) - .replace(importMod, `require($1)`) - .replace(exportDefault, `module.exports =`) - .replace(exportAllAs, `module.exports = {...(module.exports)||{}, ...require($1)}`) - .replace(exportMod, "module.exports = $1") - .replace(exportConst, `let $2 = {};\nmodule.exports.$2 = $2`) - .replace(exportPart, `module.exports.$3 = $1 $3`); - fs.writeFileSync(file, content); +console.log( + execSync( + "node " + + path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") + + " -p " + + path.join(__dirname, ".."), + { + cwd: path.join(__dirname, ".."), + shell: true, + env: process.env, + encoding: "utf8", } - } -} - -function util() { - // const child = spawn("node", `${swcBin} src --out-dir dist --sync`.split(" "), { - const child = spawn("node", `${tscBin} -b .`.split(" "), { - cwd: path.join(__dirname, "..", "..", "util"), - env: process.env, - shell: true, - }); - function log(data) { - console.log(`[util] ` + data.toString().slice(0, -1)); - } - child.stdout.on("data", log); - child.stderr.on("data", log); - child.on("error", (err) => console.error("util", err)); - return child; -} - -const start = performance.now(); - -async function main() { - console.log("[Build] starting ..."); - util(); - await Promise.all(parts.map((part) => buildPackage(path.join(__dirname, "..", "..", part)))); - transpileFiles(); -} - -main(); - -process.on("exit", () => { - console.log("[Build] took " + Math.round(performance.now() - start) + "ms"); -}); - -function walk(dir) { - var results = []; - var list = fs.readdirSync(dir); - list.forEach(function (file) { - file = path.join(dir, file); - var stat = fs.statSync(file); - if (stat && stat.isDirectory()) { - /* Recurse into a subdirectory */ - results = results.concat(walk(file)); - } else if (file.endsWith(".ts") || file.endsWith(".js")) { - /* Is a file */ - results.push(file); - } - }); - return results; -} + ) +); diff --git a/bundle/scripts/install.js b/bundle/scripts/install.js new file mode 100644 index 00000000..3008b4c5 --- /dev/null +++ b/bundle/scripts/install.js @@ -0,0 +1,14 @@ +const path = require("path"); +const fs = require("fs"); +const parts = ["api", "util", "cdn", "gateway"]; + +const bundle = require("../package.json"); + +for (const part of parts) { + const { devDependencies, dependencies } = require(path.join("..", "..", part, "package.json")); + bundle.devDependencies = { ...bundle.devDependencies, ...devDependencies }; + bundle.dependencies = { ...bundle.dependencies, ...dependencies }; + delete bundle.dependencies["@fosscord/util"]; +} + +fs.writeFileSync(path.join(__dirname, "..", "package.json"), JSON.stringify(bundle, null, "\t"), { encoding: "utf8" }); diff --git a/bundle/src/Server.ts b/bundle/src/Server.ts index 5dda2d02..d541735f 100644 --- a/bundle/src/Server.ts +++ b/bundle/src/Server.ts @@ -4,7 +4,7 @@ process.on("uncaughtException", console.error); import http from "http"; import * as Api from "@fosscord/api"; import * as Gateway from "@fosscord/gateway"; -import { CDNServer } from "@fosscord/cdn/"; +import { CDNServer } from "@fosscord/cdn"; import express from "express"; import { green, bold } from "nanocolors"; import { Config, initDatabase } from "@fosscord/util"; diff --git a/bundle/src/start.ts b/bundle/src/start.ts index fbe5fa4c..8e7c3129 100644 --- a/bundle/src/start.ts +++ b/bundle/src/start.ts @@ -1,20 +1,4 @@ // process.env.MONGOMS_DEBUG = "true"; -const tsConfigPaths = require("tsconfig-paths"); -const path = require("path"); -const baseUrl = path.join(__dirname, ".."); -const cleanup = tsConfigPaths.register({ - baseUrl, - paths: { - "@fosscord/api": ["../api/dist/index.js"], - "@fosscord/api/*": ["../api/dist/*"], - "@fosscord/gateway": ["../gateway/dist/index.js"], - "@fosscord/gateway/*": ["../gateway/dist/*"], - "@fosscord/cdn": ["../cdn/dist/index.js"], - "@fosscord/cdn/*": ["../cdn/dist/*"], - }, -}); -console.log(require("@fosscord/gateway")); - import "reflect-metadata"; import cluster from "cluster"; import os from "os"; diff --git a/bundle/src/stats.ts b/bundle/src/stats.ts index d5ceeff7..18bb85ca 100644 --- a/bundle/src/stats.ts +++ b/bundle/src/stats.ts @@ -1,11 +1,19 @@ import os from "os"; import osu from "node-os-utils"; +import { red } from "nanocolors"; export function initStats() { console.log(`[Path] running in ${__dirname}`); console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`); console.log(`[System] ${os.platform()} ${os.arch()}`); console.log(`[Process] running with pid: ${process.pid}`); + if (process.getuid && process.getuid() === 0) { + console.warn( + red( + `[Process] Warning fosscord is running as root, this highly discouraged and might expose your system vulnerable to attackers. Please run fosscord as a user without root privileges.` + ) + ); + } setInterval(async () => { const [cpuUsed, memory, network] = await Promise.all([ @@ -23,5 +31,5 @@ export function initStats() { process.memoryUsage().rss / 1024 / 1024 )}mb/${memory.totalMemMb.toFixed(0)}mb ${networkUsage}` ); - }, 1000 * 5); + }, 1000 * 10); } diff --git a/bundle/tsconfig.json b/bundle/tsconfig.json index 69725244..4e8db342 100644 --- a/bundle/tsconfig.json +++ b/bundle/tsconfig.json @@ -1,22 +1,23 @@ { - "include": ["src/**/*.ts"], + "include": ["dist/**/*.ts"], + "exclude": [], "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */ /* Basic Options */ - "incremental": true /* Enable incremental compilation */, - "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "incremental": false /* Enable incremental compilation */, + "target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, "lib": ["ES2021"] /* Specify library files to be included in the compilation. */, "allowJs": true /* Allow javascript files to be compiled. */, "checkJs": true /* Report errors in .js files. */, // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ - "declaration": true /* Generates corresponding '.d.ts' file. */, + "declaration": false /* Generates corresponding '.d.ts' file. */, "declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */, - "sourceMap": true /* Generates corresponding '.map' file. */, + "sourceMap": false /* Generates corresponding '.map' file. */, // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "./dist/" /* Redirect output structure to the directory. */, - "rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, + "rootDir": "./dist/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, // "composite": true, /* Enable project compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ // "removeComments": true, /* Do not emit comments to output. */ @@ -66,6 +67,14 @@ "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */, "emitDecoratorMetadata": true, "experimentalDecorators": true, - "baseUrl": "." + "resolveJsonModule": true, + "baseUrl": "./dist/", + "paths": { + "@fosscord/api": ["api/src/index"], + "@fosscord/gateway": ["gateway/src/index"], + "@fosscord/cdn": ["cdn/src/index"], + "@fosscord/util": ["util/src/index"] + }, + "plugins": [{ "transform": "@zerollup/ts-transform-paths" }] } } diff --git a/cdn/package-lock.json b/cdn/package-lock.json index f0cce0af..a5a81e4f 100644 Binary files a/cdn/package-lock.json and b/cdn/package-lock.json differ diff --git a/cdn/package.json b/cdn/package.json index 5d3f1589..027ba553 100644 --- a/cdn/package.json +++ b/cdn/package.json @@ -7,7 +7,7 @@ "scripts": { "postinstall": "ts-patch install -s", "test": "npm run build && jest --coverage ./tests", - "build": "npx tsc -b .", + "build": "npx tsc -p .", "start": "npm run build && node dist/start.js" }, "repository": { @@ -22,8 +22,6 @@ }, "homepage": "https://github.com/fosscord/fosscord-server#readme", "devDependencies": { - "@swc/cli": "^0.1.51", - "@swc/core": "^1.2.93", "@types/amqplib": "^0.8.1", "@types/body-parser": "^1.19.0", "@types/btoa": "^1.2.3", @@ -31,13 +29,9 @@ "@types/express": "^4.17.12", "@types/fs-extra": "^9.0.12", "@types/jsonwebtoken": "^8.5.0", - "@types/mongodb": "^3.6.9", - "@types/mongoose-autopopulate": "^0.10.1", - "@types/mongoose-lean-virtuals": "^0.5.1", "@types/multer": "^1.4.7", "@types/node": "^14.17.0", "@types/node-fetch": "^2.5.7", - "@types/uuid": "^8.3.0", "@zerollup/ts-transform-paths": "^1.7.18", "ts-patch": "^1.4.4" }, @@ -45,7 +39,6 @@ "@fosscord/util": "file:../util", "body-parser": "^1.19.0", "btoa": "^1.2.1", - "cheerio": "^1.0.0-rc.5", "dotenv": "^10.0.0", "exif-be-gone": "^1.2.0", "express": "^4.17.1", @@ -61,8 +54,7 @@ "nanocolors": "^0.2.12", "node-fetch": "^2.6.1", "supertest": "^6.1.6", - "typescript": "^4.1.2", - "uuid": "^8.3.2" + "typescript": "^4.1.2" }, "jest": { "setupFilesAfterEnv": [ diff --git a/cdn/src/Server.ts b/cdn/src/Server.ts index 590eda6f..cac34a80 100644 --- a/cdn/src/Server.ts +++ b/cdn/src/Server.ts @@ -1,5 +1,5 @@ import { Server, ServerOptions } from "lambert-server"; -import { Config, initDatabase } from "@fosscord/util"; +import { Config, initDatabase, registerRoutes } from "@fosscord/util"; import path from "path"; import avatarsRoute from "./routes/avatars"; import bodyParser from "body-parser"; @@ -23,13 +23,19 @@ export class CDNServer extends Server { "Content-security-policy", "default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';" ); - res.set("Access-Control-Allow-Headers", req.header("Access-Control-Request-Headers") || "*"); - res.set("Access-Control-Allow-Methods", req.header("Access-Control-Request-Methods") || "*"); + res.set( + "Access-Control-Allow-Headers", + req.header("Access-Control-Request-Headers") || "*" + ); + res.set( + "Access-Control-Allow-Methods", + req.header("Access-Control-Request-Methods") || "*" + ); next(); }); this.app.use(bodyParser.json({ inflate: true, limit: "10mb" })); - await this.registerRoutes(path.join(__dirname, "routes/")); + await registerRoutes(this, path.join(__dirname, "routes/")); this.app.use("/icons/", avatarsRoute); this.log("verbose", "[Server] Route /icons registered"); diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json new file mode 100644 index 00000000..4d56041e Binary files /dev/null and b/dashboard/package-lock.json differ diff --git a/dashboard/package.json b/dashboard/package.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/dashboard/package.json @@ -0,0 +1 @@ +{} diff --git a/gateway/package-lock.json b/gateway/package-lock.json index 07ae02f9..085e40c0 100644 Binary files a/gateway/package-lock.json and b/gateway/package-lock.json differ diff --git a/gateway/package.json b/gateway/package.json index ddbddeff..d630c56b 100644 --- a/gateway/package.json +++ b/gateway/package.json @@ -8,23 +8,17 @@ "postinstall": "npx ts-patch install -s", "test": "echo \"Error: no test specified\" && exit 1", "start": "npm run build && node dist/start.js", - "build": "npx tsc -b .", + "build": "npx tsc -p .", "dev": "tsnd --respawn src/start.ts" }, "keywords": [], "author": "Fosscord", "license": "ISC", "devDependencies": { - "@swc/cli": "^0.1.51", - "@swc/core": "^1.2.93", "@types/amqplib": "^0.8.1", "@types/jsonwebtoken": "^8.5.0", - "@types/mongodb": "^3.6.9", - "@types/mongoose-autopopulate": "^0.10.1", - "@types/mongoose-lean-virtuals": "^0.5.1", "@types/node": "^14.17.9", "@types/node-fetch": "^2.5.12", - "@types/uuid": "^8.3.0", "@types/ws": "^7.4.0", "@zerollup/ts-transform-paths": "^1.7.18", "ts-node-dev": "^1.1.6", @@ -33,16 +27,13 @@ }, "dependencies": { "@fosscord/util": "file:../util", - "ajv": "^8.5.0", "amqplib": "^0.8.0", "dotenv": "^8.2.0", "jsonwebtoken": "^8.5.1", "lambert-server": "^1.2.11", "missing-native-js-functions": "^1.2.17", - "mongoose-autopopulate": "^0.12.3", "node-fetch": "^2.6.1", "typeorm": "^0.2.37", - "uuid": "^8.3.2", "ws": "^7.4.2" }, "optionalDependencies": { diff --git a/gateway/src/events/Connection.ts b/gateway/src/events/Connection.ts index c1a6b618..2cf22f7d 100644 --- a/gateway/src/events/Connection.ts +++ b/gateway/src/events/Connection.ts @@ -24,6 +24,7 @@ export async function Connection( request: IncomingMessage ) { try { + // @ts-ignore socket.on("close", Close); // @ts-ignore socket.on("message", Message); @@ -68,12 +69,10 @@ export async function Connection( }); socket.readyTimeout = setTimeout(() => { - Session.delete({ session_id: socket.session_id }); //should we await? return socket.close(CLOSECODES.Session_timed_out); }, 1000 * 30); } catch (error) { console.error(error); - Session.delete({ session_id: socket.session_id }); //should we await? return socket.close(CLOSECODES.Unknown_error); } } diff --git a/gateway/src/opcodes/Identify.ts b/gateway/src/opcodes/Identify.ts index b81c7bf4..5d5b72d1 100644 --- a/gateway/src/opcodes/Identify.ts +++ b/gateway/src/opcodes/Identify.ts @@ -41,7 +41,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { return this.close(CLOSECODES.Authentication_failed); } this.user_id = decoded.id; - if (!identify.intents) identify.intents = 0b11111111111111n; + if (!identify.intents) identify.intents = BigInt("0b11111111111111"); this.intents = new Intents(identify.intents); if (identify.shard) { this.shard_id = identify.shard[0]; diff --git a/gateway/src/opcodes/LazyRequest.ts b/gateway/src/opcodes/LazyRequest.ts index d37e32da..f5fd561a 100644 --- a/gateway/src/opcodes/LazyRequest.ts +++ b/gateway/src/opcodes/LazyRequest.ts @@ -41,6 +41,7 @@ export async function onLazyRequest(this: WebSocket, { d }: Payload) { const items = []; for (const role of roles) { + // @ts-ignore const [role_members, other_members] = partition(members, (m: Member) => m.roles.find((r) => r.id === role.id) ); @@ -53,9 +54,12 @@ export async function onLazyRequest(this: WebSocket, { d }: Payload) { groups.push(group); for (const member of role_members) { - member.roles = member.roles.filter((x) => x.id !== guild_id); + member.roles = member.roles.filter((x: Role) => x.id !== guild_id); items.push({ - member: { ...member, roles: member.roles.map((x) => x.id) }, + member: { + ...member, + roles: member.roles.map((x: Role) => x.id), + }, }); } members = other_members; @@ -84,7 +88,9 @@ export async function onLazyRequest(this: WebSocket, { d }: Payload) { } function partition(array: T[], isValid: Function) { + // @ts-ignore return array.reduce( + // @ts-ignore ([pass, fail], elem) => { return isValid(elem) ? [[...pass, elem], fail] diff --git a/rtc/package-lock.json b/rtc/package-lock.json new file mode 100644 index 00000000..f39aac1c Binary files /dev/null and b/rtc/package-lock.json differ diff --git a/rtc/package.json b/rtc/package.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/rtc/package.json @@ -0,0 +1 @@ +{} diff --git a/util/package-lock.json b/util/package-lock.json index 0ce50636..fa4549c6 100644 Binary files a/util/package-lock.json and b/util/package-lock.json differ diff --git a/util/package.json b/util/package.json index 32204081..5efc16ae 100644 --- a/util/package.json +++ b/util/package.json @@ -8,7 +8,7 @@ "start": "npm run build && node dist/", "test": "npm run build && jest", "postinstall": "npm run build", - "build": "npx tsc -b ." + "build": "npx tsc -p ." }, "repository": { "type": "git", @@ -28,22 +28,15 @@ }, "homepage": "https://docs.fosscord.com/", "devDependencies": { - "@swc/cli": "^0.1.51", - "@swc/core": "^1.2.93", "@types/amqplib": "^0.8.1", "@types/jsonwebtoken": "^8.5.0", - "@types/mongoose-autopopulate": "^0.10.1", "@types/multer": "^1.4.7", "@types/node": "^14.17.9", "@types/node-fetch": "^2.5.12", "jest": "^27.0.6" }, "dependencies": { - "ajv": "^8.6.2", "amqplib": "^0.8.0", - "class-validator": "^0.13.1", - "dot-prop": "^6.0.1", - "env-paths": "^2.2.1", "jsonwebtoken": "^8.5.1", "lambert-server": "^1.2.11", "missing-native-js-functions": "^1.2.17", @@ -54,7 +47,6 @@ "pg": "^8.7.1", "reflect-metadata": "^0.1.13", "sqlite3": "^5.0.2", - "tsconfig-paths": "^3.11.0", "typeorm": "^0.2.37", "typescript": "^4.4.2", "typescript-json-schema": "^0.50.1" diff --git a/util/src/entities/BaseClass.ts b/util/src/entities/BaseClass.ts index beccf04b..d20078e5 100644 --- a/util/src/entities/BaseClass.ts +++ b/util/src/entities/BaseClass.ts @@ -1,19 +1,8 @@ import "reflect-metadata"; -import { - BaseEntity, - BeforeInsert, - BeforeUpdate, - EntityMetadata, - FindConditions, - ObjectIdColumn, - PrimaryColumn, -} from "typeorm"; +import { BaseEntity, EntityMetadata, FindConditions, ObjectIdColumn, PrimaryColumn } from "typeorm"; import { Snowflake } from "../util/Snowflake"; import "missing-native-js-functions"; -// TODO use class-validator https://typeorm.io/#/validation with class annotators (isPhone/isEmail) combined with types from typescript-json-schema -// btw. we don't use class-validator for everything, because we need to explicitly set the type instead of deriving it from typescript also it doesn't easily support nested objects - export class BaseClassWithoutId extends BaseEntity { constructor(props?: any) { super(); @@ -42,7 +31,7 @@ export class BaseClassWithoutId extends BaseEntity { for (const key in props) { if (!properties.has(key)) continue; // @ts-ignore - const setter = this[`set${key.capitalize()}`]; + const setter = this[`set${key.capitalize()}`]; // use setter function if it exists if (setter) { setter.call(this, props[key]); @@ -53,12 +42,6 @@ export class BaseClassWithoutId extends BaseEntity { } } - @BeforeUpdate() - @BeforeInsert() - validate() { - return this; - } - toJSON(): any { return Object.fromEntries( this.metadata.columns // @ts-ignore @@ -76,42 +59,6 @@ export class BaseClassWithoutId extends BaseEntity { const repository = this.getRepository(); return repository.decrement(conditions, propertyPath, value); } - - // static async delete(criteria: FindConditions, options?: RemoveOptions) { - // if (!criteria) throw new Error("You need to specify delete criteria"); - - // const repository = this.getRepository(); - // const promises = repository.metadata.relations.map(async (x) => { - // if (x.orphanedRowAction !== "delete") return; - - // const foreignKey = - // x.foreignKeys.find((key) => key.entityMetadata === repository.metadata) || - // x.inverseRelation?.foreignKeys[0]; // find foreign key for this entity - // if (!foreignKey) { - // throw new Error( - // `Foreign key not found for entity ${repository.metadata.name} in relation ${x.propertyName}` - // ); - // } - // const id = (criteria as any)[foreignKey.referencedColumnNames[0]]; - // if (!id) throw new Error("id missing in criteria options " + foreignKey.referencedColumnNames); - - // if (x.relationType === "many-to-many") { - // return getConnection() - // .createQueryBuilder() - // .relation(this, x.propertyName) - // .of(id) - // .remove({ [foreignKey.columnNames[0]]: id }); - // } else if ( - // x.relationType === "one-to-one" || - // x.relationType === "many-to-one" || - // x.relationType === "one-to-many" - // ) { - // return (x.inverseEntityMetadata.target as any).delete({ [foreignKey.columnNames[0]]: id }); - // } - // }); - // await Promise.all(promises); - // return super.delete(criteria, options); - // } } export const PrimaryIdColumn = process.env.DATABASE?.startsWith("mongodb") ? ObjectIdColumn : PrimaryColumn; diff --git a/util/src/entities/Message.ts b/util/src/entities/Message.ts index 04c3c7aa..c7f7a1d1 100644 --- a/util/src/entities/Message.ts +++ b/util/src/entities/Message.ts @@ -46,9 +46,6 @@ export enum MessageType { @Entity("messages") export class Message extends BaseClass { - @Column() - id: string; - @Column({ nullable: true }) @RelationId((message: Message) => message.channel) channel_id: string; diff --git a/util/src/entities/RateLimit.ts b/util/src/entities/RateLimit.ts index fa9c32c1..f5916f6b 100644 --- a/util/src/entities/RateLimit.ts +++ b/util/src/entities/RateLimit.ts @@ -3,9 +3,6 @@ import { BaseClass } from "./BaseClass"; @Entity("rate_limits") export class RateLimit extends BaseClass { - @Column() - id: "global" | "error" | string; // channel_239842397 | guild_238927349823 | webhook_238923423498 - @Column() // no relation as it also executor_id: string; diff --git a/util/src/entities/User.ts b/util/src/entities/User.ts index 97564af3..662ab031 100644 --- a/util/src/entities/User.ts +++ b/util/src/entities/User.ts @@ -198,7 +198,7 @@ export class User extends BaseClass { // randomly generates a discriminator between 1 and 9999 and checks max five times if it already exists // if it all five times already exists, abort with USERNAME_TOO_MANY_USERS error // else just continue - // TODO: is there any better way to generate a random discriminator only once, without checking if it already exists in the mongodb database? + // TODO: is there any better way to generate a random discriminator only once, without checking if it already exists in the database? for (let tries = 0; tries < 5; tries++) { discriminator = Math.randomIntBetween(1, 9999).toString().padStart(4, "0"); exists = await User.findOne({ where: { discriminator, username: username }, select: ["id"] }); @@ -219,7 +219,7 @@ export class User extends BaseClass { // if nsfw_allowed is null/undefined it'll require date_of_birth to set it to true/false const language = req.language === "en" ? "en-US" : req.language || "en-US"; - const user = await new User({ + const user = new User({ created_at: new Date(), username: username, discriminator, @@ -246,7 +246,10 @@ export class User extends BaseClass { }, settings: { ...defaultSettings, locale: language }, fingerprints: [], - }).save(); + }); + + console.log(user); + await user.save(); if (Config.get().guild.autoJoin.enabled) { for (const guild of Config.get().guild.autoJoin.guilds || []) { diff --git a/util/src/entities/Webhook.ts b/util/src/entities/Webhook.ts index 8382435f..9d8609ae 100644 --- a/util/src/entities/Webhook.ts +++ b/util/src/entities/Webhook.ts @@ -12,9 +12,6 @@ export enum WebhookType { @Entity("webhooks") export class Webhook extends BaseClass { - @Column() - id: string; - @Column({ type: "simple-enum", enum: WebhookType }) type: WebhookType; diff --git a/util/src/index.ts b/util/src/index.ts index fc00d46b..ae0f7e54 100644 --- a/util/src/index.ts +++ b/util/src/index.ts @@ -1,12 +1,6 @@ import "reflect-metadata"; -// export * as Constants from "../util/Constants"; export * from "./util/index"; export * from "./interfaces/index"; export * from "./entities/index"; export * from "./dtos/index"; - -// import Config from "../util/Config"; -// import db, { MongooseCache, toObject } from "./util/Database"; - -// export { Config }; diff --git a/util/src/util/TraverseDirectory.ts b/util/src/util/TraverseDirectory.ts new file mode 100644 index 00000000..275b7dcc --- /dev/null +++ b/util/src/util/TraverseDirectory.ts @@ -0,0 +1,10 @@ +import { Server, traverseDirectory } from "lambert-server"; + +const DEFAULT_FILTER = /^([^\.].*)(?