diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ab03d840 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.vscode/ +node_modules/ +dist/ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..9d7a1f51 Binary files /dev/null and b/package-lock.json differ diff --git a/package.json b/package.json new file mode 100644 index 00000000..847c688f --- /dev/null +++ b/package.json @@ -0,0 +1,38 @@ +{ + "name": "discord-cdn", + "version": "1.0.0", + "description": "cdn for discord clone", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/discord-open-source/discord-cdn.git" + }, + "keywords": [], + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/discord-open-source/discord-cdn/issues" + }, + "homepage": "https://github.com/discord-open-source/discord-cdn#readme", + "dependencies": { + "body-parser": "^1.19.0", + "btoa": "^1.2.1", + "cheerio": "^1.0.0-rc.5", + "express": "^4.17.1", + "express-async-errors": "^3.1.1", + "lambert-db": "^1.0.5", + "missing-native-js-functions": "^1.0.8", + "multer": "^1.4.2", + "node-fetch": "^2.6.1" + }, + "devDependencies": { + "@types/btoa": "^1.2.3", + "@types/express": "^4.17.9", + "@types/multer": "^1.4.5", + "@types/node": "^14.14.16", + "@types/node-fetch": "^2.5.7" + } +} diff --git a/src/Server.ts b/src/Server.ts new file mode 100644 index 00000000..7d93c444 --- /dev/null +++ b/src/Server.ts @@ -0,0 +1,93 @@ +import express, { Application, Router, Request, Response, NextFunction } from "express"; +import { MongoDatabase, Database } from "lambert-db"; +import { Server as HTTPServer } from "http"; +import { traverseDirectory } from "./Util"; +import bodyParser from "body-parser"; +import "express-async-errors"; + +const log = console.log; +console.log = (content) => { + log(`[${new Date().toTimeString().split(" ")[0]}]`, content); +}; + +export type ServerOptions = { + db: string; + port: number; + host: string; +}; + +declare global { + namespace Express { + interface Request { + server: Server; + } + } +} + +export class Server { + app: Application; + http: HTTPServer; + db: Database; + routes: Router[]; + options: ServerOptions; + + constructor(options: Partial = { port: 3000, host: "0.0.0.0" }) { + this.app = express(); + this.db = new MongoDatabase(options?.db); + this.options = options as ServerOptions; + } + + async init() { + await this.db.init(); + + console.log("[Database] connected..."); + await new Promise((res, rej) => { + this.http = this.app.listen(this.options.port, this.options.host, () => res(null)); + }); + this.routes = await this.registerRoutes(__dirname + "/routes/"); + } + + async registerRoutes(root: string) { + this.app.use((req, res, next) => { + req.server = this; + next(); + }); + const routes = await traverseDirectory({ dirname: root, recursive: true }, this.registerRoute.bind(this, root)); + this.app.use((err: string | Error, req: Request, res: Response, next: NextFunction) => { + res.status(400).send(err); + next(err); + }); + return routes; + } + + registerRoute(root: string, file: string): any { + if (root.endsWith("/") || root.endsWith("\\")) root = root.slice(0, -1); // removes slash at the end of the root dir + let path = file.replace(root, ""); // remove root from path and + path = path.split(".").slice(0, -1).join("."); // trancate .js/.ts file extension of path + if (path.endsWith("/index")) path = path.slice(0, -6); // delete index from path + + try { + var router = require(file); + if (router.router) router = router.router; + if (router.default) router = router.default; + if (!router || router?.prototype?.constructor?.name !== "router") + throw `File doesn't export any default router`; + this.app.use(path, router); + console.log(`[Routes] ${path} registerd`); + + return router; + } catch (error) { + console.error(new Error(`[Server] ¯\\_(ツ)_/¯ Failed to register route ${path}: ${error}`)); + } + } + + async destroy() { + await this.db.destroy(); + await new Promise((res, rej) => { + this.http.close((err) => { + if (err) return rej(err); + return res(""); + }); + }); + } +} diff --git a/src/Util.ts b/src/Util.ts new file mode 100644 index 00000000..291372c1 --- /dev/null +++ b/src/Util.ts @@ -0,0 +1,38 @@ +import fs from "fs/promises"; +import "missing-native-js-functions"; + +export interface traverseDirectoryOptions { + dirname: string; + filter?: RegExp; + excludeDirs?: RegExp; + recursive?: boolean; +} + +const DEFAULT_EXCLUDE_DIR = /^\./; +const DEFAULT_FILTER = /^([^\.].*)\.js$/; + +export async function traverseDirectory( + options: traverseDirectoryOptions, + action: (path: string) => T +): Promise { + if (!options.filter) options.filter = DEFAULT_FILTER; + if (!options.excludeDirs) options.excludeDirs = DEFAULT_EXCLUDE_DIR; + + const routes = await fs.readdir(options.dirname); + const promises = []>routes.map(async (file) => { + const path = options.dirname + file; + const stat = await fs.lstat(path); + if (path.match(options.excludeDirs)) return; + + if (stat.isFile() && path.match(options.filter)) { + return action(path); + } else if (options.recursive && stat.isDirectory()) { + return traverseDirectory({ ...options, dirname: path + "/" }, action); + } + }); + const result = await Promise.all(promises); + + const t = <(T | undefined)[]>result.flat(); + + return t.filter((x) => x != undefined); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 00000000..d8025968 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,14 @@ +import { Server } from "./Server"; + +const server = new Server(); +server + .init() + .then(() => { + console.log("[Server] started on :" + server.options.port); + }) + .catch((e) => console.error("[Server] Error starting: ", e)); + +//// server +//// .destroy() +//// .then(() => console.log("[Server] closed.")) +//// .catch((e) => console.log("[Server] Error closing: ", e)); diff --git a/src/routes/attachments.ts.disabled b/src/routes/attachments.ts.disabled new file mode 100644 index 00000000..db1a7efc --- /dev/null +++ b/src/routes/attachments.ts.disabled @@ -0,0 +1,19 @@ +import { Router } from "express"; +import multer from "multer"; +const multer_ = multer(); + +const router = Router(); +router.post("/:file", multer_.single("attachment"), async (req, res) => { + const { buffer } = req.file; + + res.set("Content-Type", "image/png"); + res.send(buffer); +}); +router.get("/:hash/:file", async (req, res) => { + res.send(`${req.params.hash}/${req.params.file}`); +}); +router.delete("/:hash/:file", async (req, res) => { + res.send("remove"); +}); + +export default router; diff --git a/src/routes/external.ts b/src/routes/external.ts new file mode 100644 index 00000000..14980b05 --- /dev/null +++ b/src/routes/external.ts @@ -0,0 +1,83 @@ +import bodyParser from "body-parser"; +import { Router } from "express"; +import fetch from "node-fetch"; +import cheerio from "cheerio"; +import btoa from "btoa"; +import { URL } from "url"; + +const router = Router(); + +type crawled = { + title: string; + type: string; + description: string; + url: string; + image_url: string; +}; + +const DEFAULT_FETCH_OPTIONS: any = { + redirect: "follow", + follow: 1, + headers: { + "user-agent": "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)", + }, + size: 1024 * 1024 * 8, + compress: true, + method: "GET", +}; + +router.post("/", bodyParser.json(), async (req, res) => { + if (!req.body) throw new Error("Invalid Body (url missing) \nExample: url:https://discord.com"); + + const { db } = req.server; + const { url } = req.body; + + const ID = btoa(url); + + const cache = await db.data.crawler({ id: ID }).get(); + if (cache) return res.send(cache); + + try { + const request = await fetch(url, DEFAULT_FETCH_OPTIONS); + + const text = await request.text(); + const ツ: any = cheerio.load(text); + + const ogTitle = ツ('meta[property="og:title"]').attr("content"); + const ogDescription = ツ('meta[property="og:description"]').attr("content"); + const ogImage = ツ('meta[property="og:image"]').attr("content"); + const ogUrl = ツ('meta[property="og:url"]').attr("content"); + const ogType = ツ('meta[property="og:type"]').attr("content"); + + const filename = new URL(url).host.split(".")[0]; + + const ImageResponse = await fetch(ogImage, DEFAULT_FETCH_OPTIONS); + const ImageType = ImageResponse.headers.get("content-type"); + const ImageExtension = ImageType?.split("/")[1]; + const ImageResponseBuffer = (await ImageResponse.buffer()).toString("base64"); + const cachedImage = `/external/${ID}/${filename}.${ImageExtension}`; + + await db.data.externals.push({ image: ImageResponseBuffer, id: ID, type: ImageType }); + + const new_cache_entry = { id: ID, ogTitle, ogDescription, cachedImage, ogUrl, ogType }; + await db.data.crawler.push(new_cache_entry); + + res.send(new_cache_entry); + } catch (error) { + console.log(error); + + throw new Error("Couldn't fetch website"); + } +}); + +router.get("/:id/:filename", async (req, res) => { + const { db } = req.server; + const { id, filename } = req.params; + const { image, type } = await db.data.externals({ id: id }).get(); + const imageBuffer = Buffer.from(image, "base64"); + + res.set("Content-Type", type); + res.send(imageBuffer); +}); + +export default router; diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..26be7b1b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,69 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Basic Options */ + // "incremental": true, /* Enable incremental compilation */ + "target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "lib": ["ES2015"] /* Specify library files to be included in the compilation. */, + "allowJs": true /* Allow javascript files to be compiled. */, + "checkJs": true /* Report errors in .js files. */, + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ + "declaration": true /* Generates corresponding '.d.ts' file. */, + "declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */, + "inlineSourceMap": true /* Emit a single file with source maps instead of having a separate file. */, + // "sourceMap": true /* Generates corresponding '.map' file. */, + // "outFile": "./", /* Concatenate and emit output to single file. */ + "outDir": "./dist/" /* Redirect output structure to the directory. */, + "rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, + // "composite": true, /* Enable project compilation */ + // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, + // "strictNullChecks": true, /* Enable strict null checks. */ + // "strictFunctionTypes": true, /* Enable strict checking of function types. */ + // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ + "strictPropertyInitialization": false /* Enable strict checking of property initialization in classes. */, + // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + + /* Additional Checks */ + // "noUnusedLocals": true, /* Report errors on unused locals. */ + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + + /* Module Resolution Options */ + // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + // "typeRoots": [], /* List of folders to include type definitions from. */ + "types": ["node"] /* Type declaration files to be included in compilation. */, + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + + /* Source Map Options */ + // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + + /* Advanced Options */ + "skipLibCheck": true /* Skip type checking of declaration files. */, + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + } +}