From 8f66556f5acf149fadd2fe0ac9e03f5e6225026e Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 27 May 2021 18:54:15 +0200 Subject: [PATCH] :construction: WIP rewrite --- .env.example | 1 + .gitignore | 1 + package-lock.json | Bin 81746 -> 99841 bytes package.json | 4 +- src/Server.ts | 32 +++------ src/Snowflake.js | 145 -------------------------------------- src/Util.ts | 38 ---------- src/index.ts | 10 ++- src/routes/attachments.ts | 13 ++-- src/routes/external.ts | 22 +++--- src/util/FileStorage.ts | 7 ++ src/util/Storage.ts | 14 ++++ tsconfig.json | 1 + 13 files changed, 57 insertions(+), 231 deletions(-) create mode 100644 .env.example delete mode 100644 src/Snowflake.js delete mode 100644 src/Util.ts create mode 100644 src/util/FileStorage.ts create mode 100644 src/util/Storage.ts diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..2a1176ff --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +STORAGE_LOCATION=files/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 234771f2..6054dfe8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode/ node_modules/ .DS_Store +.env \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a2bdd9283e6c4be1e7b0ad4a6353b27833faa694..9582bc4059efa90927b387b7d8a6a037902c144a 100644 GIT binary patch delta 12735 zcmeHNX^b1!eaC1et>o2ZE7IzWc2|)cndZ(6heOUdabw>1@SNdM$MIYwhaAof=irq? zF%UFBP&9U?1=BQb;tiX%qJix!k4X zwGzjc><4-=rW8M}jbIn9otcSbw$8u}-H#*B#TZXFiMb1>)qB50)*u*!H zjR)^J2>xw6Vh%OYfIF(vP*p>8vzDo9s%FWoDNa`-n<)5urRB27D(;@7n-(h@TkG+g zmfK(1bG+%n=NquCnL2M`0;t)o>8du}z_+?$SR3*GFY37t6?D~N>f4Ua3~%A@@(t>YLviU;#9o`F3?E{Ty` zF_&twP1dZ{Jn4+plp0o~)DAV$%&4wgeYn=6GE_UI7Ds&>%3W~I+5Url4^Ez0DebXu z$Zv%Zn;lO1ve!DR5HfylAGFS3{(L48AaQRAQ(W<4+bB|Lc~CRDQKFtnT6obb40AzM z@HGm9m}ykxJlUenG!cvtMN#VXNYe6?V!D#RxJE+_DaAZoALO?fxX%{$zVj`+sn~u3 zIbo~s+BaUpAlT8BXSY=%P};8JZs!Kt-~aSFllJ*DfkXr=N7RyO`dL40XoYIq<@d=A z&F7B>a=rvcR7r{tq3ygi>UR~ZU1F0XRj6|WN+A6~yq58HO&?+T20qQS!UcBD|Dipq zVL!d^%8m#Ncl%@ElS|0v_F<#jj4rH^jYSD=ns6i23&2rrS3cd!;Y^M8MvNre@DB1u zD;;2oP^1GFQ?>9=3VGTY4?^@4dd%wh1UBq%dGT!5z!9!Y#p>N!wO~b>O=3_({b4^F znO}f8|C8k-?|~q{ePeZY{+~X!*Z#<#9|9xmKUin_1fWg9b$_wQ^_YUlk;Ss+vXDRZzf)B)&lnT zW2ZlMWO))l)>z)t>N%ES2RU>F7(37j_#arnliWq7>X9*$ESYGaVbNGHSICi3#Vb;- ztnMnBbgUerHBN~RD@B!{YGl!CwOoOiM{5~4tEF7EOtxRmj!+ltVp6?S+#j-pN@IF} z-a2cTZb1|^x7Y=~Xll3^Fx+j=5{uOb3$I=gghILM|uC$uWZyUMVNi`-kG0q{D-qeLo%Bdx3k5R ztEOpc#zQbXo807A)egAEs$!_R0gIZ-iMlvyONxE((z(g!SHFAkzNtWN9xO3g#dMoe z2Uyf;`*9w&U;5fI^73iNlsKzi2SII@$?=*Ox_H)=ANEyIoQ1j-MC zZF~LnsmX7wfA8XfxnH)?55`vKFJSZeqZ{~Y$1-U%;vbpSQ6-)bhdtg;xtaw`mHg2J zSBI%Yi^-AAPQb*5naW7dBYqED*IKdkz!Mu#gV=zUamJ5jBHijpQnMMAX~r^1oCg8y zkYu=3t~EOk`>R)66pGwd^_IS?k{5MsNQ$6~K;2EtaFo0sNCUzQh}tKB+kEP+p8IA4 zGdZ0A1l&*F2mH4jlt_VSX+5%7N856)Mb)DLs~m2n(*05*nS_~vsh0YQg0HF627Dx& z57inTHIc?L`zxpTtdquSl-Ulm+c>9&L8CkgLNBH!(CVP4r|)$ z{nLP-Pk#Z0#apIKQ3uTJGU`U1*lFi~Z;esdhKv=eI!E<%?5BgU zr)Q3QLyD|vsBGhcZYIlBKb zkU{`T&K<2serM0}R+zS-@Tz_K40>WaZaa+5W4N6^bBS>^QMsK?HEWqtwWT&tCZZ*& zn4hX8%e=*V!r?aDHM+%8p%BVeq;yNnqXG(Bf=6i;n=T-7?|^9>a$Atb_@LuOmkJ(iyc64^X0VlJE@%nwxy8YbQy^NrD!!eBK_|aex zNfy*>h0nH3K|_@_m&Be2J?$J*i-^DLZ-OImz3w~giImfr`GHgHo*^MdRREB_a zq=Hpim1d%*pmMs|jFooJ&bOG$T_`xhn)*FFG*m#-Ma*#_B8_?zX(Gs z_MUxfj4vvWEUuKU#jvD@g()T!ki|%|HR|H(pb;OkA(|XkD?L7K^i+n8C%dqL4=M>C zT&d)I=Ez?l2Z5{^SBhEKte8=(Tqw!YG-avPXPpAg{^GaMgW%b@&S2!n-@Zh5rE~=z zG$WB{iVh@wQ7M?=L<8%SfzoK$#>@VAZ=|tsFZ4D%ZM<(9%{C_gqfvlqrN|^ z6~ko)m~B`WeWHN&VNLLocs8uXYt^FAE|d%wi34#~fWD@e4IHTbV*2y~OOti4nr?agMUy(q+NWF|u{E2xo7x!B5Q&_q9Gq~e(hQRt`R zaNpVd-tBGX4#3WTeia-B$a0=V+%KC2?qQMA{39?7gt_)d>u2lHWS2H2uD}=gffn*M z%2`)1Sx4!N;m|v}z2nT4*nw*&$J{$1bVpyaiA5^25x2wCfBf*m8MIihn>1q$l@TaN zD~!j|y^@bfF;V-*qo?oLq5yo+w5TLMfSuhLazfL>8({i{Z(KN!y2C=Y5^3h*gEHoA zBn6U=3==MdXJy}1O%pHsg@Yg{vOGL~W*Imz!hRxufic3ZWG~^z%B?Jg_E?SQhQTlv zP)V4fuu`g8CE$F8jN~}O<4LihqMpa~ii+22g4b)7lUQHIqOzh#Yp&FgHmSJJh1PPj z1nR5p;2BjjB~@W%_g49_Wll{BmBwJ^Wx;-2Itg0;=u9z&e~gM+oBF)hw3=-`RoHb-%iV1fphb+ zB!bEV5Y~rNz899Sy?i;K=6LvpMXNiD429LS;lkV!n6qzOI%~fmUjWVwB)m?~0(5a^ zi7e_XG2Lo2o)>CVfhd$D112P?lpDH~WUh3FlnbW2uq7Q8{%?8v=brtR%H-xnn>?sc=1qlZdiH{R;Lu_- z&bjZP1%ZwOM2f27IV@86Y1LPlev3@XL@G*^4D3{H%sa`07(giLF-b_6wruz>+JQHxk zxu5pho|7l%VmM0lZVuMo_r5iJb1)Zbl2S8EAR9;Quj}_tV?F3W=!IT0ESnXEO8cS% z!86K`VQ()L9JClFPo*iTCAa*RTA_z1IikWwLaO7Vf=erRa-lMn>Icm3Q1vEzV!q>5 z>S5d^&I*Dn*8TP;j8ir@M8?KZ==k{geb5@sw6dvqyg?J*zSQz3Xtg$q7HGtu2n+}f9j;~&8nZy9$E?A9UsN&6C0U_3R=lc{=Gv6{kTt{`003_>pBR6H<>;Q5fu zMRO?La^-;Zt`eCW$jM4Dn@y|5fY9hidquB_u$h9lr-?$Q&Bg0YM472$fBpQ#^}&DK zzpPoxkXMDN4_%AxwSWEN@bRrNhp+jg{nW=VF_nT*XbE*!3-8FTA?%B#;<*~9M`1$q zR2wXk%$Y*Bl&E?tBCa-*xiD$4;bbzM>uIEmDh4&3>4`m|QX6tg%ft7GY(zkI>b1Qe zesTYy#omik7oDpF0d)q$_m*nBj5l>U9HyIn*3}D8LyNPz^;)^q6KNO@SDBa*j`<6zG~A2WPk!cp z`(Hk=Ho5ZT->vL*T!E8s*{)9>n!SKji*YpuLL{3Bc#^I_!c}LCP*`G+9SsxfU*1C?wq0f8DMe^0#QyilW8eGnx7zzY`<*4w zUbCP6A16};sYU`hKI^vu*=7$wF7LF_wDkK7?1tbx6au=`_o^6JUf;1Jp9}g9jAv1qUX;6dF&&fTeW}pi-#wF z`NxOelwi*|a2z@d4*c@uZ@=Wb>qyU>`ttav2SK4O?0vg^{&UMafXM=OI1MdtUd(4( zlI5-IVxylwwjE_E8wR)cLop{KaN1FcN-CL0?F$MiLB1B=-JN#wu8bO0ggD5-HMJJTI6nW z>3`q$I^ZL|+u$P$adEpHzlAcsdX&U_8Lazxc!ES?<>_*H%LS7|e;=c`IvKxsx z1oGN&6Z|%~$!=7CY4iknW%MNSazd{HM7eNV5ar>G$s;d?58N#XWmfh4LKwt8+wRSv$^etz}N^z%WIcm2Q_2!B<*eE!b# z@)r}%F8Br53H!{4pS`}a4n_C9PM!Vo9qH`%AHE)XJM@)%?rKy2`m-2x|MD;2)i2+n nuKxI&4?)-7*f#u_MWJ)2ZtcT^-x`6W?%?~czxN%`KP>$}rm0L^ delta 2383 zcmd7SYiv_h902f~y}E7^L12sxM)wdCyY;^M6cSn2u4{X5d)xK4TUFq;z3b!NwY|6P zZ95W~L}Eli@F>?uqAnzagb={{p(Yv~iHXKQ%tS*-+yv1Nen2JBkYJ*YB_csXNDPbL z&iUu$od5s)&-p#`V$&DDHyr#@`C$v_g6n}%(QF0V)T1B?8jJ=b0k74qhQ0M3c&S;V z2AXy^!?p9jK)P=Q)OBjqj#VU3r`xuJ`o*cFQ!p7-P zegq}s6RBiIE=d7diVoYQA#d7PRVx{)@fpd!bkB)a+Q7Mpbrpt$^pUs``=aX;4p0 zpVUHR0Z}e_K_fi4Zx!6NpkpL|v=l@Gd@d19Q+g&AaGH2W!Bge!VkC_Ykw#}6Ph(lh zB{6Qo>&Gz3JRGYOcyUrBJYL*4Vqqi3L_8k1mzYv!kVc(B!2}Ogbjno^u(wN`SV~LW zj$FQ&&ShDl=P_N+cFcf6`lz&}_^UU9O5)%v!B>o*&oCMhi+Z6Ru=6|9J*qrQKLs)sm&C}a>k z1mEq`G1;8tOlJko#tz3(6BW0!vP_r#Hc3SNp}f_` zyTTz+mQ9k6q^4|Urzss7$%p;HjN6D|GJNUCX84A(8MYj5hjm&+85jj?ls$bw$Hd(e zG1?faQsZO#QJxLpRJQ2Q594Dok+qL7;}M?7=y@^Wm@3kd98VOJ)k47^txzLWaVi$Z z$~Ff(D#%WbMcui0Ny-_qK_i@bO{d;CJkzSq9l6+~etLYqNqeiE@XQ`p@0~qH29P^L z@bYXo{JFJ(5mOXWHCec5?35i;V7mnqWB5{NwE93?rV;G*z^+pSMJ{ir?f4KjQr znkys4s*v-Eo~qBxkBtgMuv(?8q+oI&1rKe#nc|}oSI7x6H!vaPCYN3rzSte~ul^Od zNtu(D;wHHAtgiL80@5T&m&k4dLwD-5an0!966o{Q|8|;at+2bXN2^ zCXsSQSPY&S?^9>hYpuWnSM}q{LK!@;IGj1w%j86dg{D0ElHZ&O+nj+R0VSni8S|G5 zX}XXe4g^huK>|&2LA}u-STc5d%t2;-awN_2I9i;F3ZqVJ#1ga<=;#=NWsvdlkOhi6 zI@Gv!(gCQS&MBiw_Z&uj`-(s6<#Sj?N z!nq??YNvJpcVq2a4y^g#akP|o|LvOG-6I30fHYXGy~WKvg0%5&AjNAx3gCm5e{dAO XFB~OnTm?L$QFcu)estp=aIXGO7Zp6R diff --git a/package.json b/package.json index 489b67dd..69707a51 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,11 @@ }, "homepage": "https://github.com/discord-open-source/discord-cdn#readme", "dependencies": { + "@fosscord/server-util": "^1.3.3", "body-parser": "^1.19.0", "btoa": "^1.2.1", "cheerio": "^1.0.0-rc.5", + "dotenv": "^10.0.0", "express": "^4.17.1", "express-async-errors": "^3.1.1", "lambert-db": "^1.2.3", @@ -37,7 +39,7 @@ "@types/btoa": "^1.2.3", "@types/express": "^4.17.11", "@types/multer": "^1.4.5", - "@types/node": "^14.14.43", + "@types/node": "^14.17.0", "@types/node-fetch": "^2.5.7", "@types/uuid": "^8.3.0" } diff --git a/src/Server.ts b/src/Server.ts index 3e8c9321..3ad794be 100644 --- a/src/Server.ts +++ b/src/Server.ts @@ -1,42 +1,28 @@ -import { MongoDatabase, Database } from "lambert-db"; import { Server, ServerOptions } from "lambert-server"; +import { Config, db } from "@fosscord/server-util"; +import path from "path"; -const log = console.log; -console.log = (content) => { - log(`[${new Date().toTimeString().split(" ")[0]}]`, content); -}; - -declare global { - namespace Express { - interface Request { - cdn: CDNServer; - } - } -} - -export interface CDNServerOptions extends ServerOptions { - db: string; -} +export interface CDNServerOptions extends ServerOptions {} export class CDNServer extends Server { - db: Database; public options: CDNServerOptions; - constructor(options: Partial) { + constructor(options?: Partial) { super(options); - - this.db = new MongoDatabase(options?.db); } async start() { console.log("[Database] connecting ..."); - await this.db.init(); + // @ts-ignore + await (db as Promise); + await Config.init(); console.log("[Database] connected"); + + await this.registerRoutes(path.join(__dirname, "routes")); return super.start(); } async stop() { - await this.db.destroy(); return super.stop(); } } diff --git a/src/Snowflake.js b/src/Snowflake.js deleted file mode 100644 index feb5eb41..00000000 --- a/src/Snowflake.js +++ /dev/null @@ -1,145 +0,0 @@ -// @ts-nocheck - -// github.com/discordjs/discord.js/blob/master/src/util/Snowflake.js -"use strict"; - -// Discord epoch (2015-01-01T00:00:00.000Z) -const EPOCH = 1420070400000; -let INCREMENT = 0; - -/** - * A container for useful snowflake-related methods. - */ -class SnowflakeUtil { - constructor() { - throw new Error(`The ${this.constructor.name} class may not be instantiated.`); - } - - /** - * A Twitter snowflake, except the epoch is 2015-01-01T00:00:00.000Z - * ``` - * If we have a snowflake '266241948824764416' we can represent it as binary: - * - * 64 22 17 12 0 - * 000000111011000111100001101001000101000000 00001 00000 000000000000 - * number of ms since Discord epoch worker pid increment - * ``` - * @typedef {string} Snowflake - */ - - /** - * Transforms a snowflake from a decimal string to a bit string. - * @param {Snowflake} num Snowflake to be transformed - * @returns {string} - * @private - */ - static idToBinary(num) { - let bin = ""; - let high = parseInt(num.slice(0, -10)) || 0; - let low = parseInt(num.slice(-10)); - while (low > 0 || high > 0) { - bin = String(low & 1) + bin; - low = Math.floor(low / 2); - if (high > 0) { - low += 5000000000 * (high % 2); - high = Math.floor(high / 2); - } - } - return bin; - } - - /** - * Transforms a snowflake from a bit string to a decimal string. - * @param {string} num Bit string to be transformed - * @returns {Snowflake} - * @private - */ - static binaryToID(num) { - let dec = ""; - - while (num.length > 50) { - const high = parseInt(num.slice(0, -32), 2); - const low = parseInt((high % 10).toString(2) + num.slice(-32), 2); - - dec = (low % 10).toString() + dec; - num = - Math.floor(high / 10).toString(2) + - Math.floor(low / 10) - .toString(2) - .padStart(32, "0"); - } - - num = parseInt(num, 2); - while (num > 0) { - dec = (num % 10).toString() + dec; - num = Math.floor(num / 10); - } - - return dec; - } - - /** - * Generates a Discord snowflake. - * This hardcodes the worker ID as 1 and the process ID as 0. - * @param {number|Date} [timestamp=Date.now()] Timestamp or date of the snowflake to generate - * @returns {Snowflake} The generated snowflake - */ - static generate(timestamp = Date.now()) { - if (timestamp instanceof Date) timestamp = timestamp.getTime(); - if (typeof timestamp !== "number" || isNaN(timestamp)) { - throw new TypeError( - `"timestamp" argument must be a number (received ${isNaN(timestamp) ? "NaN" : typeof timestamp})` - ); - } - if (INCREMENT >= 4095) INCREMENT = 0; - const BINARY = `${(timestamp - EPOCH).toString(2).padStart(42, "0")}0000100000${(INCREMENT++) - .toString(2) - .padStart(12, "0")}`; - return SnowflakeUtil.binaryToID(BINARY); - } - - /** - * A deconstructed snowflake. - * @typedef {Object} DeconstructedSnowflake - * @property {number} timestamp Timestamp the snowflake was created - * @property {Date} date Date the snowflake was created - * @property {number} workerID Worker ID in the snowflake - * @property {number} processID Process ID in the snowflake - * @property {number} increment Increment in the snowflake - * @property {string} binary Binary representation of the snowflake - */ - - /** - * Deconstructs a Discord snowflake. - * @param {Snowflake} snowflake Snowflake to deconstruct - * @returns {DeconstructedSnowflake} Deconstructed snowflake - */ - static deconstruct(snowflake) { - const BINARY = SnowflakeUtil.idToBinary(snowflake).toString(2).padStart(64, "0"); - const res = { - timestamp: parseInt(BINARY.substring(0, 42), 2) + EPOCH, - workerID: parseInt(BINARY.substring(42, 47), 2), - processID: parseInt(BINARY.substring(47, 52), 2), - increment: parseInt(BINARY.substring(52, 64), 2), - binary: BINARY, - }; - Object.defineProperty(res, "date", { - get: function get() { - return new Date(this.timestamp); - }, - enumerable: true, - }); - return res; - } - - /** - * Discord's epoch value (2015-01-01T00:00:00.000Z). - * @type {number} - * @readonly - */ - static get EPOCH() { - return EPOCH; - } -} - -module.exports = SnowflakeUtil; diff --git a/src/Util.ts b/src/Util.ts deleted file mode 100644 index 291372c1..00000000 --- a/src/Util.ts +++ /dev/null @@ -1,38 +0,0 @@ -import fs from "fs/promises"; -import "missing-native-js-functions"; - -export interface traverseDirectoryOptions { - dirname: string; - filter?: RegExp; - excludeDirs?: RegExp; - recursive?: boolean; -} - -const DEFAULT_EXCLUDE_DIR = /^\./; -const DEFAULT_FILTER = /^([^\.].*)\.js$/; - -export async function traverseDirectory( - options: traverseDirectoryOptions, - action: (path: string) => T -): Promise { - if (!options.filter) options.filter = DEFAULT_FILTER; - if (!options.excludeDirs) options.excludeDirs = DEFAULT_EXCLUDE_DIR; - - const routes = await fs.readdir(options.dirname); - const promises = []>routes.map(async (file) => { - const path = options.dirname + file; - const stat = await fs.lstat(path); - if (path.match(options.excludeDirs)) return; - - if (stat.isFile() && path.match(options.filter)) { - return action(path); - } else if (options.recursive && stat.isDirectory()) { - return traverseDirectory({ ...options, dirname: path + "/" }, action); - } - }); - const result = await Promise.all(promises); - - const t = <(T | undefined)[]>result.flat(); - - return t.filter((x) => x != undefined); -} diff --git a/src/index.ts b/src/index.ts index 64fca7e5..cdf88fd9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,14 @@ import { CDNServer } from "./Server"; +import dotenv from "dotenv"; +dotenv.config(); -const server = new CDNServer({ db: "" }); +if (process.env.STORAGE_LOCATION) { + if (!process.env.STORAGE_LOCATION.startsWith("/")) { + process.env.STORAGE_LOCATION = __dirname + "/../" + process.env.STORAGE_LOCATION; + } +} else process.env.STORAGE_LOCATION = __dirname + "/../files/"; + +const server = new CDNServer(); server .start() .then(() => { diff --git a/src/routes/attachments.ts b/src/routes/attachments.ts index 9f016174..87368e48 100644 --- a/src/routes/attachments.ts +++ b/src/routes/attachments.ts @@ -1,6 +1,7 @@ import { Router } from "express"; import multer from "multer"; -import Snowflake from "../Snowflake"; +import { Snowflake } from "@fosscord/server-util"; +import { storage } from "../util/Storage"; const multer_ = multer(); const router = Router(); @@ -11,11 +12,11 @@ type Attachment = { id: string; type: string; }; - router.post("/:filename", multer_.single("attachment"), async (req, res) => { const { buffer, mimetype } = req.file; const { filename } = req.params; - const { db } = req.cdn; + + // storage.set(filename, ); const File: Attachment = { filename, @@ -23,14 +24,9 @@ router.post("/:filename", multer_.single("attachment"), async (req, res) => { id: Snowflake.generate(), type: mimetype, }; - - if (!(await db.data.attachments.push(File))) throw new Error("Error uploading file"); - - return res.status(201).send({ success: true, message: "attachment uploaded", id: File.id, filename }); }); router.get("/:hash/:filename", async (req, res) => { - const { db } = req.cdn; const { hash, filename } = req.params; const File: Attachment = await db.data.attachments({ id: hash, filename: filename }).get(); @@ -41,7 +37,6 @@ router.get("/:hash/:filename", async (req, res) => { router.delete("/:hash/:filename", async (req, res) => { const { hash, filename } = req.params; - const { db } = req.cdn; await db.data.attachments({ id: hash, filename: filename }).delete(); return res.send({ success: true, message: "attachment deleted" }); diff --git a/src/routes/external.ts b/src/routes/external.ts index f75f6a66..045eb7da 100644 --- a/src/routes/external.ts +++ b/src/routes/external.ts @@ -2,8 +2,7 @@ import bodyParser from "body-parser"; import { Router } from "express"; import fetch from "node-fetch"; import cheerio from "cheerio"; -import btoa from "btoa"; -import { URL } from "url"; +import crypto from "crypto"; const router = Router(); @@ -30,25 +29,21 @@ const DEFAULT_FETCH_OPTIONS: any = { router.post("/", bodyParser.json(), async (req, res) => { if (!req.body) throw new Error("Invalid Body (url missing) \nExample: url:https://discord.com"); - const { db } = req.cdn; const { url } = req.body; - const ID = btoa(url); - - const cache = await db.data.crawler({ id: ID }).get(); - if (cache) return res.send(cache); + const hash = crypto.createHash("md5").update(url).digest("hex"); try { const request = await fetch(url, DEFAULT_FETCH_OPTIONS); const text = await request.text(); - const ツ: any = cheerio.load(text); + const $ = cheerio.load(text); - const ogTitle = ツ('meta[property="og:title"]').attr("content"); - const ogDescription = ツ('meta[property="og:description"]').attr("content"); - const ogImage = ツ('meta[property="og:image"]').attr("content"); - const ogUrl = ツ('meta[property="og:url"]').attr("content"); - const ogType = ツ('meta[property="og:type"]').attr("content"); + const ogTitle = $('meta[property="og:title"]').attr("content"); + const ogDescription = $('meta[property="og:description"]').attr("content"); + const ogImage = $('meta[property="og:image"]').attr("content"); + const ogUrl = $('meta[property="og:url"]').attr("content"); + const ogType = $('meta[property="og:type"]').attr("content"); const filename = new URL(url).host.split(".")[0]; @@ -72,7 +67,6 @@ router.post("/", bodyParser.json(), async (req, res) => { }); router.get("/:id/:filename", async (req, res) => { - const { db } = req.cdn; const { id, filename } = req.params; const { image, type } = await db.data.externals({ id: id }).get(); const imageBuffer = Buffer.from(image, "base64"); diff --git a/src/util/FileStorage.ts b/src/util/FileStorage.ts new file mode 100644 index 00000000..01be0050 --- /dev/null +++ b/src/util/FileStorage.ts @@ -0,0 +1,7 @@ +import { Storage } from "./Storage"; + +export class FileStorage implements Storage { + async get(path: string, prefix?: string) {} + + async set(path: string, value: any) {} +} diff --git a/src/util/Storage.ts b/src/util/Storage.ts new file mode 100644 index 00000000..ad00fbb7 --- /dev/null +++ b/src/util/Storage.ts @@ -0,0 +1,14 @@ +import { FileStorage } from "./FileStorage"; + +export interface Storage { + set(hash: string, data: any, prefix?: string): Promise; + get(hash: string, prefix?: string): Promise; +} + +var storage: Storage; + +if (process.env.STORAGE_PROVIDER === "file") { + storage = new FileStorage(); +} + +export { storage }; diff --git a/tsconfig.json b/tsconfig.json index 1422446c..08e39435 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,5 @@ { + "include": ["src/**/*.ts"], "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */