From efc522a0c6a08e60de5efce0e076d61445fd9627 Mon Sep 17 00:00:00 2001 From: Flam3rboy <34555296+Flam3rboy@users.noreply.github.com> Date: Thu, 1 Jul 2021 09:33:54 +0200 Subject: [PATCH] :construction: rate limit --- package-lock.json | Bin 653984 -> 643630 bytes package.json | 3 +- src/Server.ts | 8 ++- src/middlewares/Authentication.ts | 10 +++- src/middlewares/RateLimit.ts | 61 +++++++++++++++++++++- src/routes/guilds/#guild_id/widget.png.ts | 12 ++--- src/util/ipAddress.ts | 15 ++---- 7 files changed, 85 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index 90fb4674bb666c845948aef3b5a1a8c72570cfbe..0580a5ea7da51e099ef20bf166d5e5411182ba0e 100644 GIT binary patch delta 448 zcmZ4RRejwO^$DLD4JUqivH1(52GivGwKAJ!SmGHcPi7V0e2mr4Z~DhZM*Zm*rn72J zPV7+#^-VHyD@@P!P4zSi(@rfh%`FJb_9{%x%nJ7l$tp9q^ocSn3Mxx?OR3U!Dfcfo zHw`v+G%EA)4Jr(Xu*?Z{GAj+SjPmzQF%2#W%nCA#jPmoa4Bq@Occ%Wc=-Wo>M+U;DrOqn6u7dA1eFmCH&Ix8}rFNjHMyTbye+1s{rM=ghgO_!*mw$<^6>$%eIG0uzJxP8VV zcB}c*Q_|T*w#!E_voLOd>dYY{zx{49hvgNB=M)+^xh7B9Ex3JxC8yMc?Ww0YtqlNz C45mo{ delta 5383 zcmdUzeT>`W9mnOor?ix#Yp*SBFL!-aa~Io*^Tu|9yuZKOd1Hce9OvT1j*~cXUKnkm znl=f6T0(i4wm}a%HfaOVRW#G~Ps_A1F%aB@_SZI1t!QWxqg^*?XcEjxdp#)AR!zIX z{;~Z0Y)ilA`+NKNdEwOllYiZR>JN+ys2a^nQiy1 zaE(ahtC?=QRmmEIDQ6rOv&FQ_=d>i7VmKG!;%$btTdFKh_p8x(GF|S$UZV$FET#jU zcDcy4N+cCbwz_9d;}?%9$S%W!2VYWch3p{lM_k9ia5%SWYt7k%GIde6yVVcg&>tyqwN#BIX1)!UZTtq9vn%V zTeQVRScAZwMohj1wCQXqox;RqB)nr->MY1}9rrOedPgfAz zY>jS`1%@3hOH7w8>X@2NL=47}ggP(Z)Ny>1mW(C7$&oFFugyx|)4Fo2RTu7r;YFrZ zZWjy`&t2WzwQIf2vakt55isGD876s_$aHfL0h!fuOxG@k@V9X; zrUNfbs`gEhwQH$`Vg}@@j@lW4;p!|sw@A0ZI)@@Ue8~d6T;C{dTGCe#q>Tx6z6Q>p zmu~^_vnmDn1EzxYMf!#p1X|kBKe5hX;L;)aZlV%P)o5d*(DUfwrpQZQo}cD$S`YUE*;oswl8Hz^PA&Gzyq9gntAo?H z4jg(;u2Nt1QUKFB<>X)?Ya6z|3#lJi|MAd(<$-z&xCF};VA-rd_Lgo4oaFzFra2V6 z_=a*1A>|^AxtM-mY_!a73?uPu!JF^vy{%q0jfzco)Y53uY`0a)B|;e_6hksb94=Qq zwyx+SVKat=t9iCcN-AI^!{%^r%bx}lUxg5G@&lDdHZu>F3W{0q%)8LkZ8thGc>bgP zgEtSEz=Bn?6TBEyZ3ZwUCqd+KXwQikp#7iv04!7TrvTDX>>Kv?$f`JKTvl!eM~-O~ z;P9W7@Fu1%1$B|`N*i9e1?UIrX>iV_RpL@Y1-pKK%ukewiYwlX)WdG9>uH$G1p;@m z{s@|~xSWKU=_YE`JlA3BokhxS5gkRV!{LkAai3odCiR3M=wV0ILe$G4J(&Kr!VEro zT()a?u`AmPL3Qx-!}6_9v>*kDRUvre)%G@aXw>I`<3q*t;LSUfc&jUPX*c6AQ9*yX zmBGyci=MTXOCsM%maJ_bQ;C&3SUYY{!r>I>^D>U2%i=4g>t2VqO9a@(28tD(Vmsvy zGNyK^oJ8T+Sop)^b~P@CQOjB_2S%7<>SMb}>JrzhgyoB*P#)Pa18Oy(=9Thco9#J@ zq{{T@nOXV-U3tH*|`eOsK%v;0jMUMR*joG3{GvA&&p6UIDJ_S z1O7v`5*+xOdLx7mjz6a(Y~2`@^oy=aji-t!Gga?o6WMg1w|2wiF!qZHd}nYzQY^K zMy=hZ(ow0@eT*z#6vy>mp#Emt}V!$RwQIZy%-bLOHpaisiaOrDXKxjs@9x{Q% z>vG+7Z!TO^?0cdoITIt5>cKZW2f&^0C~lQYS`IE~rU|Ut^x6cfPFKogY_ZXIdMIDS z+H(gyB?f_QdRyO}%$KA2ys=v0{6VZNn)1ut$bn?e>m#uL$+v0kty`D_;(EN?sr zzV)OOf|C!)EI=24=D?p=^T9iI@c#YisGfD;NIxYsc|0RNX&9it+)H$_jY}lxvaAI9=X)h3GAEu0Y9c zwIgsfU0{41u~^Fa+D;UUGkDSL?xee^G!~_D{z^Ym6RKi!B~1FwCzjw3RZuqUMvw!b0eHBnVfLTR*Z{Or5uWIY^fHRTQ?!Hi&_q>eki5ud;6kn5a#~;trS%Oi8gToQ zAw8et&7jH7(m0WLK(tHgNZC)(}9 z`(IMYxz9H(UdENPVE!?+7McflO=_J|^S=eW|ET)lhzn1E*Dos9f%;}?cl8f7o4{e6 z`q>czD#5+GwbR3|%&7nF{~U0jG+%duc2)h}XH0-2F+Vp1_=sj7czBmm3yNDcTNER< z1k2mB$P;(SH$y0x__=y!ZQu_~zpRyhh!pA#niYrzji_q!Dg&OHRii5mc={vF-Zf&t zrK6G$qr6r3NS0-U7C3iVGo>7}Gnk&xYy}ZgX&SR1c#2f+TO;tB_?~L_bq`RRb}K+` zk!ivAXQld@dsTb8gnnZY)&%{)`HRYfC!UdQ1G)v35}e}|>sKLgC_&)1iO+=iMpdG@ z{)@PLQ0n!HL+bgb|6|gV4k*@u^hRewAihhp1uXm!(hi^Y$Q2N{Yo}%dIQLuCM(~{UY)Yd?x+x diff --git a/package.json b/package.json index 0b1adb3b..2d1882b4 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/fosscord/fosscord-api#readme", "dependencies": { - "@fosscord/server-util": "^1.3.20", + "@fosscord/server-util": "^1.3.21", "@types/jest": "^26.0.22", "@types/json-schema": "^7.0.7", "ajv": "^8.4.0", @@ -49,7 +49,6 @@ "i18next-http-middleware": "^3.1.3", "i18next-node-fs-backend": "^2.1.3", "image-size": "^1.0.0", - "ipdata": "^1.1.3", "jsonwebtoken": "^8.5.1", "lambert-server": "^1.2.5", "missing-native-js-functions": "^1.2.6", diff --git a/src/Server.ts b/src/Server.ts index c1e66bfd..54c8db0d 100644 --- a/src/Server.ts +++ b/src/Server.ts @@ -13,6 +13,7 @@ import express, { Router, Request, Response } from "express"; import fetch, { Response as FetchResponse } from "node-fetch"; import mongoose from "mongoose"; import path from "path"; +import RateLimit from "./middlewares/RateLimit"; // this will return the new updated document for findOneAndUpdate mongoose.set("returnOriginal", false); // https://mongoosejs.com/docs/api/model.html#model_Model.findOneAndUpdate @@ -54,7 +55,8 @@ export class FosscordServer extends Server { db.collection("roles").createIndex({ id: 1 }, { unique: true }), db.collection("emojis").createIndex({ id: 1 }, { unique: true }), db.collection("invites").createIndex({ code: 1 }, { unique: true }), - db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }) // after 0 seconds of expires_at the invite will get delete + db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }), // after 0 seconds of expires_at the invite will get delete + db.collection("ratelimits").createIndex({ created_at: 1 }, { expireAfterSeconds: 1000 }) ]); } @@ -67,6 +69,7 @@ export class FosscordServer extends Server { this.app.use(CORS); this.app.use(Authentication); + this.app.use(RateLimit({ count: 10, error: 10, window: 5 })); this.app.use(BodyParser({ inflate: true, limit: 1024 * 1024 * 2 })); const languages = await fs.readdir(path.join(__dirname, "..", "locales")); const namespaces = await fs.readdir(path.join(__dirname, "..", "locales", "en")); @@ -91,6 +94,9 @@ export class FosscordServer extends Server { const prefix = Router(); // @ts-ignore this.app = prefix; + prefix.use("/guilds/:id", RateLimit({ count: 10, window: 5 })); + prefix.use("/webhooks/:id", RateLimit({ count: 10, window: 5 })); + prefix.use("/channels/:id", RateLimit({ count: 10, window: 5 })); this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/")); app.use("/api", prefix); // allow unversioned requests diff --git a/src/middlewares/Authentication.ts b/src/middlewares/Authentication.ts index 4b38f1d4..76b335ad 100644 --- a/src/middlewares/Authentication.ts +++ b/src/middlewares/Authentication.ts @@ -11,10 +11,14 @@ export const NO_AUTHORIZATION_ROUTES = [ /^\/api(\/v\d+)?\/guilds\/\d+\/widget\.(json|png)/ ]; +export const API_PREFIX = /^\/api(\/v\d+)?/; +export const API_PREFIX_TRAILING_SLASH = /^\/api(\/v\d+)?\//; + declare global { namespace Express { interface Request { user_id: any; + user_bot: boolean; token: any; } } @@ -23,17 +27,19 @@ declare global { export async function Authentication(req: Request, res: Response, next: NextFunction) { if (req.method === "OPTIONS") return res.sendStatus(204); if (!req.url.startsWith("/api")) return next(); - if (req.url.startsWith("/api/v8/invites") && req.method === "GET") return next(); + const apiPath = req.url.replace(API_PREFIX, ""); + if (apiPath.startsWith("/invites") && req.method === "GET") return next(); if (NO_AUTHORIZATION_ROUTES.some((x) => x.test(req.url))) return next(); if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401)); try { const { jwtSecret } = Config.get().security; - const decoded: any = await checkToken(req.headers.authorization, jwtSecret); + const { decoded, user }: any = await checkToken(req.headers.authorization, jwtSecret); req.token = decoded; req.user_id = decoded.id; + req.user_bot = user.bot; return next(); } catch (error) { return next(new HTTPError(error.toString(), 400)); diff --git a/src/middlewares/RateLimit.ts b/src/middlewares/RateLimit.ts index e610d55b..ab69113e 100644 --- a/src/middlewares/RateLimit.ts +++ b/src/middlewares/RateLimit.ts @@ -1,5 +1,6 @@ -import { db, MongooseCache } from "@fosscord/server-util"; +import { db, MongooseCache, Bucket } from "@fosscord/server-util"; import { NextFunction, Request, Response } from "express"; +import { API_PREFIX, API_PREFIX_TRAILING_SLASH } from "./Authentication"; const Cache = new MongooseCache(db.collection("ratelimits"), [{ $match: { blocked: true } }], { onlyEvents: false, array: true }); @@ -22,10 +23,66 @@ TODO: use config values */ -export default function RateLimit(opts: { bucket?: string; window: number; count: number }) { +export default function RateLimit(opts: { + bucket?: string; + window: number; + count: number; + bot?: number; + error?: number; + webhook?: number; + oauth?: number; + GET?: number; + MODIFY?: number; +}) { Cache.init(); // will only initalize it once return async (req: Request, res: Response, next: NextFunction) => { + const bucket_id = req.path.replace(API_PREFIX_TRAILING_SLASH, ""); + const user_id = req.user_id; + const max_hits = req.user_bot ? opts.bot : opts.count; + const offender = Cache.data.find((x: Bucket) => x.user && x.id === bucket_id) as Bucket | null; + + if (offender && offender.blocked) { + const reset = offender.created_at.getTime() + opts.window; + const resetAfterMs = reset - Date.now(); + const resetAfterSec = resetAfterMs / 1000; + const global = bucket_id === "global"; + + return ( + res + .status(429) + .set("X-RateLimit-Limit", `${max_hits}`) + .set("X-RateLimit-Remaining", "0") + .set("X-RateLimit-Reset", `${reset}`) + .set("X-RateLimit-Reset-After", `${resetAfterSec}`) + .set("X-RateLimit-Global", `${global}`) + .set("Retry-After", `${Math.ceil(resetAfterSec)}`) + .set("X-RateLimit-Bucket", `${bucket_id}`) + // TODO: error rate limit message translation + .send({ message: "You are being rate limited.", retry_after: resetAfterSec, global }) + ); + } next(); + console.log(req.route); + + if (opts.error) { + res.once("finish", () => { + // check if error and increment error rate limit + }); + } + + db.collection("ratelimits").updateOne( + { bucket: bucket_id }, + { + $set: { + id: bucket_id, + user_id, + created_at: new Date(), + $cond: { if: { $gt: ["$hits", max_hits] }, then: true, else: false } + }, + $inc: { hits: 1 } + }, + { upsert: true } + ); }; } diff --git a/src/routes/guilds/#guild_id/widget.png.ts b/src/routes/guilds/#guild_id/widget.png.ts index ea947c5d..839a8129 100644 --- a/src/routes/guilds/#guild_id/widget.png.ts +++ b/src/routes/guilds/#guild_id/widget.png.ts @@ -1,9 +1,8 @@ import { Request, Response, Router } from "express"; import { GuildModel } from "@fosscord/server-util"; import { HTTPError } from "lambert-server"; -import { Image } from "canvas"; import fs from "fs"; -import path from "path" +import path from "path"; const router: Router = Router(); @@ -35,7 +34,7 @@ router.get("/", async (req: Request, res: Response) => { const sizeOf = require("image-size"); // TODO: Widget style templates need Fosscord branding - const source = path.join(__dirname, "..", "..", "..", "..", "assets","widget", `${style}.png`) + const source = path.join(__dirname, "..", "..", "..", "..", "assets", "widget", `${style}.png`); if (!fs.existsSync(source)) { throw new HTTPError("Widget template does not exist.", 400); } @@ -85,16 +84,17 @@ router.get("/", async (req: Request, res: Response) => { }); async function drawIcon(canvas: any, x: number, y: number, scale: number, icon: string) { - const img = new Image(); + // @ts-ignore + const img = new require("canvas").Image(); img.src = icon; - + // Do some canvas clipping magic! canvas.save(); canvas.beginPath(); const r = scale / 2; // use scale to determine radius canvas.arc(x + r, y + r, r, 0, 2 * Math.PI, false); // start circle at x, and y coords + radius to find center - + canvas.clip(); canvas.drawImage(img, x, y, scale, scale); diff --git a/src/util/ipAddress.ts b/src/util/ipAddress.ts index 74090d61..f2c8fd4d 100644 --- a/src/util/ipAddress.ts +++ b/src/util/ipAddress.ts @@ -1,13 +1,7 @@ import { Config } from "@fosscord/server-util"; import { Request } from "express"; // use ipdata package instead of simple fetch because of integrated caching -import IPData, { LookupResponse } from "ipdata"; - -var ipdata: IPData; -const cacheConfig = { - max: 1000, // max size - maxAge: 1000 * 60 * 60 * 24 // max age in ms (i.e. one day) -}; +import fetch from "node-fetch"; const exampleData = { ip: "", @@ -66,15 +60,14 @@ const exampleData = { status: 200 }; -export async function IPAnalysis(ip: string): Promise { +export async function IPAnalysis(ip: string): Promise { const { ipdataApiKey } = Config.get().security; if (!ipdataApiKey) return { ...exampleData, ip }; - if (!ipdata) ipdata = new IPData(ipdataApiKey, cacheConfig); - return await ipdata.lookup(ip); + return (await fetch(`https://api.ipdata.co/${ip}?api-key=${ipdataApiKey}`)).json(); } -export function isProxy(data: LookupResponse) { +export function isProxy(data: typeof exampleData) { if (!data || !data.asn || !data.threat) return false; if (data.asn.type !== "isp") return true; if (Object.values(data.threat).some((x) => x)) return true;