Merge pull request #162 from fosscord/feat--rate-limit

[Feature] Rate Limit

package.json

@@ -29,7 +29,7 @@
 	},
 	"homepage": "https://github.com/fosscord/fosscord-api#readme",
 	"dependencies": {
-		"@fosscord/server-util": "^1.3.20",
+		"@fosscord/server-util": "^1.3.23",
 		"@types/jest": "^26.0.22",
 		"@types/json-schema": "^7.0.7",
 		"ajv": "^8.4.0",
@@ -49,7 +49,6 @@
 		"i18next-http-middleware": "^3.1.3",
 		"i18next-node-fs-backend": "^2.1.3",
 		"image-size": "^1.0.0",
-		"ipdata": "^1.1.3",
 		"jsonwebtoken": "^8.5.1",
 		"lambert-server": "^1.2.5",
 		"missing-native-js-functions": "^1.2.6",
@@ -57,7 +56,8 @@
 		"mongoose-autopopulate": "^0.12.3",
 		"mongoose-long": "^0.3.2",
 		"multer": "^1.4.2",
-		"node-fetch": "^2.6.1"
+		"node-fetch": "^2.6.1",
+		"require_optional": "^1.0.1"
 	},
 	"devDependencies": {
 		"@types/bcrypt": "^3.0.0",

src/Server.ts

@@ -13,6 +13,7 @@ import express, { Router, Request, Response } from "express";
 import fetch, { Response as FetchResponse } from "node-fetch";
 import mongoose from "mongoose";
 import path from "path";
+import RateLimit from "./middlewares/RateLimit";
 
 // this will return the new updated document for findOneAndUpdate
 mongoose.set("returnOriginal", false); // https://mongoosejs.com/docs/api/model.html#model_Model.findOneAndUpdate
@@ -54,7 +55,8 @@ export class FosscordServer extends Server {
 			db.collection("roles").createIndex({ id: 1 }, { unique: true }),
 			db.collection("emojis").createIndex({ id: 1 }, { unique: true }),
 			db.collection("invites").createIndex({ code: 1 }, { unique: true }),
-			db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }) // after 0 seconds of expires_at the invite will get delete
+			db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }), // after 0 seconds of expires_at the invite will get delete
+			db.collection("ratelimits").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 })
 		]);
 	}
 
@@ -91,6 +93,10 @@ export class FosscordServer extends Server {
 		const prefix = Router();
 		// @ts-ignore
 		this.app = prefix;
+		prefix.use(RateLimit({ bucket: "global", count: 10, error: 10, window: 5, bot: 250 }));
+		prefix.use("/guilds/:id", RateLimit({ count: 10, window: 5 }));
+		prefix.use("/webhooks/:id", RateLimit({ count: 10, window: 5 }));
+		prefix.use("/channels/:id", RateLimit({ count: 10, window: 5 }));
 
 		this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/"));
 		app.use("/api", prefix); // allow unversioned requests

src/middlewares/Authentication.ts

@@ -11,10 +11,14 @@ export const NO_AUTHORIZATION_ROUTES = [
 	/^\/api(\/v\d+)?\/guilds\/\d+\/widget\.(json|png)/
 ];
 
+export const API_PREFIX = /^\/api(\/v\d+)?/;
+export const API_PREFIX_TRAILING_SLASH = /^\/api(\/v\d+)?\//;
+
 declare global {
 	namespace Express {
 		interface Request {
 			user_id: any;
+			user_bot: boolean;
 			token: any;
 		}
 	}
@@ -23,17 +27,19 @@ declare global {
 export async function Authentication(req: Request, res: Response, next: NextFunction) {
 	if (req.method === "OPTIONS") return res.sendStatus(204);
 	if (!req.url.startsWith("/api")) return next();
-	if (req.url.startsWith("/api/v8/invites") && req.method === "GET") return next();
+	const apiPath = req.url.replace(API_PREFIX, "");
+	if (apiPath.startsWith("/invites") && req.method === "GET") return next();
 	if (NO_AUTHORIZATION_ROUTES.some((x) => x.test(req.url))) return next();
 	if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401));
 
 	try {
 		const { jwtSecret } = Config.get().security;
 
-		const decoded: any = await checkToken(req.headers.authorization, jwtSecret);
+		const { decoded, user }: any = await checkToken(req.headers.authorization, jwtSecret);
 
 		req.token = decoded;
 		req.user_id = decoded.id;
+		req.user_bot = user.bot;
 		return next();
 	} catch (error) {
 		return next(new HTTPError(error.toString(), 400));

src/middlewares/RateLimit.ts

@@ -1,7 +1,16 @@
-import { db, MongooseCache } from "@fosscord/server-util";
+import { db, MongooseCache, Bucket } from "@fosscord/server-util";
 import { NextFunction, Request, Response } from "express";
+import { getIpAdress } from "../util/ipAddress";
+import { API_PREFIX_TRAILING_SLASH } from "./Authentication";
 
-const Cache = new MongooseCache(db.collection("ratelimits"), [{ $match: { blocked: true } }], { onlyEvents: false, array: true });
+const Cache = new MongooseCache(
+	db.collection("ratelimits"),
+	[
+		// TODO: uncomment $match and fix error: not receiving change events
+		// { $match: { blocked: true } }
+	],
+	{ onlyEvents: false, array: true }
+);
 
 // Docs: https://discord.com/developers/docs/topics/rate-limits
 
@@ -22,10 +31,100 @@ TODO: use config values
 
 */
 
-export default function RateLimit(opts: { bucket?: string; window: number; count: number }) {
+export default function RateLimit(opts: {
+	bucket?: string;
+	window: number;
+	count: number;
+	bot?: number;
+	error?: number;
+	webhook?: number;
+	oauth?: number;
+	GET?: number;
+	MODIFY?: number;
+}) {
 	Cache.init(); // will only initalize it once
 
 	return async (req: Request, res: Response, next: NextFunction) => {
+		const bucket_id = opts.bucket || req.path.replace(API_PREFIX_TRAILING_SLASH, "");
+		const user_id = req.user_id || getIpAdress(req);
+		var max_hits = opts.count;
+		if (opts.bot && req.user_bot) max_hits = opts.bot;
+		if (opts.GET && ["GET", "OPTIONS", "HEAD"].includes(req.method)) max_hits = opts.GET;
+		else if (opts.MODIFY && ["POST", "DELETE", "PATCH", "PUT"].includes(req.method)) max_hits = opts.MODIFY;
+
+		const offender = Cache.data?.find((x: Bucket) => x.user_id == user_id && x.id === bucket_id) as Bucket | null;
+
+		if (offender && offender.blocked) {
+			const reset = offender.expires_at.getTime();
+			const resetAfterMs = reset - Date.now();
+			const resetAfterSec = resetAfterMs / 1000;
+			const global = bucket_id === "global";
+			console.log("blocked", { resetAfterMs });
+
+			if (resetAfterMs > 0) {
+				return (
+					res
+						.status(429)
+						.set("X-RateLimit-Limit", `${max_hits}`)
+						.set("X-RateLimit-Remaining", "0")
+						.set("X-RateLimit-Reset", `${reset}`)
+						.set("X-RateLimit-Reset-After", `${resetAfterSec}`)
+						.set("X-RateLimit-Global", `${global}`)
+						.set("Retry-After", `${Math.ceil(resetAfterSec)}`)
+						.set("X-RateLimit-Bucket", `${bucket_id}`)
+						// TODO: error rate limit message translation
+						.send({ message: "You are being rate limited.", retry_after: resetAfterSec, global })
+				);
+			} else {
+				// mongodb ttl didn't update yet -> manually update/delete
+				db.collection("ratelimits").updateOne(
+					{ id: bucket_id, user_id },
+					{ $set: { hits: 0, expires_at: new Date(Date.now() + opts.window * 1000), blocked: false } }
+				);
+			}
+		}
 		next();
+
+		if (opts.error) {
+			res.once("finish", () => {
+				// check if error and increment error rate limit
+				if (res.statusCode >= 400) {
+					// TODO: use config rate limit values
+					return hitRoute({ bucket_id: "error", user_id, max_hits: opts.error as number, window: opts.window });
+				}
+			});
+		}
+
+		return hitRoute({ user_id, bucket_id, max_hits, window: opts.window });
 	};
 }
+
+function hitRoute(opts: { user_id: string; bucket_id: string; max_hits: number; window: number }) {
+	return db.collection("ratelimits").updateOne(
+		{ id: opts.bucket_id, user_id: opts.user_id },
+		[
+			{
+				$replaceRoot: {
+					newRoot: {
+						// similar to $setOnInsert
+						$mergeObjects: [
+							{
+								id: opts.bucket_id,
+								user_id: opts.user_id,
+								expires_at: new Date(Date.now() + opts.window * 1000)
+							},
+							"$$ROOT"
+						]
+					}
+				}
+			},
+			{
+				$set: {
+					hits: { $sum: [{ $ifNull: ["$hits", 0] }, 1] },
+					blocked: { $gt: ["$hits", opts.max_hits] }
+				}
+			}
+		],
+		{ upsert: true }
+	);
+}

src/routes/guilds/#guild_id/widget.png.ts

@@ -1,9 +1,8 @@
 import { Request, Response, Router } from "express";
 import { GuildModel } from "@fosscord/server-util";
 import { HTTPError } from "lambert-server";
-import { Image } from "canvas";
 import fs from "fs";
-import path from "path"
+import path from "path";
 
 const router: Router = Router();
 
@@ -35,7 +34,7 @@ router.get("/", async (req: Request, res: Response) => {
 	const sizeOf = require("image-size");
 
 	// TODO: Widget style templates need Fosscord branding
-	const source = path.join(__dirname, "..", "..", "..", "..", "assets","widget", `${style}.png`)
+	const source = path.join(__dirname, "..", "..", "..", "..", "assets", "widget", `${style}.png`);
 	if (!fs.existsSync(source)) {
 		throw new HTTPError("Widget template does not exist.", 400);
 	}
@@ -85,16 +84,17 @@ router.get("/", async (req: Request, res: Response) => {
 });
 
 async function drawIcon(canvas: any, x: number, y: number, scale: number, icon: string) {
-	const img = new Image();
+	// @ts-ignore
+	const img = new require("canvas").Image();
 	img.src = icon;
-	
+
 	// Do some canvas clipping magic!
 	canvas.save();
 	canvas.beginPath();
 
 	const r = scale / 2; // use scale to determine radius
 	canvas.arc(x + r, y + r, r, 0, 2 * Math.PI, false); // start circle at x, and y coords + radius to find center
-	
+
 	canvas.clip();
 	canvas.drawImage(img, x, y, scale, scale);
 

src/util/ipAddress.ts

@@ -1,13 +1,7 @@
 import { Config } from "@fosscord/server-util";
 import { Request } from "express";
 // use ipdata package instead of simple fetch because of integrated caching
-import IPData, { LookupResponse } from "ipdata";
-
-var ipdata: IPData;
-const cacheConfig = {
-	max: 1000, // max size
-	maxAge: 1000 * 60 * 60 * 24 // max age in ms (i.e. one day)
-};
+import fetch from "node-fetch";
 
 const exampleData = {
 	ip: "",
@@ -66,15 +60,14 @@ const exampleData = {
 	status: 200
 };
 
-export async function IPAnalysis(ip: string): Promise<LookupResponse> {
+export async function IPAnalysis(ip: string): Promise<typeof exampleData> {
 	const { ipdataApiKey } = Config.get().security;
 	if (!ipdataApiKey) return { ...exampleData, ip };
-	if (!ipdata) ipdata = new IPData(ipdataApiKey, cacheConfig);
 
-	return await ipdata.lookup(ip);
+	return (await fetch(`https://api.ipdata.co/${ip}?api-key=${ipdataApiKey}`)).json();
 }
 
-export function isProxy(data: LookupResponse) {
+export function isProxy(data: typeof exampleData) {
 	if (!data || !data.asn || !data.threat) return false;
 	if (data.asn.type !== "isp") return true;
 	if (Object.values(data.threat).some((x) => x)) return true;