🚧 rate limit

package.json

@@ -29,7 +29,7 @@
 	},
 	"homepage": "https://github.com/fosscord/fosscord-api#readme",
 	"dependencies": {
-		"@fosscord/server-util": "^1.3.20",
+		"@fosscord/server-util": "^1.3.21",
 		"@types/jest": "^26.0.22",
 		"@types/json-schema": "^7.0.7",
 		"ajv": "^8.4.0",
@@ -49,7 +49,6 @@
 		"i18next-http-middleware": "^3.1.3",
 		"i18next-node-fs-backend": "^2.1.3",
 		"image-size": "^1.0.0",
-		"ipdata": "^1.1.3",
 		"jsonwebtoken": "^8.5.1",
 		"lambert-server": "^1.2.5",
 		"missing-native-js-functions": "^1.2.6",

src/Server.ts

@@ -13,6 +13,7 @@ import express, { Router, Request, Response } from "express";
 import fetch, { Response as FetchResponse } from "node-fetch";
 import mongoose from "mongoose";
 import path from "path";
+import RateLimit from "./middlewares/RateLimit";
 
 // this will return the new updated document for findOneAndUpdate
 mongoose.set("returnOriginal", false); // https://mongoosejs.com/docs/api/model.html#model_Model.findOneAndUpdate
@@ -54,7 +55,8 @@ export class FosscordServer extends Server {
 			db.collection("roles").createIndex({ id: 1 }, { unique: true }),
 			db.collection("emojis").createIndex({ id: 1 }, { unique: true }),
 			db.collection("invites").createIndex({ code: 1 }, { unique: true }),
-			db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }) // after 0 seconds of expires_at the invite will get delete
+			db.collection("invites").createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 }), // after 0 seconds of expires_at the invite will get delete
+			db.collection("ratelimits").createIndex({ created_at: 1 }, { expireAfterSeconds: 1000 })
 		]);
 	}
 
@@ -67,6 +69,7 @@ export class FosscordServer extends Server {
 
 		this.app.use(CORS);
 		this.app.use(Authentication);
+		this.app.use(RateLimit({ count: 10, error: 10, window: 5 }));
 		this.app.use(BodyParser({ inflate: true, limit: 1024 * 1024 * 2 }));
 		const languages = await fs.readdir(path.join(__dirname, "..", "locales"));
 		const namespaces = await fs.readdir(path.join(__dirname, "..", "locales", "en"));
@@ -91,6 +94,9 @@ export class FosscordServer extends Server {
 		const prefix = Router();
 		// @ts-ignore
 		this.app = prefix;
+		prefix.use("/guilds/:id", RateLimit({ count: 10, window: 5 }));
+		prefix.use("/webhooks/:id", RateLimit({ count: 10, window: 5 }));
+		prefix.use("/channels/:id", RateLimit({ count: 10, window: 5 }));
 
 		this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/"));
 		app.use("/api", prefix); // allow unversioned requests

src/middlewares/Authentication.ts

@@ -11,10 +11,14 @@ export const NO_AUTHORIZATION_ROUTES = [
 	/^\/api(\/v\d+)?\/guilds\/\d+\/widget\.(json|png)/
 ];
 
+export const API_PREFIX = /^\/api(\/v\d+)?/;
+export const API_PREFIX_TRAILING_SLASH = /^\/api(\/v\d+)?\//;
+
 declare global {
 	namespace Express {
 		interface Request {
 			user_id: any;
+			user_bot: boolean;
 			token: any;
 		}
 	}
@@ -23,17 +27,19 @@ declare global {
 export async function Authentication(req: Request, res: Response, next: NextFunction) {
 	if (req.method === "OPTIONS") return res.sendStatus(204);
 	if (!req.url.startsWith("/api")) return next();
-	if (req.url.startsWith("/api/v8/invites") && req.method === "GET") return next();
+	const apiPath = req.url.replace(API_PREFIX, "");
+	if (apiPath.startsWith("/invites") && req.method === "GET") return next();
 	if (NO_AUTHORIZATION_ROUTES.some((x) => x.test(req.url))) return next();
 	if (!req.headers.authorization) return next(new HTTPError("Missing Authorization Header", 401));
 
 	try {
 		const { jwtSecret } = Config.get().security;
 
-		const decoded: any = await checkToken(req.headers.authorization, jwtSecret);
+		const { decoded, user }: any = await checkToken(req.headers.authorization, jwtSecret);
 
 		req.token = decoded;
 		req.user_id = decoded.id;
+		req.user_bot = user.bot;
 		return next();
 	} catch (error) {
 		return next(new HTTPError(error.toString(), 400));

src/middlewares/RateLimit.ts

@@ -1,5 +1,6 @@
-import { db, MongooseCache } from "@fosscord/server-util";
+import { db, MongooseCache, Bucket } from "@fosscord/server-util";
 import { NextFunction, Request, Response } from "express";
+import { API_PREFIX, API_PREFIX_TRAILING_SLASH } from "./Authentication";
 
 const Cache = new MongooseCache(db.collection("ratelimits"), [{ $match: { blocked: true } }], { onlyEvents: false, array: true });
 
@@ -22,10 +23,66 @@ TODO: use config values
 
 */
 
-export default function RateLimit(opts: { bucket?: string; window: number; count: number }) {
+export default function RateLimit(opts: {
+	bucket?: string;
+	window: number;
+	count: number;
+	bot?: number;
+	error?: number;
+	webhook?: number;
+	oauth?: number;
+	GET?: number;
+	MODIFY?: number;
+}) {
 	Cache.init(); // will only initalize it once
 
 	return async (req: Request, res: Response, next: NextFunction) => {
+		const bucket_id = req.path.replace(API_PREFIX_TRAILING_SLASH, "");
+		const user_id = req.user_id;
+		const max_hits = req.user_bot ? opts.bot : opts.count;
+		const offender = Cache.data.find((x: Bucket) => x.user && x.id === bucket_id) as Bucket | null;
+
+		if (offender && offender.blocked) {
+			const reset = offender.created_at.getTime() + opts.window;
+			const resetAfterMs = reset - Date.now();
+			const resetAfterSec = resetAfterMs / 1000;
+			const global = bucket_id === "global";
+
+			return (
+				res
+					.status(429)
+					.set("X-RateLimit-Limit", `${max_hits}`)
+					.set("X-RateLimit-Remaining", "0")
+					.set("X-RateLimit-Reset", `${reset}`)
+					.set("X-RateLimit-Reset-After", `${resetAfterSec}`)
+					.set("X-RateLimit-Global", `${global}`)
+					.set("Retry-After", `${Math.ceil(resetAfterSec)}`)
+					.set("X-RateLimit-Bucket", `${bucket_id}`)
+					// TODO: error rate limit message translation
+					.send({ message: "You are being rate limited.", retry_after: resetAfterSec, global })
+			);
+		}
 		next();
+		console.log(req.route);
+
+		if (opts.error) {
+			res.once("finish", () => {
+				// check if error and increment error rate limit
+			});
+		}
+
+		db.collection("ratelimits").updateOne(
+			{ bucket: bucket_id },
+			{
+				$set: {
+					id: bucket_id,
+					user_id,
+					created_at: new Date(),
+					$cond: { if: { $gt: ["$hits", max_hits] }, then: true, else: false }
+				},
+				$inc: { hits: 1 }
+			},
+			{ upsert: true }
+		);
 	};
 }

src/routes/guilds/#guild_id/widget.png.ts

@@ -1,9 +1,8 @@
 import { Request, Response, Router } from "express";
 import { GuildModel } from "@fosscord/server-util";
 import { HTTPError } from "lambert-server";
-import { Image } from "canvas";
 import fs from "fs";
-import path from "path"
+import path from "path";
 
 const router: Router = Router();
 
@@ -35,7 +34,7 @@ router.get("/", async (req: Request, res: Response) => {
 	const sizeOf = require("image-size");
 
 	// TODO: Widget style templates need Fosscord branding
-	const source = path.join(__dirname, "..", "..", "..", "..", "assets","widget", `${style}.png`)
+	const source = path.join(__dirname, "..", "..", "..", "..", "assets", "widget", `${style}.png`);
 	if (!fs.existsSync(source)) {
 		throw new HTTPError("Widget template does not exist.", 400);
 	}
@@ -85,7 +84,8 @@ router.get("/", async (req: Request, res: Response) => {
 });
 
 async function drawIcon(canvas: any, x: number, y: number, scale: number, icon: string) {
-	const img = new Image();
+	// @ts-ignore
+	const img = new require("canvas").Image();
 	img.src = icon;
 
 	// Do some canvas clipping magic!

src/util/ipAddress.ts

@@ -1,13 +1,7 @@
 import { Config } from "@fosscord/server-util";
 import { Request } from "express";
 // use ipdata package instead of simple fetch because of integrated caching
-import IPData, { LookupResponse } from "ipdata";
+import fetch from "node-fetch";
-
-var ipdata: IPData;
-const cacheConfig = {
-	max: 1000, // max size
-	maxAge: 1000 * 60 * 60 * 24 // max age in ms (i.e. one day)
-};
 
 const exampleData = {
 	ip: "",
@@ -66,15 +60,14 @@ const exampleData = {
 	status: 200
 };
 
-export async function IPAnalysis(ip: string): Promise<LookupResponse> {
+export async function IPAnalysis(ip: string): Promise<typeof exampleData> {
 	const { ipdataApiKey } = Config.get().security;
 	if (!ipdataApiKey) return { ...exampleData, ip };
-	if (!ipdata) ipdata = new IPData(ipdataApiKey, cacheConfig);
 
-	return await ipdata.lookup(ip);
+	return (await fetch(`https://api.ipdata.co/${ip}?api-key=${ipdataApiKey}`)).json();
 }
 
-export function isProxy(data: LookupResponse) {
+export function isProxy(data: typeof exampleData) {
 	if (!data || !data.asn || !data.threat) return false;
 	if (data.asn.type !== "isp") return true;
 	if (Object.values(data.threat).some((x) => x)) return true;