Merge pull request #147 from DiegoMagdaleno/master

New way to do config no more hardcoded localhost
This commit is contained in:
Flam3rboy 2021-05-23 20:48:25 +02:00 committed by GitHub
commit e1f0eb3c2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 412 additions and 128 deletions

4
.gitignore vendored
View File

@ -105,3 +105,7 @@ dist
.DS_STORE
src/ready.json
# Docker
.docker/config/*
!.docker/config/.keep

BIN
package-lock.json generated

Binary file not shown.

View File

@ -29,11 +29,18 @@
},
"homepage": "https://github.com/fosscord/fosscord-api#readme",
"dependencies": {
"@fosscord/server-util": "^1.2.6",
"@fosscord/server-util": "^1.3.1",
"@types/jest": "^26.0.22",
"@types/json-schema": "^7.0.7",
"ajv": "^8.4.0",
"ajv-formats": "^2.1.0",
"assert": "^1.5.0",
"atomically": "^1.7.0",
"bcrypt": "^5.0.1",
"body-parser": "^1.19.0",
"dot-prop": "^6.0.1",
"dotenv": "^8.2.0",
"env-paths": "^2.2.1",
"express": "^4.17.1",
"express-validator": "^6.9.2",
"i18next": "^19.8.5",

View File

@ -0,0 +1,93 @@
const { Snowflake } = require("@fosscord/server-util");
const crypto = require('crypto');
const fs = require('fs');
const defaultConfig = {
// TODO: Get the network interfaces dinamically
gateway: "ws://localhost",
general: {
instance_id: Snowflake.generate(),
},
permissions: {
user: {
createGuilds: true,
}
},
limits: {
user: {
maxGuilds: 100,
maxUsername: 32,
maxFriends: 1000,
},
guild: {
maxRoles: 250,
maxMembers: 250000,
maxChannels: 500,
maxChannelsInCategory: 50,
hideOfflineMember: 1000,
},
message: {
characters: 2000,
ttsCharacters: 200,
maxReactions: 20,
maxAttachmentSize: 8388608,
maxBulkDelete: 100,
},
channel: {
maxPins: 50,
maxTopic: 1024,
},
rate: {
ip: {
enabled: true,
count: 1000,
timespan: 1000 * 60 * 10,
},
routes: {},
},
},
security: {
jwtSecret: crypto.randomBytes(256).toString("base64"),
forwadedFor: null,
// forwadedFor: "X-Forwarded-For" // nginx/reverse proxy
// forwadedFor: "CF-Connecting-IP" // cloudflare:
captcha: {
enabled: false,
service: null,
sitekey: null,
secret: null,
},
},
login: {
requireCaptcha: false,
},
register: {
email: {
necessary: true,
allowlist: false,
blocklist: true,
domains: [], // TODO: efficiently save domain blocklist in database
// domains: fs.readFileSync(__dirname + "/blockedEmailDomains.txt", { encoding: "utf8" }).split("\n"),
},
dateOfBirth: {
necessary: true,
minimum: 13,
},
requireInvite: false,
requireCaptcha: true,
allowNewRegistration: true,
allowMultipleAccounts: true,
password: {
minLength: 8,
minNumbers: 2,
minUpperCase: 2,
minSymbols: 0,
blockInsecureCommonPasswords: false,
},
},
}
let data = JSON.stringify(defaultConfig);
fs.writeFileSync('./.docker/config/api.json', data);

View File

@ -3,7 +3,7 @@ import fs from "fs/promises";
import { Connection } from "mongoose";
import { Server, ServerOptions } from "lambert-server";
import { Authentication, CORS, GlobalRateLimit } from "./middlewares/";
import Config from "./util/Config";
import * as Config from "./util/Config";
import { db } from "@fosscord/server-util";
import i18next from "i18next";
import i18nextMiddleware, { I18next } from "i18next-http-middleware";
@ -55,7 +55,7 @@ export class FosscordServer extends Server {
await (db as Promise<Connection>);
await this.setupSchema();
console.log("[DB] connected");
await Promise.all([Config.init()]);
//await Promise.all([Config.init()]);
this.app.use(GlobalRateLimit);
this.app.use(Authentication);

View File

@ -1,6 +1,7 @@
import { NextFunction, Request, Response } from "express";
import { HTTPError } from "lambert-server";
import { checkToken } from "@fosscord/server-util";
import * as Config from "../util/Config"
export const NO_AUTHORIZATION_ROUTES = [
"/api/v8/auth/login",
@ -27,7 +28,10 @@ export async function Authentication(req: Request, res: Response, next: NextFunc
// TODO: check if user is banned/token expired
try {
const decoded: any = await checkToken(req.headers.authorization);
const { jwtSecret } = Config.apiConfig.getAll().security;
const decoded: any = await checkToken(req.headers.authorization, jwtSecret);
req.token = decoded;
req.user_id = decoded.id;

View File

@ -1,5 +1,6 @@
import { NextFunction, Request, Response } from "express";
import Config from "../util/Config";
import * as Config from '../util/Config'
import crypto from "crypto";
// TODO: use mongodb ttl index
// TODO: increment count on serverside
@ -43,7 +44,7 @@ export async function GlobalRateLimit(req: Request, res: Response, next: NextFun
}
export function getIpAdress(req: Request): string {
const { forwadedFor } = Config.get().security;
const { forwadedFor } = Config.apiConfig.getAll().security;
const ip = forwadedFor ? <string>req.headers[forwadedFor] : req.ip;
return ip.replaceAll(".", "_").replaceAll(":", "_");
}

View File

@ -3,7 +3,7 @@ import { check, FieldErrors, Length } from "../../util/instanceOf";
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import { UserModel } from "@fosscord/server-util";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { adjustEmail } from "./register";
const router: Router = Router();
@ -25,7 +25,9 @@ router.post(
const query: any[] = [{ phone: login }];
if (email) query.push({ email });
const config = Config.get();
// TODO: Rewrite this to have the proper config syntax on the new method
const config = Config.apiConfig.getAll();
if (config.login.requireCaptcha && config.security.captcha.enabled) {
if (!captcha_key) {
@ -69,7 +71,7 @@ export async function generateToken(id: string) {
return new Promise((res, rej) => {
jwt.sign(
{ id: id, iat },
Config.get().security.jwtSecret,
Config.apiConfig.getAll().security.jwtSecret,
{
algorithm,
},

View File

@ -1,5 +1,5 @@
import { Request, Response, Router } from "express";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { trimSpecial, User, Snowflake, UserModel } from "@fosscord/server-util";
import bcrypt from "bcrypt";
import { check, Email, EMAIL_REGEX, FieldErrors, Length } from "../../util/instanceOf";
@ -40,7 +40,7 @@ router.post(
// TODO: check password strength
// adjusted_email will be slightly modified version of the user supplied email -> e.g. protection against GMail Trick
let adjusted_email: string | undefined = adjustEmail(email);
let adjusted_email: string | null = adjustEmail(email);
// adjusted_password will be the hash of the password
let adjusted_password: string = "";
@ -52,7 +52,7 @@ router.post(
let discriminator = "";
// get register Config
const { register, security } = Config.get();
const { register, security } = Config.apiConfig.getAll();
// check if registration is allowed
if (!register.allowNewRegistration) {
@ -90,13 +90,13 @@ router.post(
},
});
}
} else if (register.email.required) {
} else if (register.email.necessary) {
throw FieldErrors({
email: { code: "BASE_TYPE_REQUIRED", message: req.t("common:field.BASE_TYPE_REQUIRED") },
});
}
if (register.dateOfBirth.required && !date_of_birth) {
if (register.dateOfBirth.necessary && !date_of_birth) {
throw FieldErrors({
date_of_birth: { code: "BASE_TYPE_REQUIRED", message: req.t("common:field.BASE_TYPE_REQUIRED") },
});
@ -181,7 +181,7 @@ router.post(
mobile: false,
premium: false,
premium_type: 0,
phone: undefined,
phone: null,
mfa_enabled: false,
verified: false,
presence: {
@ -253,7 +253,7 @@ router.post(
}
);
export function adjustEmail(email: string): string | undefined {
export function adjustEmail(email: string): string | null {
// body parser already checked if it is a valid email
const parts = <RegExpMatchArray>email.match(EMAIL_REGEX);
// @ts-ignore

View File

@ -1,7 +1,7 @@
import { Router } from "express";
import { ChannelModel, getPermission, MessageDeleteBulkEvent, MessageModel } from "@fosscord/server-util";
import { HTTPError } from "lambert-server";
import Config from "../../../../util/Config";
import * as Config from "../../../../util/Config";
import { emitEvent } from "../../../../util/Event";
import { check } from "../../../../util/instanceOf";
@ -20,7 +20,7 @@ router.post("/", check({ messages: [String] }), async (req, res) => {
const permission = await getPermission(req.user_id, channel?.guild_id, channel_id, { channel });
permission.hasThrow("MANAGE_MESSAGES");
const { maxBulkDelete } = Config.get().limits.message;
const { maxBulkDelete } = Config.apiConfig.getAll().limits.message;
const { messages } = req.body as { messages: string[] };
if (messages.length < 2) throw new HTTPError("You must at least specify 2 messages to bulk delete");

View File

@ -1,6 +1,6 @@
import { ChannelModel, ChannelPinsUpdateEvent, getPermission, MessageModel, MessageUpdateEvent, toObject } from "@fosscord/server-util";
import { Router, Request, Response } from "express";
import Config from "../../../util/Config";
import * as Config from "../../../util/Config";
import { HTTPError } from "lambert-server";
import { emitEvent } from "../../../util/Event";
@ -18,7 +18,7 @@ router.put("/:message_id", async (req: Request, res: Response) => {
if (channel.guild_id) permission.hasThrow("MANAGE_MESSAGES");
const pinned_count = await MessageModel.count({ channel_id, pinned: true }).exec();
const { maxPins } = Config.get().limits.channel;
const { maxPins } = Config.apiConfig.getAll().limits.channel;
if (pinned_count >= maxPins) throw new HTTPError("Max pin count reached: " + maxPins);
await MessageModel.updateOne({ id: message_id }, { pinned: true }).exec();

View File

@ -1,12 +1,11 @@
import { Router } from "express";
import Config from "../util/Config";
import * as Config from "../util/Config"
const router = Router();
router.get("/", (req, res) => {
const endpoint = Config.getAll()?.gateway?.endpoint;
res.send({ url: endpoint || "ws://localhost:3002" });
const { gateway } = Config.apiConfig.getAll();
res.send({ url: gateway || "ws://localhost:3002" });
});
export default router;

View File

@ -3,7 +3,7 @@ import { RoleModel, GuildModel, Snowflake, Guild, RoleDocument } from "@fosscord
import { HTTPError } from "lambert-server";
import { check } from "./../../util/instanceOf";
import { GuildCreateSchema } from "../../schema/Guild";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { getPublicUser } from "../../util/User";
import { addMember } from "../../util/Member";
import { createChannel } from "../../util/Channel";
@ -15,7 +15,7 @@ const router: Router = Router();
router.post("/", check(GuildCreateSchema), async (req: Request, res: Response) => {
const body = req.body as GuildCreateSchema;
const { maxGuilds } = Config.get().limits.user;
const { maxGuilds } = Config.apiConfig.getAll().limits.user;
const user = await getPublicUser(req.user_id, { guilds: true });
if (user.guilds.length >= maxGuilds) {

View File

@ -5,7 +5,7 @@ import { HTTPError } from "lambert-server";
import { GuildTemplateCreateSchema } from "../../../schema/Guild";
import { getPublicUser } from "../../../util/User";
import { check } from "../../../util/instanceOf";
import Config from "../../../util/Config";
import * as Config from "../../../util/Config";
import { addMember } from "../../../util/Member";
router.get("/:code", async (req: Request, res: Response) => {
@ -21,7 +21,7 @@ router.post("/:code", check(GuildTemplateCreateSchema), async (req: Request, res
const { code } = req.params;
const body = req.body as GuildTemplateCreateSchema;
const { maxGuilds } = Config.get().limits.user;
const { maxGuilds } = Config.apiConfig.getAll().limits.user;
const user = await getPublicUser(req.user_id, { guilds: true });
if (user.guilds.length >= maxGuilds) {

View File

@ -1,19 +1,6 @@
import { Config, Snowflake } from "@fosscord/server-util";
import crypto from "crypto";
export default {
init() {
return Config.init({ api: DefaultOptions });
},
get(): DefaultOptions {
return Config.getAll().api;
},
set(val: any) {
return Config.setAll({ api: val });
},
getAll: Config.getAll,
setAll: Config.setAll,
};
import Ajv, { JSONSchemaType } from "ajv"
import { getConfigPathForFile } from "@fosscord/server-util/dist/util/Config";
import {Config} from "@fosscord/server-util"
export interface RateLimitOptions {
count: number;
@ -21,6 +8,7 @@ export interface RateLimitOptions {
}
export interface DefaultOptions {
gateway: string;
general: {
instance_id: string;
};
@ -64,7 +52,7 @@ export interface DefaultOptions {
login?: RateLimitOptions;
register?: RateLimitOptions;
};
channel?: {};
channel?: string;
// TODO: rate limit configuration for all routes
};
};
@ -84,13 +72,13 @@ export interface DefaultOptions {
};
register: {
email: {
required: boolean;
necessary: boolean;
allowlist: boolean;
blocklist: boolean;
domains: string[];
};
dateOfBirth: {
required: boolean;
necessary: boolean;
minimum: number; // in years
};
requireCaptcha: boolean;
@ -107,85 +95,271 @@ export interface DefaultOptions {
};
}
export const DefaultOptions: DefaultOptions = {
general: {
instance_id: Snowflake.generate(),
},
permissions: {
user: {
createGuilds: true,
},
},
limits: {
user: {
maxGuilds: 100,
maxUsername: 32,
maxFriends: 1000,
},
guild: {
maxRoles: 250,
maxMembers: 250000,
maxChannels: 500,
maxChannelsInCategory: 50,
hideOfflineMember: 1000,
},
message: {
characters: 2000,
ttsCharacters: 200,
maxReactions: 20,
maxAttachmentSize: 8388608,
maxBulkDelete: 100,
},
channel: {
maxPins: 50,
maxTopic: 1024,
},
rate: {
ip: {
enabled: true,
count: 1000,
timespan: 1000 * 60 * 10,
const schema: JSONSchemaType<DefaultOptions> & {
definitions: {
rateLimitOptions: JSONSchemaType<RateLimitOptions>
}
} = {
type: "object",
definitions: {
rateLimitOptions: {
type: "object",
properties: {
count: { type: "number" },
timespan: { type: "number" },
},
routes: {},
required: ["count", "timespan"],
},
},
security: {
jwtSecret: crypto.randomBytes(256).toString("base64"),
forwadedFor: null,
// forwadedFor: "X-Forwarded-For" // nginx/reverse proxy
// forwadedFor: "CF-Connecting-IP" // cloudflare:
captcha: {
enabled: false,
service: null,
sitekey: null,
secret: null,
properties: {
gateway: {
type: "string"
},
general: {
type: "object",
properties: {
instance_id: {
type: "string"
}
},
required: ["instance_id"],
additionalProperties: false
},
permissions: {
type: "object",
properties: {
user: {
type: "object",
properties: {
createGuilds: {
type: "boolean"
}
},
required: ["createGuilds"],
additionalProperties: false
}
},
required: ["user"],
additionalProperties: false
},
limits: {
type: "object",
properties: {
user: {
type: "object",
properties: {
maxFriends: {
type: "number"
},
maxGuilds: {
type: "number"
},
maxUsername: {
type: "number"
}
},
required: ["maxFriends", "maxGuilds", "maxUsername"],
additionalProperties: false
},
guild: {
type: "object",
properties: {
maxRoles: {
type: "number"
},
maxMembers: {
type: "number"
},
maxChannels: {
type: "number"
},
maxChannelsInCategory: {
type: "number"
},
hideOfflineMember: {
type: "number"
}
},
required: ["maxRoles", "maxMembers", "maxChannels", "maxChannelsInCategory", "hideOfflineMember"],
additionalProperties: false
},
message: {
type: "object",
properties: {
characters: {
type: "number"
},
ttsCharacters: {
type: "number"
},
maxReactions: {
type: "number"
},
maxAttachmentSize: {
type: "number"
},
maxBulkDelete: {
type: "number"
}
},
required: ["characters", "ttsCharacters", "maxReactions", "maxAttachmentSize", "maxBulkDelete"],
additionalProperties: false
},
channel: {
type: "object",
properties: {
maxPins: {
type: "number"
},
maxTopic: {
type: "number"
}
},
required: ["maxPins", "maxTopic"],
additionalProperties: false
},
rate: {
type: "object",
properties: {
ip: {
type: "object",
properties: {
enabled: { type: "boolean" },
count: { type: "number" },
timespan: { type: "number" }
},
required: ["enabled", "count", "timespan"],
additionalProperties: false
},
routes: {
type: "object",
properties: {
auth: {
type: "object",
properties: {
login: { $ref: '#/definitions/rateLimitOptions' },
register: { $ref: '#/definitions/rateLimitOptions' }
},
nullable: true,
required: [],
additionalProperties: false
},
channel: {
type: "string",
nullable: true
}
},
required: [],
additionalProperties: false
}
},
required: ["ip", "routes"]
}
},
required: ["channel", "guild", "message", "rate", "user"],
additionalProperties: false
},
security: {
type: "object",
properties: {
jwtSecret: {
type: "string"
},
forwadedFor: {
type: "string",
nullable: true
},
captcha: {
type: "object",
properties: {
enabled: { type: "boolean" },
service: {
type: "string",
enum: ["hcaptcha", "recaptcha", null],
nullable: true
},
sitekey: {
type: "string",
nullable: true
},
secret: {
type: "string",
nullable: true
}
},
required: ["enabled", "secret", "service", "sitekey"],
additionalProperties: false
}
},
required: ["captcha", "forwadedFor", "jwtSecret"],
additionalProperties: false
},
login: {
type: "object",
properties: {
requireCaptcha: { type: "boolean" }
},
required: ["requireCaptcha"],
additionalProperties: false
},
register: {
type: "object",
properties: {
email: {
type: "object",
properties: {
necessary: { type: "boolean" },
allowlist: { type: "boolean" },
blocklist: { type: "boolean" },
domains: {
type: "array",
items: {
type: "string"
}
}
},
required: ["allowlist", "blocklist", "domains", "necessary"],
additionalProperties: false
},
dateOfBirth: {
type: "object",
properties: {
necessary: { type: "boolean" },
minimum: { type: "number" }
},
required: ["minimum", "necessary"],
additionalProperties: false
},
requireCaptcha: { type: "boolean" },
requireInvite: { type: "boolean" },
allowNewRegistration: { type: "boolean" },
allowMultipleAccounts: { type: "boolean" },
password: {
type: "object",
properties: {
minLength: { type: "number" },
minNumbers: { type: "number" },
minUpperCase: { type: "number" },
minSymbols: { type: "number" },
blockInsecureCommonPasswords: { type: "boolean" }
},
required: ["minLength", "minNumbers", "minUpperCase", "minSymbols", "blockInsecureCommonPasswords"],
additionalProperties: false
}
},
required: ["allowMultipleAccounts", "allowNewRegistration", "dateOfBirth", "email", "password", "requireCaptcha", "requireInvite"],
additionalProperties: false
},
},
login: {
requireCaptcha: false,
},
register: {
email: {
required: true,
allowlist: false,
blocklist: true,
domains: [], // TODO: efficiently save domain blocklist in database
// domains: fs.readFileSync(__dirname + "/blockedEmailDomains.txt", { encoding: "utf8" }).split("\n"),
},
dateOfBirth: {
required: true,
minimum: 13,
},
requireInvite: false,
requireCaptcha: true,
allowNewRegistration: true,
allowMultipleAccounts: true,
password: {
minLength: 8,
minNumbers: 2,
minUpperCase: 2,
minSymbols: 0,
blockInsecureCommonPasswords: false,
},
},
};
required: ["gateway", "general", "limits", "login", "permissions", "register", "security"],
additionalProperties: false
}
const ajv = new Ajv();
const validator = ajv.compile(schema);
const configPath = getConfigPathForFile("fosscord", "api", ".json");
export const apiConfig = new Config<DefaultOptions>({path: configPath, schemaValidator: validator, schema: schema});

View File

@ -14,7 +14,7 @@ import {
} from "@fosscord/server-util";
import { HTTPError } from "lambert-server";
import Config from "./Config";
import * as Config from "./Config";
import { emitEvent } from "./Event";
import { getPublicUser } from "./User";
@ -39,7 +39,7 @@ export async function isMember(user_id: string, guild_id: string) {
export async function addMember(user_id: string, guild_id: string, cache?: { guild?: GuildDocument }) {
const user = await getPublicUser(user_id, { guilds: true });
const { maxGuilds } = Config.get().limits.user;
const { maxGuilds } = Config.apiConfig.getAll().limits.user;
if (user.guilds.length >= maxGuilds) {
throw new HTTPError(`You are at the ${maxGuilds} server limit.`, 403);
}

View File

@ -1,5 +1,5 @@
import "missing-native-js-functions";
import Config from "./Config";
import * as Config from "./Config";
const reNUMBER = /[0-9]/g;
const reUPPERCASELETTER = /[A-Z]/g;
@ -23,7 +23,7 @@ export function check(password: string): number {
minUpperCase,
minSymbols,
blockInsecureCommonPasswords,
} = Config.get().register.password;
} = Config.apiConfig.getAll().register.password;
var strength = 0;
// checks for total password len