diff --git a/gateway/src/opcodes/Identify.ts b/gateway/src/opcodes/Identify.ts index 464d584c..644d0984 100644 --- a/gateway/src/opcodes/Identify.ts +++ b/gateway/src/opcodes/Identify.ts @@ -106,7 +106,7 @@ export async function onIdentify(this: WebSocket, data: Payload) { }), guild_experiments: [], // TODO geo_ordered_rtc_regions: [], // TODO - relationships: user.user_data.relationships, + relationships: user.data.relationships, read_state: { // TODO entries: [], diff --git a/util/package-lock.json b/util/package-lock.json index 0ecdaea9..f30fcf47 100644 Binary files a/util/package-lock.json and b/util/package-lock.json differ diff --git a/util/package.json b/util/package.json index a132a99c..fc474d68 100644 --- a/util/package.json +++ b/util/package.json @@ -44,6 +44,7 @@ "dot-prop": "^6.0.1", "env-paths": "^2.2.1", "jsonwebtoken": "^8.5.1", + "lambert-server": "^1.2.8", "missing-native-js-functions": "^1.2.10", "node-fetch": "^2.6.1", "patch-package": "^6.4.7", diff --git a/util/patches/test-performance+1.1.3.patch b/util/patches/test-performance+1.1.3.patch deleted file mode 100644 index 23b4900b..00000000 --- a/util/patches/test-performance+1.1.3.patch +++ /dev/null @@ -1,381 +0,0 @@ -diff --git a/node_modules/test-performance/cjs/index.js b/node_modules/test-performance/cjs/index.js -index 65d5904..04638a1 100644 ---- a/node_modules/test-performance/cjs/index.js -+++ b/node_modules/test-performance/cjs/index.js -@@ -1,4 +1,4 @@ --'use strict'; -+"use strict"; - - /*! ***************************************************************************** - Copyright (c) Microsoft Corporation. -@@ -15,50 +15,139 @@ OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. - ***************************************************************************** */ - -+const { performance } = require("perf_hooks"); -+ - function __awaiter(thisArg, _arguments, P, generator) { -- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } -- return new (P || (P = Promise))(function (resolve, reject) { -- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } -- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } -- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } -- step((generator = generator.apply(thisArg, _arguments || [])).next()); -- }); -+ function adopt(value) { -+ return value instanceof P -+ ? value -+ : new P(function (resolve) { -+ resolve(value); -+ }); -+ } -+ return new (P || (P = Promise))(function (resolve, reject) { -+ function fulfilled(value) { -+ try { -+ step(generator.next(value)); -+ } catch (e) { -+ reject(e); -+ } -+ } -+ function rejected(value) { -+ try { -+ step(generator["throw"](value)); -+ } catch (e) { -+ reject(e); -+ } -+ } -+ function step(result) { -+ result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); -+ } -+ step((generator = generator.apply(thisArg, _arguments || [])).next()); -+ }); - } - - function __generator(thisArg, body) { -- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; -- return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; -- function verb(n) { return function (v) { return step([n, v]); }; } -- function step(op) { -- if (f) throw new TypeError("Generator is already executing."); -- while (_) try { -- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; -- if (y = 0, t) op = [op[0] & 2, t.value]; -- switch (op[0]) { -- case 0: case 1: t = op; break; -- case 4: _.label++; return { value: op[1], done: false }; -- case 5: _.label++; y = op[1]; op = [0]; continue; -- case 7: op = _.ops.pop(); _.trys.pop(); continue; -- default: -- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } -- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } -- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } -- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } -- if (t[2]) _.ops.pop(); -- _.trys.pop(); continue; -- } -- op = body.call(thisArg, _); -- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } -- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; -- } -+ var _ = { -+ label: 0, -+ sent: function () { -+ if (t[0] & 1) throw t[1]; -+ return t[1]; -+ }, -+ trys: [], -+ ops: [], -+ }, -+ f, -+ y, -+ t, -+ g; -+ return ( -+ (g = { next: verb(0), throw: verb(1), return: verb(2) }), -+ typeof Symbol === "function" && -+ (g[Symbol.iterator] = function () { -+ return this; -+ }), -+ g -+ ); -+ function verb(n) { -+ return function (v) { -+ return step([n, v]); -+ }; -+ } -+ function step(op) { -+ if (f) throw new TypeError("Generator is already executing."); -+ while (_) -+ try { -+ if ( -+ ((f = 1), -+ y && -+ (t = -+ op[0] & 2 -+ ? y["return"] -+ : op[0] -+ ? y["throw"] || ((t = y["return"]) && t.call(y), 0) -+ : y.next) && -+ !(t = t.call(y, op[1])).done) -+ ) -+ return t; -+ if (((y = 0), t)) op = [op[0] & 2, t.value]; -+ switch (op[0]) { -+ case 0: -+ case 1: -+ t = op; -+ break; -+ case 4: -+ _.label++; -+ return { value: op[1], done: false }; -+ case 5: -+ _.label++; -+ y = op[1]; -+ op = [0]; -+ continue; -+ case 7: -+ op = _.ops.pop(); -+ _.trys.pop(); -+ continue; -+ default: -+ if (!((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && (op[0] === 6 || op[0] === 2)) { -+ _ = 0; -+ continue; -+ } -+ if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { -+ _.label = op[1]; -+ break; -+ } -+ if (op[0] === 6 && _.label < t[1]) { -+ _.label = t[1]; -+ t = op; -+ break; -+ } -+ if (t && _.label < t[2]) { -+ _.label = t[2]; -+ _.ops.push(op); -+ break; -+ } -+ if (t[2]) _.ops.pop(); -+ _.trys.pop(); -+ continue; -+ } -+ op = body.call(thisArg, _); -+ } catch (e) { -+ op = [6, e]; -+ y = 0; -+ } finally { -+ f = t = 0; -+ } -+ if (op[0] & 5) throw op[1]; -+ return { value: op[0] ? op[1] : void 0, done: true }; -+ } - } - - var baselineFunctions = { -- standard: { -- expectedMsRunTime: 12, -- func: function (dummyParam) { -- }, -- }, -+ standard: { -+ expectedMsRunTime: 12, -+ func: function (dummyParam) {}, -+ }, - }; - - /** -@@ -66,28 +155,28 @@ var baselineFunctions = { - * to a baseline. - */ - function calculateExpectedPerformance(baselineExpected, baselineActual, target) { -- var performanceRatio = baselineActual / baselineExpected; -- return target / performanceRatio; -+ var performanceRatio = baselineActual / baselineExpected; -+ return target / performanceRatio; - } - - /** - * This runs a single performance test of a function - */ - function perfTest(func) { -- return __awaiter(this, void 0, void 0, function () { -- var start, end; -- return __generator(this, function (_a) { -- switch (_a.label) { -- case 0: -- start = performance.now(); -- return [4 /*yield*/, func()]; -- case 1: -- _a.sent(); -- end = performance.now(); -- return [2 /*return*/, end - start]; -- } -- }); -- }); -+ return __awaiter(this, void 0, void 0, function () { -+ var start, end; -+ return __generator(this, function (_a) { -+ switch (_a.label) { -+ case 0: -+ start = performance.now(); -+ return [4 /*yield*/, func()]; -+ case 1: -+ _a.sent(); -+ end = performance.now(); -+ return [2 /*return*/, end - start]; -+ } -+ }); -+ }); - } - - var NUMBER_OF_TESTS = 10000; -@@ -96,22 +185,31 @@ var NUMBER_OF_TESTS = 10000; - * the average in milliseconds. - */ - function getScore(func) { -- return __awaiter(this, void 0, void 0, function () { -- var tests, results; -- var _this = this; -- return __generator(this, function (_a) { -- switch (_a.label) { -- case 0: -- tests = new Array(NUMBER_OF_TESTS).fill(undefined).map(function () { return __awaiter(_this, void 0, void 0, function () { return __generator(this, function (_a) { -- return [2 /*return*/, perfTest(func)]; -- }); }); }); -- return [4 /*yield*/, Promise.all(tests)]; -- case 1: -- results = _a.sent(); -- return [2 /*return*/, results.reduce(function (acc, val) { return acc + val; }, 0) / NUMBER_OF_TESTS]; -- } -- }); -- }); -+ return __awaiter(this, void 0, void 0, function () { -+ var tests, results; -+ var _this = this; -+ return __generator(this, function (_a) { -+ switch (_a.label) { -+ case 0: -+ tests = new Array(NUMBER_OF_TESTS).fill(undefined).map(function () { -+ return __awaiter(_this, void 0, void 0, function () { -+ return __generator(this, function (_a) { -+ return [2 /*return*/, perfTest(func)]; -+ }); -+ }); -+ }); -+ return [4 /*yield*/, Promise.all(tests)]; -+ case 1: -+ results = _a.sent(); -+ return [ -+ 2 /*return*/, -+ results.reduce(function (acc, val) { -+ return acc + val; -+ }, 0) / NUMBER_OF_TESTS, -+ ]; -+ } -+ }); -+ }); - } - - /** -@@ -119,31 +217,38 @@ function getScore(func) { - * known baseline function. - */ - function runTests(baselineFunc, targetFunc) { -- return __awaiter(this, void 0, void 0, function () { -- var initialBaselineScore, initialTargetScore, midwayBaselineScore, endTargetScore, endBaselineScore, totalBaselineScore, totalTargetScore; -- return __generator(this, function (_a) { -- switch (_a.label) { -- case 0: return [4 /*yield*/, getScore(baselineFunc)]; -- case 1: -- initialBaselineScore = _a.sent(); -- return [4 /*yield*/, getScore(targetFunc)]; -- case 2: -- initialTargetScore = _a.sent(); -- return [4 /*yield*/, getScore(baselineFunc)]; -- case 3: -- midwayBaselineScore = _a.sent(); -- return [4 /*yield*/, getScore(targetFunc)]; -- case 4: -- endTargetScore = _a.sent(); -- return [4 /*yield*/, getScore(baselineFunc)]; -- case 5: -- endBaselineScore = _a.sent(); -- totalBaselineScore = (initialBaselineScore + midwayBaselineScore + endBaselineScore) / 3; -- totalTargetScore = (initialTargetScore + endTargetScore) / 2; -- return [2 /*return*/, [totalBaselineScore, totalTargetScore]]; -- } -- }); -- }); -+ return __awaiter(this, void 0, void 0, function () { -+ var initialBaselineScore, -+ initialTargetScore, -+ midwayBaselineScore, -+ endTargetScore, -+ endBaselineScore, -+ totalBaselineScore, -+ totalTargetScore; -+ return __generator(this, function (_a) { -+ switch (_a.label) { -+ case 0: -+ return [4 /*yield*/, getScore(baselineFunc)]; -+ case 1: -+ initialBaselineScore = _a.sent(); -+ return [4 /*yield*/, getScore(targetFunc)]; -+ case 2: -+ initialTargetScore = _a.sent(); -+ return [4 /*yield*/, getScore(baselineFunc)]; -+ case 3: -+ midwayBaselineScore = _a.sent(); -+ return [4 /*yield*/, getScore(targetFunc)]; -+ case 4: -+ endTargetScore = _a.sent(); -+ return [4 /*yield*/, getScore(baselineFunc)]; -+ case 5: -+ endBaselineScore = _a.sent(); -+ totalBaselineScore = (initialBaselineScore + midwayBaselineScore + endBaselineScore) / 3; -+ totalTargetScore = (initialTargetScore + endTargetScore) / 2; -+ return [2 /*return*/, [totalBaselineScore, totalTargetScore]]; -+ } -+ }); -+ }); - } - - /** -@@ -152,20 +257,22 @@ function runTests(baselineFunc, targetFunc) { - */ - var BASELINE_FUNCTION = baselineFunctions.standard; - function getPerformanceScore(func) { -- return __awaiter(this, void 0, void 0, function () { -- var _a, baseline, target; -- return __generator(this, function (_b) { -- switch (_b.label) { -- case 0: -- if (typeof func !== 'function') -- throw new Error(func + " is not a function"); -- return [4 /*yield*/, runTests(BASELINE_FUNCTION.func, func)]; -- case 1: -- _a = _b.sent(), baseline = _a[0], target = _a[1]; -- return [2 /*return*/, calculateExpectedPerformance(BASELINE_FUNCTION.expectedMsRunTime, baseline, target)]; -- } -- }); -- }); -+ return __awaiter(this, void 0, void 0, function () { -+ var _a, baseline, target; -+ return __generator(this, function (_b) { -+ switch (_b.label) { -+ case 0: -+ if (typeof func !== "function") throw new Error(func + " is not a function"); -+ return [4 /*yield*/, runTests(BASELINE_FUNCTION.func, func)]; -+ case 1: -+ (_a = _b.sent()), (baseline = _a[0]), (target = _a[1]); -+ return [ -+ 2 /*return*/, -+ calculateExpectedPerformance(BASELINE_FUNCTION.expectedMsRunTime, baseline, target), -+ ]; -+ } -+ }); -+ }); - } - - module.exports = getPerformanceScore; diff --git a/util/src/entities/BaseClass.ts b/util/src/entities/BaseClass.ts index 2d2457de..bb6ccea1 100644 --- a/util/src/entities/BaseClass.ts +++ b/util/src/entities/BaseClass.ts @@ -3,6 +3,10 @@ import { BaseEntity, BeforeInsert, BeforeUpdate, PrimaryColumn } from "typeorm"; import { Snowflake } from "../util/Snowflake"; import Ajv, { ValidateFunction } from "ajv"; import schema from "./schema.json"; +import "missing-native-js-functions"; + +// TODO use class-validator https://typeorm.io/#/validation with class annotators (isPhone/isEmail) combined with types from typescript-json-schema +// btw. we don't use class-validator for everything, because we need to explicitly set the type instead of deriving it from typescript also it doesn't easily support nested objects const ajv = new Ajv({ removeAdditional: "all", @@ -12,7 +16,6 @@ const ajv = new Ajv({ validateFormats: false, allowUnionTypes: true, }); -// const validator = ajv.compile(schema); export class BaseClass extends BaseEntity { @PrimaryColumn() @@ -43,10 +46,20 @@ export class BaseClass extends BaseEntity { delete props.opts; + const properties = new Set(this.metadata.columns.map((x: any) => x.propertyName)); + // will not include relational properties (e.g. @RelationId @ManyToMany) + for (const key in props) { if (this.hasOwnProperty(key)) continue; + if (!properties.has(key)) continue; + // @ts-ignore + const setter = this[`set${key.capitalize()}`]; - Object.defineProperty(this, key, { value: props[key] }); + if (setter) { + setter.call(this, props[key]); + } else { + Object.defineProperty(this, key, { value: props[key] }); + } } } diff --git a/util/src/entities/Config.ts b/util/src/entities/Config.ts index 60d84958..c9326be2 100644 --- a/util/src/entities/Config.ts +++ b/util/src/entities/Config.ts @@ -1,7 +1,7 @@ import { Column, Entity, JoinColumn, ManyToOne } from "typeorm"; -import { Snowflake } from "../util"; import { BaseClass } from "./BaseClass"; import crypto from "crypto"; +import { Snowflake } from "../util/Snowflake"; @Entity("config") export class ConfigEntity extends BaseClass { diff --git a/util/src/entities/ReadState.ts b/util/src/entities/ReadState.ts index 0310cb5f..70a38b84 100644 --- a/util/src/entities/ReadState.ts +++ b/util/src/entities/ReadState.ts @@ -4,6 +4,10 @@ import { Channel } from "./Channel"; import { Message } from "./Message"; import { User } from "./User"; +// for read receipts +// notification cursor and public read receipt need to be forwards-only (the former to prevent re-pinging when marked as unread, and the latter to be acceptable as a legal acknowledgement in criminal proceedings), and private read marker needs to be advance-rewind capable +// public read receipt ≥ notification cursor ≥ private fully read marker + @Entity("read_states") export class ReadState extends BaseClass { @RelationId((read_state: ReadState) => read_state.channel) diff --git a/util/src/entities/Template.ts b/util/src/entities/Template.ts index c8d2034c..1dfc3b1b 100644 --- a/util/src/entities/Template.ts +++ b/util/src/entities/Template.ts @@ -1,11 +1,11 @@ -import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; +import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn, RelationId } from "typeorm"; import { BaseClass } from "./BaseClass"; import { Guild } from "./Guild"; import { User } from "./User"; @Entity("templates") export class Template extends BaseClass { - @Column() + @PrimaryColumn() code: string; @Column() @@ -36,4 +36,7 @@ export class Template extends BaseClass { @JoinColumn({ name: "source_guild_id" }) @ManyToOne(() => Guild, (guild: Guild) => guild.id) source_guild: Guild; + + @Column("simple-json") + serialized_source_guild: Guild; } diff --git a/util/src/entities/User.ts b/util/src/entities/User.ts index bdf0d35f..03cb3af4 100644 --- a/util/src/entities/User.ts +++ b/util/src/entities/User.ts @@ -29,8 +29,8 @@ export class User extends BaseClass { setDiscriminator(val: string) { const number = Number(val); if (isNaN(number)) throw new Error("invalid discriminator"); - if (number > 0 && number < 10000) throw new Error("discriminator must be between 1 and 9999"); - this.discriminator = val; + if (number <= 0 || number > 10000) throw new Error("discriminator must be between 1 and 9999"); + this.discriminator = val.toString(); } @Column() diff --git a/util/src/entities/schema.json b/util/src/entities/schema.json index ebf66e88..7f192690 100644 --- a/util/src/entities/schema.json +++ b/util/src/entities/schema.json @@ -1198,6 +1198,380 @@ }, "type": "object" }, + "ConfigEntity": { + "properties": { + "construct": { + }, + "id": { + "type": "string" + }, + "metadata": { + }, + "opts": { + "properties": { + "id": { + "type": "string" + } + }, + "type": "object" + }, + "value": { + "$ref": "#/definitions/ConfigValue" + } + }, + "type": "object" + }, + "ConfigValue": { + "properties": { + "cdn": { + "properties": { + "endpoint": { + "type": [ + "null", + "string" + ] + }, + "endpointClient": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "gateway": { + "properties": { + "endpoint": { + "type": [ + "null", + "string" + ] + }, + "endpointClient": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "general": { + "properties": { + "instance_id": { + "type": "string" + } + }, + "type": "object" + }, + "kafka": { + "properties": { + "brokers": { + "anyOf": [ + { + "items": { + "$ref": "#/definitions/KafkaBroker" + }, + "type": "array" + }, + { + "type": "null" + } + ] + } + }, + "type": "object" + }, + "limits": { + "properties": { + "channel": { + "properties": { + "maxPins": { + "type": "number" + }, + "maxTopic": { + "type": "number" + } + }, + "type": "object" + }, + "guild": { + "properties": { + "hideOfflineMember": { + "type": "number" + }, + "maxChannels": { + "type": "number" + }, + "maxChannelsInCategory": { + "type": "number" + }, + "maxMembers": { + "type": "number" + }, + "maxRoles": { + "type": "number" + } + }, + "type": "object" + }, + "message": { + "properties": { + "maxAttachmentSize": { + "type": "number" + }, + "maxBulkDelete": { + "type": "number" + }, + "maxCharacters": { + "type": "number" + }, + "maxReactions": { + "type": "number" + }, + "maxTTSCharacters": { + "type": "number" + } + }, + "type": "object" + }, + "rate": { + "properties": { + "error": { + "$ref": "#/definitions/RateLimitOptions" + }, + "global": { + "$ref": "#/definitions/RateLimitOptions" + }, + "ip": { + "$ref": "#/definitions/Omit" + }, + "routes": { + "properties": { + "auth": { + "properties": { + "login": { + "$ref": "#/definitions/RateLimitOptions" + }, + "register": { + "$ref": "#/definitions/RateLimitOptions" + } + }, + "type": "object" + }, + "channel": { + "$ref": "#/definitions/RateLimitOptions" + }, + "guild": { + "$ref": "#/definitions/RateLimitOptions" + }, + "webhook": { + "$ref": "#/definitions/RateLimitOptions" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "user": { + "properties": { + "maxFriends": { + "type": "number" + }, + "maxGuilds": { + "type": "number" + }, + "maxUsername": { + "type": "number" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "login": { + "properties": { + "requireCaptcha": { + "type": "boolean" + } + }, + "type": "object" + }, + "permissions": { + "properties": { + "user": { + "properties": { + "createGuilds": { + "type": "boolean" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "rabbitmq": { + "properties": { + "host": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "regions": { + "properties": { + "available": { + "items": { + "$ref": "#/definitions/Region" + }, + "type": "array" + }, + "default": { + "type": "string" + } + }, + "type": "object" + }, + "register": { + "properties": { + "allowMultipleAccounts": { + "type": "boolean" + }, + "allowNewRegistration": { + "type": "boolean" + }, + "blockProxies": { + "type": "boolean" + }, + "dateOfBirth": { + "properties": { + "minimum": { + "type": "number" + }, + "necessary": { + "type": "boolean" + } + }, + "type": "object" + }, + "email": { + "properties": { + "allowlist": { + "type": "boolean" + }, + "blocklist": { + "type": "boolean" + }, + "domains": { + "items": { + "type": "string" + }, + "type": "array" + }, + "necessary": { + "type": "boolean" + } + }, + "type": "object" + }, + "password": { + "properties": { + "minLength": { + "type": "number" + }, + "minNumbers": { + "type": "number" + }, + "minSymbols": { + "type": "number" + }, + "minUpperCase": { + "type": "number" + } + }, + "type": "object" + }, + "requireCaptcha": { + "type": "boolean" + }, + "requireInvite": { + "type": "boolean" + } + }, + "type": "object" + }, + "security": { + "properties": { + "autoUpdate": { + "type": [ + "number", + "boolean" + ] + }, + "captcha": { + "properties": { + "enabled": { + "type": "boolean" + }, + "secret": { + "type": [ + "null", + "string" + ] + }, + "service": { + "anyOf": [ + { + "enum": [ + "hcaptcha", + "recaptcha" + ], + "type": "string" + }, + { + "type": "null" + } + ] + }, + "sitekey": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + }, + "forwadedFor": { + "type": [ + "null", + "string" + ] + }, + "ipdataApiKey": { + "type": [ + "null", + "string" + ] + }, + "jwtSecret": { + "type": "string" + }, + "requestSignature": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, "ConnectedAccount": { "properties": { "access_token": { @@ -1721,6 +2095,9 @@ "public_updates_channel": { "$ref": "#/definitions/Channel" }, + "public_updates_channel_id": { + "type": "string" + }, "region": { "type": "string" }, @@ -1760,6 +2137,9 @@ "vanity_url": { "$ref": "#/definitions/Invite" }, + "vanity_url_code": { + "type": "string" + }, "verification_level": { "type": "number" }, @@ -1809,6 +2189,9 @@ "widget_channel": { "$ref": "#/definitions/Channel" }, + "widget_channel_id": { + "type": "string" + }, "widget_enabled": { "type": "boolean" } @@ -2520,12 +2903,12 @@ "target_user": { "type": "string" }, + "target_user_id": { + "type": "string" + }, "target_user_type": { "type": "number" }, - "target_usser_id": { - "type": "string" - }, "temporary": { "type": "boolean" }, @@ -2615,6 +2998,17 @@ }, "type": "object" }, + "KafkaBroker": { + "properties": { + "ip": { + "type": "string" + }, + "port": { + "type": "number" + } + }, + "type": "object" + }, "ListenEventOpts": { "properties": { "acknowledge": { @@ -3507,12 +3901,12 @@ "target_user": { "type": "string" }, + "target_user_id": { + "type": "string" + }, "target_user_type": { "type": "number" }, - "target_usser_id": { - "type": "string" - }, "temporary": { "type": "boolean" }, @@ -3843,6 +4237,23 @@ }, "type": "object" }, + "Omit": { + "properties": { + "bot": { + "type": "number" + }, + "count": { + "type": "number" + }, + "onyIp": { + "type": "boolean" + }, + "window": { + "type": "number" + } + }, + "type": "object" + }, "Omit": { "properties": { "assign": { @@ -4322,6 +4733,23 @@ }, "type": "object" }, + "RateLimitOptions": { + "properties": { + "bot": { + "type": "number" + }, + "count": { + "type": "number" + }, + "onyIp": { + "type": "boolean" + }, + "window": { + "type": "number" + } + }, + "type": "object" + }, "Reaction": { "properties": { "count": { @@ -4355,6 +4783,9 @@ "last_message": { "$ref": "#/definitions/Message" }, + "last_message_id": { + "type": "string" + }, "last_pin_timestamp": { "format": "date-time", "type": "string" @@ -4763,6 +5194,29 @@ "Record": { "type": "object" }, + "Region": { + "properties": { + "custom": { + "type": "boolean" + }, + "deprecated": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "optimal": { + "type": "boolean" + }, + "vip": { + "type": "boolean" + } + }, + "type": "object" + }, "Relationship": { "properties": { "construct": { @@ -5053,6 +5507,9 @@ "creator": { "$ref": "#/definitions/User" }, + "creator_id": { + "type": "string" + }, "description": { "type": "string" }, @@ -5072,9 +5529,15 @@ }, "type": "object" }, + "serialized_source_guild": { + "$ref": "#/definitions/Guild" + }, "source_guild": { "$ref": "#/definitions/Guild" }, + "source_guild_id": { + "type": "string" + }, "updated_at": { "format": "date-time", "type": "string" @@ -5164,6 +5627,18 @@ "format": "date-time", "type": "string" }, + "data": { + "properties": { + "hash": { + "type": "string" + }, + "valid_tokens_since": { + "format": "date-time", + "type": "string" + } + }, + "type": "object" + }, "deleted": { "type": "boolean" }, @@ -5179,15 +5654,27 @@ "email": { "type": "string" }, + "fingerprints": { + "items": { + "type": "string" + }, + "type": "array" + }, "flags": { "type": "bigint" }, - "guilds": { + "guild_ids": { "items": { "type": "string" }, "type": "array" }, + "guilds": { + "items": { + "$ref": "#/definitions/Guild" + }, + "type": "array" + }, "id": { "type": "string" }, @@ -5240,24 +5727,6 @@ "system": { "type": "boolean" }, - "user_data": { - "properties": { - "fingerprints": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hash": { - "type": "string" - }, - "valid_tokens_since": { - "format": "date-time", - "type": "string" - } - }, - "type": "object" - }, "username": { "type": "string" }, @@ -5544,6 +6013,9 @@ "channel": { "$ref": "#/definitions/Channel" }, + "channel_id": { + "type": "string" + }, "construct": { }, "deaf": { @@ -5552,6 +6024,9 @@ "guild": { "$ref": "#/definitions/Guild" }, + "guild_id": { + "type": "string" + }, "id": { "type": "string" }, @@ -5588,6 +6063,9 @@ }, "user": { "$ref": "#/definitions/User" + }, + "user_id": { + "type": "string" } }, "type": "object" @@ -5634,17 +6112,26 @@ "Webhook": { "properties": { "application": { + "$ref": "#/definitions/Application" + }, + "application_id": { "type": "string" }, "avatar": { "type": "string" }, "channel": { + "$ref": "#/definitions/Channel" + }, + "channel_id": { "type": "string" }, "construct": { }, "guild": { + "$ref": "#/definitions/Guild" + }, + "guild_id": { "type": "string" }, "id": { @@ -5664,6 +6151,9 @@ "type": "object" }, "source_guild": { + "$ref": "#/definitions/Guild" + }, + "source_guild_id": { "type": "string" }, "token": { @@ -5673,6 +6163,9 @@ "$ref": "#/definitions/WebhookType" }, "user": { + "$ref": "#/definitions/User" + }, + "user_id": { "type": "string" } }, diff --git a/util/src/util/Config.ts b/util/src/util/Config.ts new file mode 100644 index 00000000..f8574f38 --- /dev/null +++ b/util/src/util/Config.ts @@ -0,0 +1,19 @@ +import "missing-native-js-functions"; +import { ConfigValue, ConfigEntity, DefaultConfigOptions } from "../entities/Config"; + +var config: ConfigEntity; +// TODO: use events to inform about config updates + +export const Config = { + init: async function init() { + config = new ConfigEntity({}, { id: "0" }); + return this.set((config.value || {}).merge(DefaultConfigOptions)); + }, + get: function get() { + return config.value as ConfigValue; + }, + set: function set(val: any) { + config.value = val.merge(config.value); + return config.save(); + }, +}; diff --git a/util/src/util/Database.ts b/util/src/util/Database.ts index 90ba9079..8bb8078e 100644 --- a/util/src/util/Database.ts +++ b/util/src/util/Database.ts @@ -1,11 +1,12 @@ import "reflect-metadata"; -import { createConnection } from "typeorm"; +import { Connection, createConnection } from "typeorm"; import * as Models from "../entities"; // UUID extension option is only supported with postgres // We want to generate all id's with Snowflakes that's why we have our own BaseEntity class var promise: Promise; +var dbConnection: Connection | undefined; export function initDatabase() { if (promise) return promise; // prevent initalizing multiple times @@ -20,7 +21,16 @@ export function initDatabase() { logging: false, }); - promise.then(() => console.log("[Database] connected")); + promise.then((connection) => { + dbConnection = connection; + console.log("[Database] connected"); + }); return promise; } + +export { dbConnection }; + +export function closeDatabase() { + dbConnection?.close(); +} diff --git a/util/src/util/checkToken.ts b/util/src/util/checkToken.ts index 58e537ea..1e203006 100644 --- a/util/src/util/checkToken.ts +++ b/util/src/util/checkToken.ts @@ -9,13 +9,10 @@ export function checkToken(token: string, jwtSecret: string): Promise { jwt.verify(token, jwtSecret, JWTOptions, async (err, decoded: any) => { if (err || !decoded) return rej("Invalid Token"); - const user = await User.findOne( - { id: decoded.id }, - { select: ["user_data", "bot", "disabled", "deleted"] } - ); + const user = await User.findOne({ id: decoded.id }, { select: ["data", "bot", "disabled", "deleted"] }); if (!user) return rej("Invalid Token"); // we need to round it to seconds as it saved as seconds in jwt iat and valid_tokens_since is stored in milliseconds - if (decoded.iat * 1000 < user.user_data.valid_tokens_since.setSeconds(0, 0)) return rej("Invalid Token"); + if (decoded.iat * 1000 < user.data.valid_tokens_since.setSeconds(0, 0)) return rej("Invalid Token"); if (user.disabled) return rej("User disabled"); if (user.deleted) return rej("User not found"); diff --git a/util/src/util/index.ts b/util/src/util/index.ts index 700ecfa5..16b98ca3 100644 --- a/util/src/util/index.ts +++ b/util/src/util/index.ts @@ -1,13 +1,13 @@ export * from "./Database"; -export * from "./Regex"; -export * from "./String"; export * from "./BitField"; +export * from "./Config"; +export * from "./checkToken"; +export * from "./Event"; export * from "./Intents"; export * from "./MessageFlags"; export * from "./Permissions"; export * from "./Snowflake"; -export * from "./toBigInt"; export * from "./RabbitMQ"; -export * from "./Event"; -export * from "./checkToken"; +export * from "./Regex"; +export * from "./String"; diff --git a/util/src/util/toBigInt.ts b/util/src/util/toBigInt.ts deleted file mode 100644 index b7985928..00000000 --- a/util/src/util/toBigInt.ts +++ /dev/null @@ -1,4 +0,0 @@ -export default function toBigInt(string: string): bigint { - return BigInt(string); -} - diff --git a/util/tests/User.test.js b/util/tests/User.test.js new file mode 100644 index 00000000..b87c753d --- /dev/null +++ b/util/tests/User.test.js @@ -0,0 +1,31 @@ +const { initDatabase, closeDatabase } = require("../dist/util/Database"); +const { User } = require("../dist/entities/User"); +jest.setTimeout(10000); + +beforeAll((done) => { + initDatabase().then(() => { + new User().validate(); // warm up schema/model + done(); + }); +}); + +afterAll(() => { + closeDatabase(); +}); + +describe("User", () => { + test("valid discriminator: 1", async () => { + new User({ discriminator: "1" }).validate(); + }); + test("invalid discriminator: test", async () => { + expect(() => { + new User({ discriminator: "test" }).validate(); + }).toThrow(); + }); + + test("invalid discriminator: 0", async () => { + expect(() => { + new User({ discriminator: "0" }).validate(); + }).toThrow(); + }); +}); diff --git a/util/tests/validate.test.js b/util/tests/validate.test.js index c885a167..434422f8 100644 --- a/util/tests/validate.test.js +++ b/util/tests/validate.test.js @@ -1,4 +1,4 @@ -const { initDatabase } = require("../dist/util/Database"); +const { initDatabase, closeDatabase } = require("../dist/util/Database"); const { User } = require("../dist/entities/User"); jest.setTimeout(10000); @@ -9,13 +9,15 @@ beforeAll((done) => { }); }); +afterAll(() => { + closeDatabase(); +}); + describe("Validate model class properties", () => { - describe("User", () => { - test("object instead of string", async () => { - expect(() => { - new User({ username: {} }).validate(); - }).toThrow(); - }); + test("object instead of string", async () => { + expect(() => { + new User({}, { id: {} }).validate(); + }).toThrow(); }); test("validation should be faster than 20ms", () => { diff --git a/util/util/Config.ts b/util/util/Config.ts deleted file mode 100644 index c65c4e44..00000000 --- a/util/util/Config.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { Schema, model, Types, Document } from "mongoose"; -import "missing-native-js-functions"; -import { Snowflake } from "./Snowflake"; -import crypto from "crypto"; - -var config: any; - -export default { - init: async function init(defaultOpts: any = DefaultOptions) { - config = await db.collection("config").findOneOrFail({}); - return this.set((config || {}).merge(defaultOpts)); - }, - get: function get() { - return config as DefaultOptions; - }, - set: function set(val: any) { - config = val.merge(config); - return db.collection("config").update({}, { $set: val }, { upsert: true }); - }, -}; - -export interface RateLimitOptions { - bot?: number; - count: number; - window: number; - onyIp?: boolean; -} - -export interface Region { - id: string; - name: string; - vip: boolean; - custom: boolean; - deprecated: boolean; - optimal: boolean; -} - -export interface KafkaBroker { - ip: string; - port: number; -} - -export interface DefaultOptions { - gateway: { - endpointClient: string | null; - endpoint: string | null; - }; - cdn: { - endpointClient: string | null; - endpoint: string | null; - }; - general: { - instance_id: string; - }; - permissions: { - user: { - createGuilds: boolean; - }; - }; - limits: { - user: { - maxGuilds: number; - maxUsername: number; - maxFriends: number; - }; - guild: { - maxRoles: number; - maxMembers: number; - maxChannels: number; - maxChannelsInCategory: number; - hideOfflineMember: number; - }; - message: { - maxCharacters: number; - maxTTSCharacters: number; - maxReactions: number; - maxAttachmentSize: number; - maxBulkDelete: number; - }; - channel: { - maxPins: number; - maxTopic: number; - }; - rate: { - ip: Omit; - global: RateLimitOptions; - error: RateLimitOptions; - routes: { - guild: RateLimitOptions; - webhook: RateLimitOptions; - channel: RateLimitOptions; - auth: { - login: RateLimitOptions; - register: RateLimitOptions; - }; - // TODO: rate limit configuration for all routes - }; - }; - }; - security: { - autoUpdate: boolean | number; - requestSignature: string; - jwtSecret: string; - forwadedFor: string | null; // header to get the real user ip address - captcha: { - enabled: boolean; - service: "recaptcha" | "hcaptcha" | null; // TODO: hcaptcha, custom - sitekey: string | null; - secret: string | null; - }; - ipdataApiKey: string | null; - }; - login: { - requireCaptcha: boolean; - }; - register: { - email: { - necessary: boolean; // we have to use necessary instead of required as the cli tool uses json schema and can't use required - allowlist: boolean; - blocklist: boolean; - domains: string[]; - }; - dateOfBirth: { - necessary: boolean; - minimum: number; // in years - }; - requireCaptcha: boolean; - requireInvite: boolean; - allowNewRegistration: boolean; - allowMultipleAccounts: boolean; - blockProxies: boolean; - password: { - minLength: number; - minNumbers: number; - minUpperCase: number; - minSymbols: number; - }; - }; - regions: { - default: string; - available: Region[]; - }; - rabbitmq: { - host: string | null; - }; - kafka: { - brokers: KafkaBroker[] | null; - }; -} - -export const DefaultOptions: DefaultOptions = { - gateway: { - endpointClient: null, - endpoint: null, - }, - cdn: { - endpointClient: null, - endpoint: null, - }, - general: { - instance_id: Snowflake.generate(), - }, - permissions: { - user: { - createGuilds: true, - }, - }, - limits: { - user: { - maxGuilds: 100, - maxUsername: 32, - maxFriends: 1000, - }, - guild: { - maxRoles: 250, - maxMembers: 250000, - maxChannels: 500, - maxChannelsInCategory: 50, - hideOfflineMember: 1000, - }, - message: { - maxCharacters: 2000, - maxTTSCharacters: 200, - maxReactions: 20, - maxAttachmentSize: 8388608, - maxBulkDelete: 100, - }, - channel: { - maxPins: 50, - maxTopic: 1024, - }, - rate: { - ip: { - count: 500, - window: 5, - }, - global: { - count: 20, - window: 5, - bot: 250, - }, - error: { - count: 10, - window: 5, - }, - routes: { - guild: { - count: 5, - window: 5, - }, - webhook: { - count: 5, - window: 20, - }, - channel: { - count: 5, - window: 20, - }, - auth: { - login: { - count: 5, - window: 60, - }, - register: { - count: 2, - window: 60 * 60 * 12, - }, - }, - }, - }, - }, - security: { - autoUpdate: true, - requestSignature: crypto.randomBytes(32).toString("base64"), - jwtSecret: crypto.randomBytes(256).toString("base64"), - forwadedFor: null, - // forwadedFor: "X-Forwarded-For" // nginx/reverse proxy - // forwadedFor: "CF-Connecting-IP" // cloudflare: - captcha: { - enabled: false, - service: null, - sitekey: null, - secret: null, - }, - ipdataApiKey: "eca677b284b3bac29eb72f5e496aa9047f26543605efe99ff2ce35c9", - }, - login: { - requireCaptcha: false, - }, - register: { - email: { - necessary: true, - allowlist: false, - blocklist: true, - domains: [], // TODO: efficiently save domain blocklist in database - // domains: fs.readFileSync(__dirname + "/blockedEmailDomains.txt", { encoding: "utf8" }).split("\n"), - }, - dateOfBirth: { - necessary: true, - minimum: 13, - }, - requireInvite: false, - requireCaptcha: true, - allowNewRegistration: true, - allowMultipleAccounts: true, - blockProxies: true, - password: { - minLength: 8, - minNumbers: 2, - minUpperCase: 2, - minSymbols: 0, - }, - }, - regions: { - default: "fosscord", - available: [{ id: "fosscord", name: "Fosscord", vip: false, custom: false, deprecated: false, optimal: false }], - }, - rabbitmq: { - host: null, - }, - kafka: { - brokers: null, - }, -}; - -export const ConfigSchema = new Schema({}, { strict: false }); - -export interface DefaultOptionsDocument extends DefaultOptions, Document {} - -export const ConfigModel = model("Config", ConfigSchema, "config");