diff --git a/.gitignore b/.gitignore index bc780d64..9ac5d944 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ assets/cache .env config.json assets/cacheMisses +assets/client_test +scripts/client.js +src/api/middlewares/TestClient.ts +yarn.lock .vscode/settings.json diff --git a/package-lock.json b/package-lock.json index 8927001f..3713fc53 100644 Binary files a/package-lock.json and b/package-lock.json differ diff --git a/package.json b/package.json index 99a0f2d3..85edb0e5 100644 --- a/package.json +++ b/package.json @@ -38,14 +38,13 @@ }, "homepage": "https://fosscord.com", "devDependencies": { - "@types/amqplib": "^0.8.2", + "@types/amqplib": "^0.10.1", "@types/bcrypt": "^5.0.0", "@types/body-parser": "^1.19.2", "@types/cookie-parser": "^1.4.3", "@types/express": "^4.17.15", - "@types/i18next-node-fs-backend": "^2.1.1", "@types/json-bigint": "^1.0.1", - "@types/jsonwebtoken": "^8.5.9", + "@types/jsonwebtoken": "^9.0.1", "@types/morgan": "^1.9.3", "@types/multer": "^1.4.7", "@types/node": "^18.7.20", @@ -62,7 +61,7 @@ "husky": "^8.0.0", "prettier": "^2.7.1", "pretty-quick": "^3.1.3", - "typescript": "^4.9.4" + "typescript": "^5.0.2" }, "dependencies": { "@aws-sdk/client-s3": "^3.178.0", @@ -80,21 +79,20 @@ "exif-be-gone": "^1.3.1", "fast-zlib": "^2.0.1", "fido2-lib": "^3.3.5", - "file-type": "16.5", + "file-type": "16.5.4", "form-data": "^4.0.0", - "i18next": "^21.9.2", - "i18next-http-middleware": "^3.2.1", - "i18next-node-fs-backend": "^2.1.3", + "i18next": "^22.4.12", + "i18next-fs-backend": "^2.1.1", "image-size": "^1.0.2", "json-bigint": "^1.0.0", - "jsonwebtoken": "^8.5.1", + "jsonwebtoken": "^9.0.0", "lambert-server": "^1.2.12", "missing-native-js-functions": "^1.2.18", "module-alias": "^2.2.2", "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "node-2fa": "^2.0.3", - "node-fetch": "^2.6.7", + "node-fetch": "^2.6.9", "node-os-utils": "^1.3.7", "nodemailer": "^6.9.0", "picocolors": "^1.0.0", @@ -114,10 +112,10 @@ "@fosscord/util": "dist/util" }, "optionalDependencies": { + "better-sqlite3": "^8.2.0", "erlpack": "^0.1.4", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", - "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport", - "sqlite3": "^5.1.5" + "nodemailer-sendgrid-transport": "github:Maria-Golomb/nodemailer-sendgrid-transport" } } diff --git a/src/api/Server.ts b/src/api/Server.ts index 032e923e..f88f94c0 100644 --- a/src/api/Server.ts +++ b/src/api/Server.ts @@ -32,7 +32,7 @@ import "missing-native-js-functions"; import morgan from "morgan"; import path from "path"; import { red } from "picocolors"; -import { Authentication, CORS } from "./middlewares/"; +import { CORS, initAuthentication } from "./middlewares/"; import { BodyParser } from "./middlewares/BodyParser"; import { ErrorHandler } from "./middlewares/ErrorHandler"; import { initRateLimits } from "./middlewares/RateLimit"; @@ -106,7 +106,7 @@ export class FosscordServer extends Server { // @ts-ignore this.app = api; - api.use(Authentication); + initAuthentication(api); await initRateLimits(api); await initTranslation(api); diff --git a/src/api/middlewares/Authentication.ts b/src/api/middlewares/Authentication.ts index 400a16f4..0aa585e5 100644 --- a/src/api/middlewares/Authentication.ts +++ b/src/api/middlewares/Authentication.ts @@ -18,8 +18,9 @@ import { checkToken, Config, Rights } from "@fosscord/util"; import * as Sentry from "@sentry/node"; -import { NextFunction, Request, Response } from "express"; +import { NextFunction, Request, Response, Router } from "express"; import { HTTPError } from "lambert-server"; +import { createSecretKey, KeyObject } from "crypto"; export const NO_AUTHORIZATION_ROUTES = [ // Authentication routes @@ -69,6 +70,16 @@ declare global { } } +let jwtPublicKey: KeyObject; + +// Initialize the jwt secret as a key object so it does not need to be regenerated for each request. +export function initAuthentication(api: Router) { + jwtPublicKey = createSecretKey( + Buffer.from(Config.get().security.jwtSecret), + ); + api.use(Authentication); +} + export async function Authentication( req: Request, res: Response, @@ -90,11 +101,9 @@ export async function Authentication( Sentry.setUser({ id: req.user_id }); try { - const { jwtSecret } = Config.get().security; - const { decoded, user } = await checkToken( req.headers.authorization, - jwtSecret, + jwtPublicKey, ); req.token = decoded; diff --git a/src/api/middlewares/Translation.ts b/src/api/middlewares/Translation.ts index 60ff4ad7..0ddc56bb 100644 --- a/src/api/middlewares/Translation.ts +++ b/src/api/middlewares/Translation.ts @@ -18,11 +18,20 @@ import fs from "fs"; import path from "path"; -import i18next from "i18next"; -import i18nextMiddleware from "i18next-http-middleware"; -import i18nextBackend from "i18next-node-fs-backend"; +import i18next, { TFunction } from "i18next"; +import i18nextBackend from "i18next-fs-backend"; import { Router } from "express"; +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + t: TFunction; + language?: string; + } + } +} + const ASSET_FOLDER_PATH = path.join(__dirname, "..", "..", "..", "assets"); export async function initTranslation(router: Router) { @@ -34,21 +43,33 @@ export async function initTranslation(router: Router) { .filter((x) => x.endsWith(".json")) .map((x) => x.slice(0, x.length - 5)); - await i18next - .use(i18nextBackend) - .use(i18nextMiddleware.LanguageDetector) - .init({ - preload: languages, - // debug: true, - fallbackLng: "en", - ns, - backend: { - loadPath: - path.join(ASSET_FOLDER_PATH, "locales") + - "/{{lng}}/{{ns}}.json", - }, - load: "all", - }); + await i18next.use(i18nextBackend).init({ + preload: languages, + // debug: true, + fallbackLng: "en", + ns, + backend: { + loadPath: + path.join(ASSET_FOLDER_PATH, "locales") + + "/{{lng}}/{{ns}}.json", + }, + load: "all", + }); - router.use(i18nextMiddleware.handle(i18next, {})); + router.use((req, res, next) => { + let lng = "en"; + if (req.headers["accept-language"]) { + lng = req.headers["accept-language"].split(",")[0]; + } + req.language = lng; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + req.t = (key: string | string[], options?: any) => { + return i18next.t(key, { + ...options, + lng, + }); + }; + next(); + }); } diff --git a/src/api/routes/guilds/#guild_id/templates.ts b/src/api/routes/guilds/#guild_id/templates.ts index 8a8c53fe..707675e5 100644 --- a/src/api/routes/guilds/#guild_id/templates.ts +++ b/src/api/routes/guilds/#guild_id/templates.ts @@ -39,6 +39,7 @@ const TemplateGuildProjection: (keyof Guild)[] = [ "system_channel_id", "system_channel_flags", "icon", + "id", ]; router.get("/", route({}), async (req: Request, res: Response) => { diff --git a/src/api/routes/users/@me/notes.ts b/src/api/routes/users/@me/notes.ts index 87d45277..07c07d72 100644 --- a/src/api/routes/users/@me/notes.ts +++ b/src/api/routes/users/@me/notes.ts @@ -52,17 +52,17 @@ router.put("/:id", route({}), async (req: Request, res: Response) => { where: { owner: { id: owner.id }, target: { id: target.id } }, }) ) { - Note.update( + await Note.update( { owner: { id: owner.id }, target: { id: target.id } }, { owner, target, content: note }, ); } else { - Note.insert({ + await Note.create({ id: Snowflake.generate(), owner, target, content: note, - }); + }).save(); } } else { await Note.delete({ diff --git a/src/api/util/handlers/Message.ts b/src/api/util/handlers/Message.ts index 65dbcdfe..8818904e 100644 --- a/src/api/util/handlers/Message.ts +++ b/src/api/util/handlers/Message.ts @@ -274,7 +274,7 @@ export async function sendMessage(opts: MessageOptions) { const message = await handleMessage({ ...opts, timestamp: new Date() }); await Promise.all([ - Message.insert(message), + message.save(), emitEvent({ event: "MESSAGE_CREATE", channel_id: opts.channel_id, diff --git a/src/util/cache/Cache.ts b/src/util/cache/Cache.ts new file mode 100644 index 00000000..fb66c2e3 --- /dev/null +++ b/src/util/cache/Cache.ts @@ -0,0 +1,85 @@ +/* + Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2023 Fosscord and Fosscord Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { EntityMetadata, FindOptionsWhere } from "typeorm"; +import { LocalCache } from "./LocalCache"; + +declare module "typeorm" { + interface BaseEntity { + metadata?: EntityMetadata; + cache: CacheManager; + } +} + +export type BaseEntityWithId = { id: string; [name: string]: any }; +export type Criteria = + | string + | string[] + | number + | number[] + | FindOptionsWhere; + +export interface Cache { + get(id: string): BaseEntityWithId | undefined; + set(id: string, entity: BaseEntityWithId): this; + find(options: Record): BaseEntityWithId | undefined; + filter(options: Record): BaseEntityWithId[]; + delete(id: string): boolean; +} + +export class CacheManager { + // last access time to automatically remove old entities from cache after 5 minutes of inactivity (to prevent memory leaks) + cache: Cache; + + constructor() { + this.cache = new LocalCache(); + // TODO: Config.get().cache.redis; + } + + delete(id: string) { + return this.cache.delete(id); + } + + insert(entity: BaseEntityWithId) { + if (!entity.id) return; + + return this.cache.set(entity.id, entity); + } + + find(options?: Record, select?: string[] | undefined) { + if (!options) return null; + const entity = this.cache.find(options); + if (!entity) return null; + if (!select) return entity; + + const result = {}; + for (const prop of select) { + // @ts-ignore + result[prop] = entity[prop]; + } + + // @ts-ignore + return entity.constructor.create(result); + } + + filter(options: Record) { + return this.cache.filter(options); + } +} diff --git a/src/util/cache/EntityCache.ts b/src/util/cache/EntityCache.ts new file mode 100644 index 00000000..9135fef3 --- /dev/null +++ b/src/util/cache/EntityCache.ts @@ -0,0 +1,160 @@ +/* + Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2023 Fosscord and Fosscord Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/* eslint-disable */ +import { + DataSource, + FindOneOptions, + EntityNotFoundError, + FindOptionsWhere, +} from "typeorm"; +import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity"; +import { BaseClassWithId } from "../entities/BaseClass"; +import { Config, getDatabase } from "../util"; +import { CacheManager } from "./Cache"; + +function getObjectKeysAsArray(obj?: Record) { + if (!obj) return []; + if (Array.isArray(obj)) return obj; + return Object.keys(obj); +} + +export type ThisType = { + new (): T; +} & typeof BaseEntityCache; + +interface BaseEntityCache { + constructor: typeof BaseEntityCache; +} + +// @ts-ignore +class BaseEntityCache extends BaseClassWithId { + static cache: CacheManager; + static cacheEnabled: boolean; + + public get metadata() { + return getDatabase()?.getMetadata(this.constructor)!; + } + + static useDataSource(dataSource: DataSource | null) { + super.useDataSource(dataSource); + this.cacheEnabled = Config.get().cache.enabled ?? true; + if (Config.get().cache.redis) return; // TODO: Redis cache + if (!this.cacheEnabled) return; + this.cache = new CacheManager(); + } + + static async findOne( + this: ThisType, + options: FindOneOptions, + ) { + // @ts-ignore + if (!this.cacheEnabled) return super.findOne(options); + let select = getObjectKeysAsArray(options.select); + + if (!select.length) { + // get all columns that are marked as select + getDatabase() + ?.getMetadata(this) + .columns.forEach((x) => { + if (!x.isSelect) return; + select.push(x.propertyName); + }); + } + if (options.relations) { + select.push(...getObjectKeysAsArray(options.relations)); + } + + const cacheResult = this.cache.find(options.where as never, select); + if (cacheResult) { + const hasAllProps = select.every((key) => { + if (key.includes(".")) return true; // @ts-ignore + return cacheResult[key] !== undefined; + }); + // console.log(`[Cache] get ${cacheResult.id} from ${cacheResult.constructor.name}`,); + if (hasAllProps) return cacheResult; + } + + // @ts-ignore + const result = await super.findOne(options); + if (!result) return null; + + this.cache.insert(result as any); + + return result; + } + + static async findOneOrFail( + this: ThisType, + options: FindOneOptions, + ) { + const result = await this.findOne(options); + if (!result) throw new EntityNotFoundError(this, options); + return result; + } + + save() { + if (this.constructor.cacheEnabled) this.constructor.cache.insert(this); + return super.save(); + } + + remove() { + if (this.constructor.cacheEnabled) + this.constructor.cache.delete(this.id); + return super.remove(); + } + + static async update( + this: ThisType, + criteria: FindOptionsWhere, + partialEntity: QueryDeepPartialEntity, + ) { + // @ts-ignore + const result = super.update(criteria, partialEntity); + if (!this.cacheEnabled) return result; + + const entities = this.cache.filter(criteria as never); + for (const entity of entities) { + // @ts-ignore + partialEntity.id = entity.id; + this.cache.insert(partialEntity as never); + } + + return result; + } + + static async delete( + this: ThisType, + criteria: FindOptionsWhere, + ) { + // @ts-ignore + const result = super.delete(criteria); + if (!this.cacheEnabled) return result; + + const entities = this.cache.filter(criteria as never); + for (const entity of entities) { + this.cache.delete(entity.id); + } + + return result; + } +} + +// needed, because typescript can't infer the type of the static methods with generics +const EntityCache = BaseEntityCache as unknown as typeof BaseClassWithId; + +export { EntityCache }; diff --git a/src/util/cache/LocalCache.ts b/src/util/cache/LocalCache.ts new file mode 100644 index 00000000..9c4f23b3 --- /dev/null +++ b/src/util/cache/LocalCache.ts @@ -0,0 +1,94 @@ +/* + Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2023 Fosscord and Fosscord Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +import { BaseEntityWithId, Cache } from "./Cache"; + +export const cacheTimeout = 1000 * 60 * 5; + +export class LocalCache extends Map implements Cache { + last_access = new Map(); + + constructor() { + super(); + + setInterval(() => { + const now = Date.now(); + for (const [key, value] of this.last_access) { + if (now - value > cacheTimeout) { + this.delete(key); + this.last_access.delete(key); + } + } + }, cacheTimeout); + } + + set(key: string, value: BaseEntityWithId): this { + if (this.has(key)) { + this.update(key, value); + return this; + } + this.last_access.set(key, Date.now()); + return super.set(key, value as never); + } + + get(key: string) { + const value = super.get(key); + if (value) this.last_access.set(key, Date.now()); + return value; + } + + update(id: string, entity: BaseEntityWithId) { + const oldEntity = this.get(id); + if (!oldEntity) return; + for (const key in entity) { + // @ts-ignore + if (entity[key] === undefined) continue; // @ts-ignore + oldEntity[key] = entity[key]; + } + } + + find(options: Record): BaseEntityWithId | undefined { + if (options.id && Object.keys(options).length === 1) { + return this.get(options.id); + } + for (const entity of this.values()) { + if (objectFulfillsQuery(entity, options)) return entity; + } + } + + filter(options: Record): BaseEntityWithId[] { + const result = []; + for (const entity of this.values()) { + if (objectFulfillsQuery(entity, options)) { + result.push(entity); + } + } + return result; + } +} + +function objectFulfillsQuery( + entity: BaseEntityWithId, + options: Record, +) { + for (const key in options) { + // @ts-ignore + if (entity[key] !== options[key]) return false; + } + return true; +} diff --git a/src/util/cache/index.ts b/src/util/cache/index.ts new file mode 100644 index 00000000..43f0aa96 --- /dev/null +++ b/src/util/cache/index.ts @@ -0,0 +1,3 @@ +export * from "./EntityCache"; +export * from "./Cache"; +export * from "./LocalCache"; diff --git a/src/util/config/Config.ts b/src/util/config/Config.ts index 90b98b7a..7de01b66 100644 --- a/src/util/config/Config.ts +++ b/src/util/config/Config.ts @@ -37,6 +37,7 @@ import { SecurityConfiguration, SentryConfiguration, TemplateConfiguration, + CacheConfiguration, } from "../config"; export class ConfigValue { @@ -61,4 +62,5 @@ export class ConfigValue { email: EmailConfiguration = new EmailConfiguration(); passwordReset: PasswordResetConfiguration = new PasswordResetConfiguration(); + cache: CacheConfiguration = new CacheConfiguration(); } diff --git a/src/util/config/types/CacheConfiguration.ts b/src/util/config/types/CacheConfiguration.ts new file mode 100644 index 00000000..4178090e --- /dev/null +++ b/src/util/config/types/CacheConfiguration.ts @@ -0,0 +1,22 @@ +/* + Fosscord: A FOSS re-implementation and extension of the Discord.com backend. + Copyright (C) 2023 Fosscord and Fosscord Contributors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +*/ + +export class CacheConfiguration { + enabled: boolean | null = true; + redis: string | null = null; +} diff --git a/src/util/config/types/index.ts b/src/util/config/types/index.ts index 782ebfc3..a52ca627 100644 --- a/src/util/config/types/index.ts +++ b/src/util/config/types/index.ts @@ -17,6 +17,7 @@ */ export * from "./ApiConfiguration"; +export * from "./CacheConfiguration"; export * from "./CdnConfiguration"; export * from "./DefaultsConfiguration"; export * from "./EmailConfiguration"; diff --git a/src/util/entities/Application.ts b/src/util/entities/Application.ts index 962b2a8e..e0cdc481 100644 --- a/src/util/entities/Application.ts +++ b/src/util/entities/Application.ts @@ -17,12 +17,12 @@ */ import { Column, Entity, JoinColumn, ManyToOne, OneToOne } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Team } from "./Team"; import { User } from "./User"; @Entity("applications") -export class Application extends BaseClass { +export class Application extends EntityCache { @Column() name: string; diff --git a/src/util/entities/Attachment.ts b/src/util/entities/Attachment.ts index d60ac41c..521be583 100644 --- a/src/util/entities/Attachment.ts +++ b/src/util/entities/Attachment.ts @@ -25,11 +25,11 @@ import { RelationId, } from "typeorm"; import { URL } from "url"; +import { EntityCache } from "../cache"; import { deleteFile } from "../util/cdn"; -import { BaseClass } from "./BaseClass"; @Entity("attachments") -export class Attachment extends BaseClass { +export class Attachment extends EntityCache { @Column() filename: string; // name of file attached diff --git a/src/util/entities/AuditLog.ts b/src/util/entities/AuditLog.ts index e47f92fb..9768ef04 100644 --- a/src/util/entities/AuditLog.ts +++ b/src/util/entities/AuditLog.ts @@ -17,7 +17,7 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { ChannelPermissionOverwrite } from "./Channel"; import { User } from "./User"; @@ -112,7 +112,7 @@ export enum AuditLogEvents { } @Entity("audit_logs") -export class AuditLog extends BaseClass { +export class AuditLog extends EntityCache { @JoinColumn({ name: "target_id" }) @ManyToOne(() => User) target?: User; diff --git a/src/util/entities/BackupCodes.ts b/src/util/entities/BackupCodes.ts index 61e8f12a..3d40338e 100644 --- a/src/util/entities/BackupCodes.ts +++ b/src/util/entities/BackupCodes.ts @@ -17,12 +17,12 @@ */ import { Column, Entity, JoinColumn, ManyToOne } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; import crypto from "crypto"; @Entity("backup_codes") -export class BackupCode extends BaseClass { +export class BackupCode extends EntityCache { @JoinColumn({ name: "user_id" }) @ManyToOne(() => User, { onDelete: "CASCADE" }) user: User; diff --git a/src/util/entities/Ban.ts b/src/util/entities/Ban.ts index 1693cd40..2b3b688b 100644 --- a/src/util/entities/Ban.ts +++ b/src/util/entities/Ban.ts @@ -17,12 +17,12 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; import { User } from "./User"; @Entity("bans") -export class Ban extends BaseClass { +export class Ban extends EntityCache { @Column({ nullable: true }) @RelationId((ban: Ban) => ban.user) user_id: string; diff --git a/src/util/entities/BaseClass.ts b/src/util/entities/BaseClass.ts index f4b3cf59..d2770bfd 100644 --- a/src/util/entities/BaseClass.ts +++ b/src/util/entities/BaseClass.ts @@ -25,16 +25,12 @@ import { PrimaryColumn, } from "typeorm"; import { Snowflake } from "../util/Snowflake"; -import { getDatabase } from "../util/Database"; import { OrmUtils } from "../imports/OrmUtils"; +import { getDatabase } from "../util"; export class BaseClassWithoutId extends BaseEntity { - private get construct() { - return this.constructor; - } - - private get metadata() { - return getDatabase()?.getMetadata(this.construct); + public get metadata() { + return getDatabase()?.getMetadata(this.constructor); } assign(props: object) { @@ -61,31 +57,13 @@ export class BaseClassWithoutId extends BaseEntity { ), ); } - - static increment( - conditions: FindOptionsWhere, - propertyPath: string, - value: number | string, - ) { - const repository = this.getRepository(); - return repository.increment(conditions, propertyPath, value); - } - - static decrement( - conditions: FindOptionsWhere, - propertyPath: string, - value: number | string, - ) { - const repository = this.getRepository(); - return repository.decrement(conditions, propertyPath, value); - } } export const PrimaryIdColumn = process.env.DATABASE?.startsWith("mongodb") ? ObjectIdColumn : PrimaryColumn; -export class BaseClass extends BaseClassWithoutId { +export class BaseClassWithId extends BaseClassWithoutId { @PrimaryIdColumn() id: string = Snowflake.generate(); @@ -94,4 +72,22 @@ export class BaseClass extends BaseClassWithoutId { _do_validate() { if (!this.id) this.id = Snowflake.generate(); } + + static increment( + conditions: FindOptionsWhere, + propertyPath: string, + value: number | string, + ) { + const repository = this.getRepository(); + return repository.increment(conditions, propertyPath, value); + } + + static decrement( + conditions: FindOptionsWhere, + propertyPath: string, + value: number | string, + ) { + const repository = this.getRepository(); + return repository.decrement(conditions, propertyPath, value); + } } diff --git a/src/util/entities/Channel.ts b/src/util/entities/Channel.ts index 9ce04848..218907ed 100644 --- a/src/util/entities/Channel.ts +++ b/src/util/entities/Channel.ts @@ -24,7 +24,7 @@ import { OneToMany, RelationId, } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; import { PublicUserProjection, User } from "./User"; import { HTTPError } from "lambert-server"; @@ -70,7 +70,7 @@ export enum ChannelType { } @Entity("channels") -export class Channel extends BaseClass { +export class Channel extends EntityCache { @Column() created_at: Date; diff --git a/src/util/entities/ClientRelease.ts b/src/util/entities/ClientRelease.ts index cfbc3a9b..a26bcd8a 100644 --- a/src/util/entities/ClientRelease.ts +++ b/src/util/entities/ClientRelease.ts @@ -17,10 +17,10 @@ */ import { Column, Entity } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; @Entity("client_release") -export class Release extends BaseClass { +export class Release extends EntityCache { @Column() name: string; diff --git a/src/util/entities/ConnectedAccount.ts b/src/util/entities/ConnectedAccount.ts index 33550197..8ef03fea 100644 --- a/src/util/entities/ConnectedAccount.ts +++ b/src/util/entities/ConnectedAccount.ts @@ -17,7 +17,7 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; export type PublicConnectedAccount = Pick< @@ -26,7 +26,7 @@ export type PublicConnectedAccount = Pick< >; @Entity("connected_accounts") -export class ConnectedAccount extends BaseClass { +export class ConnectedAccount extends EntityCache { @Column({ nullable: true }) @RelationId((account: ConnectedAccount) => account.user) user_id: string; diff --git a/src/util/entities/EmbedCache.ts b/src/util/entities/EmbedCache.ts index 8ff2a457..49361032 100644 --- a/src/util/entities/EmbedCache.ts +++ b/src/util/entities/EmbedCache.ts @@ -16,12 +16,12 @@ along with this program. If not, see . */ -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Entity, Column } from "typeorm"; import { Embed } from "./Message"; @Entity("embed_cache") -export class EmbedCache extends BaseClass { +export class EmbedCache extends EntityCache { @Column() url: string; diff --git a/src/util/entities/Emoji.ts b/src/util/entities/Emoji.ts index f1bf2d59..bf15f4ec 100644 --- a/src/util/entities/Emoji.ts +++ b/src/util/entities/Emoji.ts @@ -18,11 +18,11 @@ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; import { User } from "."; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; @Entity("emojis") -export class Emoji extends BaseClass { +export class Emoji extends EntityCache { @Column() animated: boolean; diff --git a/src/util/entities/Encryption.ts b/src/util/entities/Encryption.ts index 8325bdee..24fbaa6c 100644 --- a/src/util/entities/Encryption.ts +++ b/src/util/entities/Encryption.ts @@ -17,10 +17,10 @@ */ import { Column, Entity } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; @Entity("security_settings") -export class SecuritySettings extends BaseClass { +export class SecuritySettings extends EntityCache { @Column({ nullable: true }) guild_id: string; diff --git a/src/util/entities/Guild.ts b/src/util/entities/Guild.ts index b1693838..7253f118 100644 --- a/src/util/entities/Guild.ts +++ b/src/util/entities/Guild.ts @@ -26,7 +26,7 @@ import { } from "typeorm"; import { Config, handleFile, Snowflake } from ".."; import { Ban } from "./Ban"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Channel } from "./Channel"; import { Emoji } from "./Emoji"; import { Invite } from "./Invite"; @@ -67,7 +67,7 @@ export const PublicGuildRelations = [ ]; @Entity("guilds") -export class Guild extends BaseClass { +export class Guild extends EntityCache { @Column({ nullable: true }) @RelationId((guild: Guild) => guild.afk_channel) afk_channel_id?: string; diff --git a/src/util/entities/Message.ts b/src/util/entities/Message.ts index 519c431e..db98a6f2 100644 --- a/src/util/entities/Message.ts +++ b/src/util/entities/Message.ts @@ -34,7 +34,7 @@ import { OneToMany, RelationId, } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; import { Webhook } from "./Webhook"; import { Sticker } from "./Sticker"; @@ -70,7 +70,7 @@ export enum MessageType { @Entity("messages") @Index(["channel_id", "id"], { unique: true }) -export class Message extends BaseClass { +export class Message extends EntityCache { @Column({ nullable: true }) @RelationId((message: Message) => message.channel) @Index() diff --git a/src/util/entities/Note.ts b/src/util/entities/Note.ts index 196f6861..2005ab0f 100644 --- a/src/util/entities/Note.ts +++ b/src/util/entities/Note.ts @@ -17,12 +17,12 @@ */ import { Column, Entity, JoinColumn, ManyToOne, Unique } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; @Entity("notes") @Unique(["owner", "target"]) -export class Note extends BaseClass { +export class Note extends EntityCache { @JoinColumn({ name: "owner_id" }) @ManyToOne(() => User, { onDelete: "CASCADE" }) owner: User; diff --git a/src/util/entities/RateLimit.ts b/src/util/entities/RateLimit.ts index 8d00f59a..5402ad06 100644 --- a/src/util/entities/RateLimit.ts +++ b/src/util/entities/RateLimit.ts @@ -17,10 +17,10 @@ */ import { Column, Entity } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; @Entity("rate_limits") -export class RateLimit extends BaseClass { +export class RateLimit extends EntityCache { @Column() // no relation as it also executor_id: string; diff --git a/src/util/entities/ReadState.ts b/src/util/entities/ReadState.ts index 1b280d12..56856a1e 100644 --- a/src/util/entities/ReadState.ts +++ b/src/util/entities/ReadState.ts @@ -24,7 +24,7 @@ import { ManyToOne, RelationId, } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Channel } from "./Channel"; import { User } from "./User"; @@ -34,7 +34,7 @@ import { User } from "./User"; @Entity("read_states") @Index(["channel_id", "user_id"], { unique: true }) -export class ReadState extends BaseClass { +export class ReadState extends EntityCache { @Column() @RelationId((read_state: ReadState) => read_state.channel) channel_id: string; diff --git a/src/util/entities/Recipient.ts b/src/util/entities/Recipient.ts index 797349e5..32ae7936 100644 --- a/src/util/entities/Recipient.ts +++ b/src/util/entities/Recipient.ts @@ -17,10 +17,10 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; @Entity("recipients") -export class Recipient extends BaseClass { +export class Recipient extends EntityCache { @Column() @RelationId((recipient: Recipient) => recipient.channel) channel_id: string; diff --git a/src/util/entities/Relationship.ts b/src/util/entities/Relationship.ts index 740095c2..e67557aa 100644 --- a/src/util/entities/Relationship.ts +++ b/src/util/entities/Relationship.ts @@ -24,7 +24,7 @@ import { ManyToOne, RelationId, } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; export enum RelationshipType { @@ -36,7 +36,7 @@ export enum RelationshipType { @Entity("relationships") @Index(["from_id", "to_id"], { unique: true }) -export class Relationship extends BaseClass { +export class Relationship extends EntityCache { @Column({}) @RelationId((relationship: Relationship) => relationship.from) from_id: string; diff --git a/src/util/entities/Role.ts b/src/util/entities/Role.ts index 85877c12..2fb63e93 100644 --- a/src/util/entities/Role.ts +++ b/src/util/entities/Role.ts @@ -18,11 +18,11 @@ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; @Entity("roles") -export class Role extends BaseClass { +export class Role extends EntityCache { @Column({ nullable: true }) @RelationId((role: Role) => role.guild) guild_id: string; diff --git a/src/util/entities/SecurityKey.ts b/src/util/entities/SecurityKey.ts index fd7a4c5e..cdcc814b 100644 --- a/src/util/entities/SecurityKey.ts +++ b/src/util/entities/SecurityKey.ts @@ -17,11 +17,11 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; @Entity("security_keys") -export class SecurityKey extends BaseClass { +export class SecurityKey extends EntityCache { @Column({ nullable: true }) @RelationId((key: SecurityKey) => key.user) user_id: string; diff --git a/src/util/entities/Session.ts b/src/util/entities/Session.ts index 6c6f7caa..a03ff114 100644 --- a/src/util/entities/Session.ts +++ b/src/util/entities/Session.ts @@ -17,7 +17,7 @@ */ import { User } from "./User"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; import { Status } from "../interfaces/Status"; import { Activity } from "../interfaces/Activity"; @@ -25,7 +25,7 @@ import { Activity } from "../interfaces/Activity"; //TODO we need to remove all sessions on server start because if the server crashes without closing websockets it won't delete them @Entity("sessions") -export class Session extends BaseClass { +export class Session extends EntityCache { @Column({ nullable: true }) @RelationId((session: Session) => session.user) user_id: string; diff --git a/src/util/entities/Sticker.ts b/src/util/entities/Sticker.ts index cd07e65a..9139a5b1 100644 --- a/src/util/entities/Sticker.ts +++ b/src/util/entities/Sticker.ts @@ -18,7 +18,7 @@ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; import { User } from "./User"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; export enum StickerType { @@ -34,7 +34,7 @@ export enum StickerFormatType { } @Entity("stickers") -export class Sticker extends BaseClass { +export class Sticker extends EntityCache { @Column() name: string; diff --git a/src/util/entities/StickerPack.ts b/src/util/entities/StickerPack.ts index 61ab1287..cc5c392b 100644 --- a/src/util/entities/StickerPack.ts +++ b/src/util/entities/StickerPack.ts @@ -25,10 +25,10 @@ import { RelationId, } from "typeorm"; import { Sticker } from "."; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; @Entity("sticker_packs") -export class StickerPack extends BaseClass { +export class StickerPack extends EntityCache { @Column() name: string; diff --git a/src/util/entities/Team.ts b/src/util/entities/Team.ts index 7bedc4af..04d34d30 100644 --- a/src/util/entities/Team.ts +++ b/src/util/entities/Team.ts @@ -24,12 +24,12 @@ import { OneToMany, RelationId, } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { TeamMember } from "./TeamMember"; import { User } from "./User"; @Entity("teams") -export class Team extends BaseClass { +export class Team extends EntityCache { @Column({ nullable: true }) icon?: string; diff --git a/src/util/entities/TeamMember.ts b/src/util/entities/TeamMember.ts index 539da957..d2053bb7 100644 --- a/src/util/entities/TeamMember.ts +++ b/src/util/entities/TeamMember.ts @@ -17,7 +17,7 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { User } from "./User"; export enum TeamMemberState { @@ -26,7 +26,7 @@ export enum TeamMemberState { } @Entity("team_members") -export class TeamMember extends BaseClass { +export class TeamMember extends EntityCache { @Column({ type: "int" }) membership_state: TeamMemberState; diff --git a/src/util/entities/Template.ts b/src/util/entities/Template.ts index c417f1f0..101329d1 100644 --- a/src/util/entities/Template.ts +++ b/src/util/entities/Template.ts @@ -17,12 +17,12 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Guild } from "./Guild"; import { User } from "./User"; @Entity("templates") -export class Template extends BaseClass { +export class Template extends EntityCache { @Column({ unique: true }) code: string; diff --git a/src/util/entities/User.ts b/src/util/entities/User.ts index df9af328..ec7a11d6 100644 --- a/src/util/entities/User.ts +++ b/src/util/entities/User.ts @@ -34,7 +34,7 @@ import { trimSpecial, } from ".."; import { BitField } from "../util/BitField"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { ConnectedAccount } from "./ConnectedAccount"; import { Member } from "./Member"; import { Relationship } from "./Relationship"; @@ -94,7 +94,7 @@ export interface UserPrivate extends Pick { } @Entity("users") -export class User extends BaseClass { +export class User extends EntityCache { @Column() username: string; // username max length 32, min 2 (should be configurable) diff --git a/src/util/entities/VoiceState.ts b/src/util/entities/VoiceState.ts index b291c4d3..826ae0e3 100644 --- a/src/util/entities/VoiceState.ts +++ b/src/util/entities/VoiceState.ts @@ -17,7 +17,7 @@ */ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Channel } from "./Channel"; import { Guild } from "./Guild"; import { User } from "./User"; @@ -25,7 +25,7 @@ import { Member } from "./Member"; //https://gist.github.com/vassjozsef/e482c65df6ee1facaace8b3c9ff66145#file-voice_state-ex @Entity("voice_states") -export class VoiceState extends BaseClass { +export class VoiceState extends EntityCache { @Column({ nullable: true }) @RelationId((voice_state: VoiceState) => voice_state.guild) guild_id: string; diff --git a/src/util/entities/Webhook.ts b/src/util/entities/Webhook.ts index 91498a22..eeabfea5 100644 --- a/src/util/entities/Webhook.ts +++ b/src/util/entities/Webhook.ts @@ -18,7 +18,7 @@ import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm"; import { Application } from "./Application"; -import { BaseClass } from "./BaseClass"; +import { EntityCache } from "../cache"; import { Channel } from "./Channel"; import { Guild } from "./Guild"; import { User } from "./User"; @@ -30,7 +30,7 @@ export enum WebhookType { } @Entity("webhooks") -export class Webhook extends BaseClass { +export class Webhook extends EntityCache { @Column({ type: "int" }) type: WebhookType; diff --git a/src/util/schemas/Validator.ts b/src/util/schemas/Validator.ts index 1de511d3..84677e25 100644 --- a/src/util/schemas/Validator.ts +++ b/src/util/schemas/Validator.ts @@ -43,7 +43,7 @@ export const ajv = new Ajv({ allowUnionTypes: true, }); -addFormats(ajv); +addFormats(ajv as never); export function validateSchema(schema: string, data: G): G { const valid = ajv.validate(schema, normalizeBody(data)); diff --git a/src/util/util/Database.ts b/src/util/util/Database.ts index e23bb895..b0d9d24e 100644 --- a/src/util/util/Database.ts +++ b/src/util/util/Database.ts @@ -38,7 +38,8 @@ const dbConnectionString = const DatabaseType = dbConnectionString.includes("://") ? dbConnectionString.split(":")[0]?.replace("+srv", "") - : "sqlite"; + : "better-sqlite3"; + const isSqlite = DatabaseType.includes("sqlite"); const DataSourceOptions = new DataSource({ @@ -50,7 +51,7 @@ const DataSourceOptions = new DataSource({ database: isSqlite ? dbConnectionString : undefined, entities: [path.join(__dirname, "..", "entities", "*.js")], synchronize: !!process.env.DB_SYNC, - logging: false, + logging: process.env["DB_LOGGING"] === "true", bigNumberStrings: false, supportBigNumbers: true, name: "default", @@ -77,7 +78,13 @@ export async function initDatabase(): Promise { } if (!process.env.DB_SYNC) { - const supported = ["mysql", "mariadb", "postgres", "sqlite"]; + const supported = [ + "mysql", + "mariadb", + "postgres", + "sqlite", + "better-sqlite3", + ]; if (!supported.includes(DatabaseType)) { console.log( "[Database]" + @@ -113,10 +120,10 @@ export async function initDatabase(): Promise { // Manually insert every current migration to prevent this: await Promise.all( dbConnection.migrations.map((migration) => - Migration.insert({ + Migration.create({ name: migration.name, timestamp: Date.now(), - }), + }).save(), ), ); } else { diff --git a/src/util/util/Permissions.ts b/src/util/util/Permissions.ts index 996c72ea..4aeb9268 100644 --- a/src/util/util/Permissions.ts +++ b/src/util/util/Permissions.ts @@ -257,23 +257,26 @@ export async function getPermission( } if (guild_id) { - guild = await Guild.findOneOrFail({ - where: { id: guild_id }, - select: ["id", "owner_id", ...(opts.guild_select || [])], - relations: opts.guild_relations, - }); + const result = await Promise.all([ + Guild.findOneOrFail({ + where: { id: guild_id }, + select: ["id", "owner_id", ...(opts.guild_select || [])], + relations: opts.guild_relations, + }), + Member.findOneOrFail({ + where: { guild_id, id: user_id }, + relations: ["roles", ...(opts.member_relations || [])], + // select: [ + // "id", // TODO: Bug in typeorm? adding these selects breaks the query. + // "roles", + // ...(opts.member_select || []), + // ], + }), + ]); + guild = result[0]; + member = result[1]; if (guild.owner_id === user_id) return new Permissions(Permissions.FLAGS.ADMINISTRATOR); - - member = await Member.findOneOrFail({ - where: { guild_id, id: user_id }, - relations: ["roles", ...(opts.member_relations || [])], - // select: [ - // "id", // TODO: Bug in typeorm? adding these selects breaks the query. - // "roles", - // ...(opts.member_select || []), - // ], - }); } let recipient_ids = channel?.recipients?.map((x) => x.user_id); diff --git a/src/util/util/Token.ts b/src/util/util/Token.ts index 50e63c5f..b839a519 100644 --- a/src/util/util/Token.ts +++ b/src/util/util/Token.ts @@ -19,6 +19,7 @@ import jwt, { VerifyOptions } from "jsonwebtoken"; import { Config } from "./Config"; import { User } from "../entities"; +import { KeyObject } from "crypto"; export const JWTOptions: VerifyOptions = { algorithms: ["HS256"] }; @@ -62,7 +63,7 @@ async function checkEmailToken( export function checkToken( token: string, - jwtSecret: string, + jwtSecret: string | KeyObject, isEmailVerification = false, ): Promise { return new Promise((res, rej) => { @@ -86,7 +87,7 @@ export function checkToken( const user = await User.findOne({ where: { id: decoded.id }, - select: ["data", "bot", "disabled", "deleted", "rights"], + select: ["id", "data", "bot", "disabled", "deleted", "rights"], }); if (!user) return rej("Invalid Token"); diff --git a/src/util/util/index.ts b/src/util/util/index.ts index 838239b7..10ae7152 100644 --- a/src/util/util/index.ts +++ b/src/util/util/index.ts @@ -24,7 +24,7 @@ export * from "./cdn"; export * from "./Config"; export * from "./Constants"; export * from "./Database"; -export * from "./email"; +export * from "./email/index"; export * from "./Event"; export * from "./FieldError"; export * from "./Intents";