diff --git a/api/package-lock.json b/api/package-lock.json index 265e70bb..92351949 100644 Binary files a/api/package-lock.json and b/api/package-lock.json differ diff --git a/api/package.json b/api/package.json index aae31e2d..5bb2d792 100644 --- a/api/package.json +++ b/api/package.json @@ -38,6 +38,8 @@ "homepage": "https://fosscord.com", "devDependencies": { "@babel/core": "^7.15.5", + "@babel/preset-env": "^7.15.8", + "@babel/preset-typescript": "^7.15.0", "@types/amqplib": "^0.8.1", "@types/bcrypt": "^5.0.0", "@types/express": "^4.17.9", @@ -45,6 +47,7 @@ "@types/jest": "^27.0.1", "@types/jest-expect-message": "^1.0.3", "@types/jsonwebtoken": "^8.5.0", + "@types/morgan": "^1.9.3", "@types/multer": "^1.4.5", "@types/node": "^14.17.9", "@types/node-fetch": "^2.5.7", @@ -57,8 +60,7 @@ "ts-node-dev": "^1.1.6", "ts-patch": "^1.4.4", "typescript": "^4.4.2", - "typescript-json-schema": "0.50.1", - "@types/morgan": "^1.9.3" + "typescript-json-schema": "0.50.1" }, "dependencies": { "@fosscord/util": "file:../util", @@ -77,7 +79,7 @@ "i18next-node-fs-backend": "^2.1.3", "image-size": "^1.0.0", "jsonwebtoken": "^8.5.1", - "lambert-server": "^1.2.11", + "lambert-server": "^1.2.12", "missing-native-js-functions": "^1.2.17", "morgan": "^1.10.0", "multer": "^1.4.2", diff --git a/api/src/middlewares/Authentication.ts b/api/src/middlewares/Authentication.ts index 5a082751..59a181e6 100644 --- a/api/src/middlewares/Authentication.ts +++ b/api/src/middlewares/Authentication.ts @@ -9,6 +9,8 @@ export const NO_AUTHORIZATION_ROUTES = [ "/ping", "/gateway", "/experiments", + "/-/readyz", + "/-/healthz", /\/guilds\/\d+\/widget\.(json|png)/ ]; diff --git a/api/src/routes/-/healthz.ts b/api/src/routes/-/healthz.ts new file mode 100644 index 00000000..a42575f8 --- /dev/null +++ b/api/src/routes/-/healthz.ts @@ -0,0 +1,17 @@ +import { Router, Response, Request } from "express"; +import { route } from "@fosscord/api"; +import { getConnection } from "typeorm"; + +const router = Router(); + +router.get("/", route({}), (req: Request, res: Response) => { + try { + // test that the database is alive & responding + getConnection(); + return res.sendStatus(200); + } catch(e) { + res.sendStatus(503); + } +}); + +export default router; diff --git a/api/src/routes/-/readyz.ts b/api/src/routes/-/readyz.ts new file mode 100644 index 00000000..a42575f8 --- /dev/null +++ b/api/src/routes/-/readyz.ts @@ -0,0 +1,17 @@ +import { Router, Response, Request } from "express"; +import { route } from "@fosscord/api"; +import { getConnection } from "typeorm"; + +const router = Router(); + +router.get("/", route({}), (req: Request, res: Response) => { + try { + // test that the database is alive & responding + getConnection(); + return res.sendStatus(200); + } catch(e) { + res.sendStatus(503); + } +}); + +export default router; diff --git a/api/src/routes/channels/#channel_id/messages/index.ts b/api/src/routes/channels/#channel_id/messages/index.ts index 20c102ef..3e26e930 100644 --- a/api/src/routes/channels/#channel_id/messages/index.ts +++ b/api/src/routes/channels/#channel_id/messages/index.ts @@ -10,7 +10,8 @@ import { getPermission, Message, MessageCreateEvent, - uploadFile + uploadFile, + Member } from "@fosscord/util"; import { HTTPError } from "lambert-server"; import { handleMessage, postHandleMessage, route } from "@fosscord/api"; @@ -187,33 +188,34 @@ router.post( message = await message.save(); - await channel.assign({ last_message_id: message.id }).save(); - if (channel.isDm()) { const channel_dto = await DmChannelDTO.from(channel); - for (let recipient of channel.recipients!) { - if (recipient.closed) { - await emitEvent({ - event: "CHANNEL_CREATE", - data: channel_dto.excludedRecipients([recipient.user_id]), - user_id: recipient.user_id - }); - } - } - //Only one recipients should be closed here, since in group DMs the recipient is deleted not closed + await Promise.all( - channel - .recipients!.filter((r) => r.closed) - .map(async (r) => { - r.closed = false; - return await r.save(); - }) + channel.recipients!.map((recipient) => { + if (recipient.closed) { + recipient.closed = false; + return Promise.all([ + recipient.save(), + emitEvent({ + event: "CHANNEL_CREATE", + data: channel_dto.excludedRecipients([recipient.user_id]), + user_id: recipient.user_id + }) + ]); + } + }) ); } - await emitEvent({ event: "MESSAGE_CREATE", channel_id: channel_id, data: message } as MessageCreateEvent); + await Promise.all([ + channel.assign({ last_message_id: message.id }).save(), + new Member({ id: req.user_id, last_message_id: message.id }).save(), + emitEvent({ event: "MESSAGE_CREATE", channel_id: channel_id, data: message } as MessageCreateEvent) + ]); + postHandleMessage(message).catch((e) => {}); // no await as it shouldnt block the message send function and silently catch error return res.json(message); diff --git a/api/src/routes/guilds/#guild_id/prune.ts b/api/src/routes/guilds/#guild_id/prune.ts new file mode 100644 index 00000000..92809985 --- /dev/null +++ b/api/src/routes/guilds/#guild_id/prune.ts @@ -0,0 +1,82 @@ +import { Router, Request, Response } from "express"; +import { Guild, Member, Snowflake } from "@fosscord/util"; +import { LessThan, IsNull } from "typeorm"; +import { route } from "@fosscord/api"; +const router = Router(); + +//Returns all inactive members, respecting role hierarchy +export const inactiveMembers = async (guild_id: string, user_id: string, days: number, roles: string[] = []) => { + var date = new Date(); + date.setDate(date.getDate() - days); + //Snowflake should have `generateFromTime` method? Or similar? + var minId = BigInt(date.valueOf() - Snowflake.EPOCH) << BigInt(22); + + var members = await Member.find({ + where: [ + { + guild_id, + last_message_id: LessThan(minId.toString()) + }, + { + last_message_id: IsNull() + } + ], + relations: ["roles"] + }); + console.log(members); + if (!members.length) return []; + + //I'm sure I can do this in the above db query ( and it would probably be better to do so ), but oh well. + if (roles.length && members.length) members = members.filter((user) => user.roles?.some((role) => roles.includes(role.id))); + + const me = await Member.findOneOrFail({ id: user_id, guild_id }, { relations: ["roles"] }); + const myHighestRole = Math.max(...(me.roles?.map((x) => x.position) || [])); + + const guild = await Guild.findOneOrFail({ where: { id: guild_id } }); + + members = members.filter( + (member) => + member.id !== guild.owner_id && //can't kick owner + member.roles?.some( + (role) => + role.position < myHighestRole || //roles higher than me can't be kicked + me.id === guild.owner_id //owner can kick anyone + ) + ); + + return members; +}; + +router.get("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => { + const days = parseInt(req.query.days as string); + + var roles = req.query.include_roles; + if (typeof roles === "string") roles = [roles]; //express will return array otherwise + + const members = await inactiveMembers(req.params.guild_id, req.user_id, days, roles as string[]); + + res.send({ pruned: members.length }); +}); + +export interface PruneSchema { + /** + * @min 0 + */ + days: number; +} + +router.post("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => { + const days = parseInt(req.body.days); + + var roles = req.query.include_roles; + if (typeof roles === "string") roles = [roles]; + + const { guild_id } = req.params; + const members = await inactiveMembers(guild_id, req.user_id, days, roles as string[]); + + await Promise.all(members.map((x) => Member.removeFromGuild(x.id, guild_id))); + + res.send({ purged: members.length }); +}); + +export default router; diff --git a/api/src/util/Instance.ts b/api/src/util/Instance.ts index d1d9e1ab..7dcd126e 100644 --- a/api/src/util/Instance.ts +++ b/api/src/util/Instance.ts @@ -10,9 +10,9 @@ export async function initInstance() { if (autoJoin.enabled && !autoJoin.guilds?.length) { let guild = await Guild.findOne({}); - if (!guild) guild = await Guild.createGuild({}); - - // @ts-ignore - await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } }); + if (guild) { + // @ts-ignore + await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } }); + } } } diff --git a/api/tests/routes.test.ts b/api/tests/routes.test.ts index ed391dfb..2c265ee3 100644 --- a/api/tests/routes.test.ts +++ b/api/tests/routes.test.ts @@ -43,7 +43,7 @@ const request = async (path: string, opts: any = {}): Promise => { var data = await response.text(); try { - data = JSON.stringify(data); + data = JSON.parse(data); if (response.status >= 400) throw data; return data; } catch (error) { diff --git a/bundle/package-lock.json b/bundle/package-lock.json index a967e97f..e6a263fa 100644 Binary files a/bundle/package-lock.json and b/bundle/package-lock.json differ diff --git a/bundle/package.json b/bundle/package.json index 404c6758..e9c2e0c3 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -47,7 +47,7 @@ "jest": "^27.0.6", "jest-expect-message": "^1.0.2", "jest-runtime": "^27.2.1", - "ts-node": "^9.1.1", + "ts-node": "^10.2.1", "ts-node-dev": "^1.1.6", "ts-patch": "^1.4.4", "typescript": "^4.2.3", @@ -93,6 +93,8 @@ "typescript": "^4.1.2", "typescript-json-schema": "^0.50.1", "ws": "^7.4.2", - "cheerio": "^1.0.0-rc.10" + "cheerio": "^1.0.0-rc.10", + "@aws-sdk/client-s3": "^3.36.1", + "@aws-sdk/node-http-handler": "^3.36.0" } -} +} \ No newline at end of file diff --git a/bundle/scripts/build.js b/bundle/scripts/build.js index dfbaec15..dbc305a9 100644 --- a/bundle/scripts/build.js +++ b/bundle/scripts/build.js @@ -8,12 +8,12 @@ const dirs = ["api", "util", "cdn", "gateway", "bundle"]; const verbose = argv.includes("verbose") || argv.includes("v"); -if(argv.includes("clean")){ - dirs.forEach(a=>{ - var d = "../"+a+"/dist"; - if(fse.existsSync(d)) { - fse.rmSync(d,{recursive: true}); - if(verbose) console.log(`Deleted ${d}!`); +if (argv.includes("clean")) { + dirs.forEach((a) => { + var d = "../" + a + "/dist"; + if (fse.existsSync(d)) { + fse.rmSync(d, { recursive: true }); + if (verbose) console.log(`Deleted ${d}!`); } }); } @@ -24,9 +24,9 @@ fse.copySync( path.join(__dirname, "..", "dist", "api", "client_test") ); fse.copySync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales")); -dirs.forEach(a=>{ - fse.copySync("../"+a+"/src", "dist/"+a+"/src"); - if(verbose) console.log(`Copied ${"../"+a+"/dist"} -> ${"dist/"+a+"/src"}!`); +dirs.forEach((a) => { + fse.copySync("../" + a + "/src", "dist/" + a + "/src"); + if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`); }); console.log("Copying src files done"); @@ -34,10 +34,11 @@ console.log("Compiling src files ..."); console.log( execSync( - "node \"" + + 'node "' + path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") + - "\" -p \"" + - path.join(__dirname, "..") + "\"", + '" -p "' + + path.join(__dirname, "..") + + '"', { cwd: path.join(__dirname, ".."), shell: true, diff --git a/cdn/package-lock.json b/cdn/package-lock.json index a5a81e4f..1e6a796c 100644 Binary files a/cdn/package-lock.json and b/cdn/package-lock.json differ diff --git a/cdn/package.json b/cdn/package.json index 027ba553..0d4d4619 100644 --- a/cdn/package.json +++ b/cdn/package.json @@ -36,6 +36,8 @@ "ts-patch": "^1.4.4" }, "dependencies": { + "@aws-sdk/client-s3": "^3.36.1", + "@aws-sdk/node-http-handler": "^3.36.0", "@fosscord/util": "file:../util", "body-parser": "^1.19.0", "btoa": "^1.2.1", diff --git a/cdn/src/util/S3Storage.ts b/cdn/src/util/S3Storage.ts new file mode 100644 index 00000000..df5bc19c --- /dev/null +++ b/cdn/src/util/S3Storage.ts @@ -0,0 +1,60 @@ +import { S3 } from "@aws-sdk/client-s3"; +import { Readable } from "stream"; +import { Storage } from "./Storage"; + +const readableToBuffer = (readable: Readable): Promise => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + readable.on('data', chunk => chunks.push(chunk)); + readable.on('error', reject); + readable.on('end', () => resolve(Buffer.concat(chunks))); + }); + +export class S3Storage implements Storage { + public constructor( + private client: S3, + private bucket: string, + private basePath?: string, + ) {} + + /** + * Always return a string, to ensure consistency. + */ + get bucketBasePath() { + return this.basePath ?? ''; + } + + async set(path: string, data: Buffer): Promise { + await this.client.putObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}`, + Body: data + }); + } + + async get(path: string): Promise { + try { + const s3Object = await this.client.getObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath ?? ''}${path}` + }); + + if (!s3Object.Body) return null; + + const body = s3Object.Body; + + return await readableToBuffer( body); + } catch(err) { + console.error(`[CDN] Unable to get S3 object at path ${path}.`); + console.error(err); + return null; + } + } + + async delete(path: string): Promise { + await this.client.deleteObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}` + }); + } +} diff --git a/cdn/src/util/Storage.ts b/cdn/src/util/Storage.ts index 91f841a6..3332f21c 100644 --- a/cdn/src/util/Storage.ts +++ b/cdn/src/util/Storage.ts @@ -2,6 +2,8 @@ import { FileStorage } from "./FileStorage"; import path from "path"; import fse from "fs-extra"; import { bgCyan, black } from "nanocolors"; +import { S3 } from '@aws-sdk/client-s3'; +import { S3Storage } from "./S3Storage"; process.cwd(); export interface Storage { @@ -10,10 +12,10 @@ export interface Storage { delete(path: string): Promise; } -var storage: Storage; +let storage: Storage; if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { - var location = process.env.STORAGE_LOCATION; + let location = process.env.STORAGE_LOCATION; if (location) { location = path.resolve(location); } else { @@ -24,6 +26,32 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { process.env.STORAGE_LOCATION = location; storage = new FileStorage(); +} else if (process.env.STORAGE_PROVIDER === "s3") { + const + region = process.env.STORAGE_REGION, + bucket = process.env.STORAGE_BUCKET; + + if (!region) { + console.error(`[CDN] You must provide a region when using the S3 storage provider.`); + process.exit(1); + } + + if (!bucket) { + console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`); + process.exit(1); + } + + // in the S3 provider, this should be the root path in the bucket + let location = process.env.STORAGE_LOCATION; + + if (!location) { + console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`); + location = undefined; + } + + const client = new S3({ region }); + + storage = new S3Storage(client, bucket, location); } export { storage }; diff --git a/util/package-lock.json b/util/package-lock.json index 5f136dbc..5f7c8deb 100644 Binary files a/util/package-lock.json and b/util/package-lock.json differ diff --git a/util/package.json b/util/package.json index e1003114..2527c005 100644 --- a/util/package.json +++ b/util/package.json @@ -40,7 +40,7 @@ "dependencies": { "amqplib": "^0.8.0", "jsonwebtoken": "^8.5.1", - "lambert-server": "^1.2.11", + "lambert-server": "^1.2.12", "missing-native-js-functions": "^1.2.17", "multer": "^1.4.3", "nanocolors": "^0.2.12", diff --git a/util/src/entities/Member.ts b/util/src/entities/Member.ts index 7d7ac40a..12b0b49a 100644 --- a/util/src/entities/Member.ts +++ b/util/src/entities/Member.ts @@ -84,6 +84,9 @@ export class Member extends BaseClassWithoutId { @Column({ type: "simple-json" }) settings: UserGuildSettings; + @Column({ nullable: true }) + last_message_id?: string; + // TODO: update // @Column({ type: "simple-json" }) // read_state: ReadState; diff --git a/util/src/util/Snowflake.ts b/util/src/util/Snowflake.ts index f7a13388..3f6e3c63 100644 --- a/util/src/util/Snowflake.ts +++ b/util/src/util/Snowflake.ts @@ -84,7 +84,7 @@ export class Snowflake { } static generate() { - var time = BigInt(Date.now() - Snowflake.EPOCH) << 22n; + var time = BigInt(Date.now() - Snowflake.EPOCH) << BigInt(22); var worker = Snowflake.workerId << 17n; var process = Snowflake.processId << 12n; var increment = Snowflake.INCREMENT++;