diff --git a/bundle/package-lock.json b/bundle/package-lock.json index a967e97f..02c94bfc 100644 Binary files a/bundle/package-lock.json and b/bundle/package-lock.json differ diff --git a/bundle/package.json b/bundle/package.json index 404c6758..5a7c116c 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -47,7 +47,7 @@ "jest": "^27.0.6", "jest-expect-message": "^1.0.2", "jest-runtime": "^27.2.1", - "ts-node": "^9.1.1", + "ts-node": "^10.2.1", "ts-node-dev": "^1.1.6", "ts-patch": "^1.4.4", "typescript": "^4.2.3", @@ -93,6 +93,8 @@ "typescript": "^4.1.2", "typescript-json-schema": "^0.50.1", "ws": "^7.4.2", - "cheerio": "^1.0.0-rc.10" + "cheerio": "^1.0.0-rc.10", + "@aws-sdk/client-s3": "^3.36.1", + "@aws-sdk/node-http-handler": "^3.36.0" } } diff --git a/cdn/package-lock.json b/cdn/package-lock.json index a5a81e4f..9fff1e72 100644 Binary files a/cdn/package-lock.json and b/cdn/package-lock.json differ diff --git a/cdn/package.json b/cdn/package.json index 027ba553..0d4d4619 100644 --- a/cdn/package.json +++ b/cdn/package.json @@ -36,6 +36,8 @@ "ts-patch": "^1.4.4" }, "dependencies": { + "@aws-sdk/client-s3": "^3.36.1", + "@aws-sdk/node-http-handler": "^3.36.0", "@fosscord/util": "file:../util", "body-parser": "^1.19.0", "btoa": "^1.2.1", diff --git a/cdn/src/util/S3Storage.ts b/cdn/src/util/S3Storage.ts new file mode 100644 index 00000000..df5bc19c --- /dev/null +++ b/cdn/src/util/S3Storage.ts @@ -0,0 +1,60 @@ +import { S3 } from "@aws-sdk/client-s3"; +import { Readable } from "stream"; +import { Storage } from "./Storage"; + +const readableToBuffer = (readable: Readable): Promise => + new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + readable.on('data', chunk => chunks.push(chunk)); + readable.on('error', reject); + readable.on('end', () => resolve(Buffer.concat(chunks))); + }); + +export class S3Storage implements Storage { + public constructor( + private client: S3, + private bucket: string, + private basePath?: string, + ) {} + + /** + * Always return a string, to ensure consistency. + */ + get bucketBasePath() { + return this.basePath ?? ''; + } + + async set(path: string, data: Buffer): Promise { + await this.client.putObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}`, + Body: data + }); + } + + async get(path: string): Promise { + try { + const s3Object = await this.client.getObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath ?? ''}${path}` + }); + + if (!s3Object.Body) return null; + + const body = s3Object.Body; + + return await readableToBuffer( body); + } catch(err) { + console.error(`[CDN] Unable to get S3 object at path ${path}.`); + console.error(err); + return null; + } + } + + async delete(path: string): Promise { + await this.client.deleteObject({ + Bucket: this.bucket, + Key: `${this.bucketBasePath}${path}` + }); + } +} diff --git a/cdn/src/util/Storage.ts b/cdn/src/util/Storage.ts index 91f841a6..3332f21c 100644 --- a/cdn/src/util/Storage.ts +++ b/cdn/src/util/Storage.ts @@ -2,6 +2,8 @@ import { FileStorage } from "./FileStorage"; import path from "path"; import fse from "fs-extra"; import { bgCyan, black } from "nanocolors"; +import { S3 } from '@aws-sdk/client-s3'; +import { S3Storage } from "./S3Storage"; process.cwd(); export interface Storage { @@ -10,10 +12,10 @@ export interface Storage { delete(path: string): Promise; } -var storage: Storage; +let storage: Storage; if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { - var location = process.env.STORAGE_LOCATION; + let location = process.env.STORAGE_LOCATION; if (location) { location = path.resolve(location); } else { @@ -24,6 +26,32 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) { process.env.STORAGE_LOCATION = location; storage = new FileStorage(); +} else if (process.env.STORAGE_PROVIDER === "s3") { + const + region = process.env.STORAGE_REGION, + bucket = process.env.STORAGE_BUCKET; + + if (!region) { + console.error(`[CDN] You must provide a region when using the S3 storage provider.`); + process.exit(1); + } + + if (!bucket) { + console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`); + process.exit(1); + } + + // in the S3 provider, this should be the root path in the bucket + let location = process.env.STORAGE_LOCATION; + + if (!location) { + console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`); + location = undefined; + } + + const client = new S3({ region }); + + storage = new S3Storage(client, bucket, location); } export { storage };