From 37594a5bec8574e1c4b5d5dcb2d45b055ee3c58c Mon Sep 17 00:00:00 2001
From: Rory& <root@rory.gay>
Date: Fri, 17 Oct 2025 11:58:53 +0200
Subject: [PATCH] ZSTD on gateway

---
 .idea/runConfigurations/Start_Gateway.xml |   2 +-
 .idea/workspace.xml                       |   3 ++-
 package-lock.json                         | Bin 342355 -> 346348 bytes
 package.json                              |   7 +++---
 src/gateway/events/Connection.ts          |  12 ++++------
 src/gateway/events/Message.ts             |  10 ++++++--
 src/gateway/util/Send.ts                  |  28 ++++++++++++----------
 src/gateway/util/WebSocket.ts             |   7 +++++-
 8 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/.idea/runConfigurations/Start_Gateway.xml b/.idea/runConfigurations/Start_Gateway.xml
index 884158bb..e3590fa8 100644
--- a/.idea/runConfigurations/Start_Gateway.xml
+++ b/.idea/runConfigurations/Start_Gateway.xml
@@ -5,7 +5,7 @@
     <scripts>
       <script value="start:gateway" />
     </scripts>
-    <node-interpreter value="/nix/store/dcdc33kdjdhjnzg6rkmd0cx4kpwl8cac-nodejs-20.17.0/bin/node" />
+    <node-interpreter value="/run/current-system/sw/bin/node" />
     <package-manager value="npm" />
     <envs />
     <EXTENSION ID="com.fapiko.jetbrains.plugins.better_direnv.runconfigs.NodeRunConfiguration">
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 8bf9617e..33585057 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -111,6 +111,7 @@
           <value>
             <map>
               <entry key="Start API" value="STOPPED" />
+              <entry key="Start Gateway" value="STOPPED" />
               <entry key="build" value="STOPPED" />
             </map>
           </value>
@@ -118,7 +119,7 @@
       </map>
     </option>
   </component>
-  <component name="RunManager" selected="npm.Start API">
+  <component name="RunManager" selected="npm.Start Gateway">
     <list>
       <item itemvalue="Compound.Start separated" />
       <item itemvalue="npm.Start bundle" />
diff --git a/package-lock.json b/package-lock.json
index bd4bc748692ae8ce55f4e22f985469d94632679a..f166d7eadc63ba8176c7a6aa48f80c0b4e636212 100644
GIT binary patch
delta 3362
zcmb7Gdu$W;9oOCc3?v~yAtZs20O2(-$(`ReRe_xC*gikv_p1WM_W5ysoZqo=C4z*e
z{?V2JC-kd8-BL{})Bb3osJpGxSbzzpj;ft1tzEnFsMNpK720iTt4VCWGl66^kogbm
zclUjLKHtyx`_p~!b<Y<k)}Gsj^z5(X^DG@NM5;BM&C~IbM!5oA2)5?JHpGQ-#m*kM
zLjwl6lVFnjVdrZ2Ss&5|Z+;GzUg(faN;vX`P29d!OOIac<S%w2zN4$Ym+Sm<z`VI#
z6&fFTP?3hSC9usm<u^<@edWryE@?FerPg9Wtx`s9Ql-yPEvpNeTu7VC#wb_8Qo}v0
zx;&vvWMxictxgg&QDx04XU?A>eM~GN%T5HV8s&toq=#d_#h!*MwqncRYx~e`(`^XT
zBXp!xW7v^MDW_Hr@7Xjg8<md|!@D~>n!n+;=;uP}iw;PX;_+yCnjTqvNS0x%^%km;
zBwJl1viVWDYB{?gZ2HlRT5(Nu|934R=6jXe7g|@Y&?C)WE962SMsnq;*nNE5iM1Oy
zK0zJs*6Cj8JPH&~KUNe`$NBR$r<Q0Chs~d$NmJJCDJ5h&yG&}KblM1|w^`M=+3Iq$
z?rh4aV${a4CgV>S<Rx`VMHY3AqIoRmQrVNWN-ja_{PrAUDY?e%mb%p+_o|eIBzOBv
zFPA;#ftO0?{<UpEa(7N`6gg<xn&+T4yVH(A^K}b<*k4sPGA>_I@6tA8SsSgvZQ+I`
zWU$(l+KMw-wK%jkrN^DHRRe*nw<yPz6JGULE|zLk4AHo%oVV!>)tXvaQ3X}`ieV~Q
zHc?|u{Le0LZNsoI{Rc!MgYA*FCYt7MobDG1=^pDQ=)Hi^{cz!Zi&Dm5m1Y%HrzTZl
z0#kN8lMN~4WyWZ-kQs$lWr!$oLoDc*7jb_<>P%_X<AH+9-Owx5HOf+;NHtM6`qY|2
zecWUt&A1kKlZ0H5<k{}~wdQf!)*kHq<!P=Qe6<@hjV;P`0k{86BiwI(YbEXrL~<H?
zY>dd|9gI6!)mS}AXH_w7lv;z;lwu-U4CG^Z6=hW^JO!mQS;C7lOQ=jc)ZS1inxyO*
zs*q2p<qk{q5TmZJ{>0qKnYdR@cZq6ke==NQSw(8(;(Q;;t(ge^t^~IGt?8&~A~@yD
z<x8o6$)L;_{b`zP;7Ka!o2bc*RifTt$XdBbD17dcg$Q~KV<B~jp~8i5#l(8jIZZIn
z8Vf8Lj`?drv&<ZBC|vi~=^?i*kHgy>B;o23i9NFj$2P3yQiysL9h4z#qVaSgYsF=h
zRO{Dj=n!3+V#ex@ij1jQaF2_i+zPX|p`z$$RZ6mcQkS=9vicYvH3cHthBw26Gg;hN
z8&f1J(L!D-a7u_eZG95N2P*AdjOQcR%qis6*JMT39;D;iScIt<?9mJ{6(|*1zuD`t
zN+}XI6cj-=oJ_gWWXYhRY-xGX(MS<-Z>H`Tk4RZ29nEXahYppq0aLlAuO>rb?&g`4
z$mJ2bEV$_5rMH_O59wcEvVHCOEbxvk;Iq)wx-K*^gB*8j47RY@UN?KoE=Oe|sF<Sk
zUS*NiC?g6skPd|^DN3tL7K62#F|XH>wMxZRpEA=#Q6E-#@K_ipLwPTg3EFFRM^0}P
zuM6&zS6ZvYuRMlmSAjx2UV^TFAY0(gGsse&{xNc5^~0BZ*Lmcr-iObE(-L_1Hqs4`
zU2pykT}F1oTkjwe{>U{XxwYlE`53mI|L5P362YDOLoa_~13HAj*{x_FJhD?F;cpG1
zZ+8mM{jfg<cJN>R0Nu42{)CX|_ymQ%F(d}<XVB(1+U%79cxN5h%zu_eXFECK*8}kV
z=fNO<r-m-+fou9CJrZJ6HY$VXKSJ?!aW<Bx88+GM?%Z*tWk&WOn|wwu8L8%LGOt1v
z!UJJ@hRN&FjGYLp>S?n%M=683ba<jD%i?sfp~#lxWjCwUu+$+r;jR{no|J4{o>Q33
zPESGWNm-mOszfQcKYc^NUHy_3=s|+}vU?N%_ZQHQQGo({>ngeuZXZGi;7c2@4ZQY8
z=&Gk4c9K6hgQ~Z{W$~u{;xp(VfBOph+Q7+Q_404tM340ehs-x9K;X<@kzV-H47LW|
zJA)bcuf9dGm4bESGR(9G-uVt~c9_I2Of;Ld0UlvchQDzV`wiBLJ3E7E#g@U>U&cD%
zuESU-zZ+ue4zbD2te>Jnu2Vn7K3XjtUj8L^wNpg@aUJV|)60=1{OL>B*Y0`1`5y2&
z)^vu~`oXt?ZUcN%qX7ioNTcg{-2m8u2-X)ifPcf|I;@X>a}d0Phy~AX1dc9I)$|Y;
zg7-$iGTtf$01@8t)4RcL@dTSEDZgYCbhpCx;vgfQAoCv91Fv@>9cRBH0C+fVa7F_p
zJf;GpYhmF6=;IGiV9zSfZ0hANhryp8WF3aHlb{10kAd}k_W|%m2X}XJNh{6yerymP
zX24SZFA1=BROkkI3ak*d<e!6p=xlNQ<W+!U@O{47ch%28MC{+Cv*3@tt&YDw3%cOD
zGISUJ#YF%Sc-MgT^XF#4G}6*4d>J6EZQ!530zN>*q%DaYzXz+~y%}sBJT656Jgh=L
zgp==L-TZr30W)t?dB=y~x`dk<>V*f-ftBz-32Zq(`6>AMz`TZBy9Zu`JNuCy{;lu8
OT5YTNzhshwOaBYD7^aH=

delta 942
zcmYjQSxA&o6z1N0es#ueG-nKFoOCozN5>Jf#5A)AiNq}}sSpcVL?v6GPzy>5B+_iE
zQ$1v5W<`=Ha(WP)qE)5LF=?T&2Z<uVB(O!P`Tt*<?$fzv`R;el`R>;iy=yw8YccD|
zWrf1g4fPJ=dcU3xQl*rbpw+mKrDm5UJ;G9!W=mdXOR>y5_wTK$IItr{iKN<+ZOa|s
ziXt5mG2RUmTAI+Zk_Nie%9iS|ppS*q>|NFp?C2UWpvGSc>Ui{k8RUQo&7YY+j=yI*
z3Vg%PMB~&4;ZOZ@Y@Z#c7`IZ6jhmTj)+KSLRx!_INpV!0&f_CdcU$Oj%EV0M+`@;<
zYUZ_CUMR5q7TB>rLyDxNBXZ@J8K$Hfar~wTr`SgB6+*`^^AWCARbS;BgJf2rSnpy%
zG~3Oiqp);@+p%gZgj4AlzhQLfUg~k&!VMHM&6`3!*LxZPOr~>;;(Ub&^h8#NiJWw_
zTZENbi$zbqO02*oe6T!KO2+Xyk&L;MEF8;C0NCWkl5iL#17)_05+OrcvH7e>BRB1x
z8igyq=oAZRPq!H0o_F$l#p__tHS?)>td0xco&+sLxx~*RwbL~*Fd>x8E`JEctazAF
z2kHFH%~WiKh04hF82g8X(TN0zWuEMZiSSE>8MVRS{BC;fkmsv3M&=BNXKIgG)nLJ)
zm7t-fOkn>x$?Y`Em;T6tO2tg1x@<6lA_4SW2Y%#S3$gB%$vT+Z3@dbwG?$g~Dq&28
zA~*hb4HjK@i>dPv_|W!)kZDtx*0i|$YHkBB>T3sYWx<K`t^;h^e~K|;%^d(*(hXLD
zr(&c)$LU!URrJC(b*3@|eI#&eR$c+!e-1<Ql-q{kC{s6@J|Z`!P%L!8sG<N>Wwh!Q
zL@6OXW?T^gxXHju>Dd_Qls^$l>=O)1vMQ~3*a?AHeu4X)e*%lhGzpDH+#MiYq<oDO
Nm!`hdlr5Ec{{^xNN@@TA

diff --git a/package.json b/package.json
index 008d5002..f9a4a584 100644
--- a/package.json
+++ b/package.json
@@ -72,6 +72,7 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.911.0",
+		"@toondepauw/node-zstd": "^1.2.0",
 		"ajv": "^8.17.1",
 		"ajv-formats": "^3.0.1",
 		"amqplib": "^0.10.9",
@@ -102,10 +103,12 @@
 		"murmurhash-js": "^1.0.0",
 		"node-2fa": "^2.0.3",
 		"node-fetch-commonjs": "^3.3.2",
+		"pg": "^8.16.3",
 		"picocolors": "^1.1.1",
 		"probe-image-size": "^7.2.3",
 		"proxy-agent": "^6.5.0",
 		"reflect-metadata": "^0.2.2",
+		"sqlite3": "^5.1.7",
 		"tslib": "^2.8.1",
 		"typeorm": "^0.3.27",
 		"typescript-json-schema": "^0.65.1",
@@ -127,8 +130,6 @@
 		"jimp": "^1.6.0",
 		"mailgun.js": "^12.1.1",
 		"node-mailjet": "^6.0.9",
-		"nodemailer": "^7.0.9",
-		"pg": "^8.16.3",
-		"sqlite3": "^5.1.7"
+		"nodemailer": "^7.0.9"
 	}
 }
diff --git a/src/gateway/events/Connection.ts b/src/gateway/events/Connection.ts
index b187019e..bc74dee6 100644
--- a/src/gateway/events/Connection.ts
+++ b/src/gateway/events/Connection.ts
@@ -28,7 +28,8 @@ import { Message } from "./Message";
 import { Deflate, Inflate } from "fast-zlib";
 import { URL } from "url";
 import { Config, ErlpackType } from "@spacebar/util";
-// import zlib from "node:zlib";
+import zlib from "node:zlib";
+import { Decoder, Encoder } from "@toondepauw/node-zstd";
 
 let erlpack: ErlpackType | null = null;
 try {
@@ -108,7 +109,7 @@ export async function Connection(
 			return socket.close(CLOSECODES.Decode_error);
 
 		if (socket.encoding === "etf" && !erlpack)
-			throw new Error("Erlpack is not installed: 'npm i erlpack'");
+			throw new Error("Erlpack is not installed: 'npm i @yukikaze-bot/erlpack'");
 
 		socket.version = Number(searchParams.get("version")) || 8;
 		if (socket.version != 8)
@@ -121,11 +122,8 @@ export async function Connection(
 				socket.deflate = new Deflate();
 				socket.inflate = new Inflate();
 			} else if (socket.compress === "zstd-stream") {
-				// TODO
-				return socket.close(
-					CLOSECODES.Decode_error
-				);
-
+				socket.zstdEncoder = new Encoder(6);
+				socket.zstdDecoder = new Decoder();
 			} else {
 				return socket.close(CLOSECODES.Decode_error);
 			}
diff --git a/src/gateway/events/Message.ts b/src/gateway/events/Message.ts
index 6fb222b6..2987f464 100644
--- a/src/gateway/events/Message.ts
+++ b/src/gateway/events/Message.ts
@@ -45,9 +45,15 @@ export async function Message(this: WebSocket, buffer: WS.Data) {
 	) {
 		data = bigIntJson.parse(buffer.toString());
 	} else if (this.encoding === "json" && buffer instanceof Buffer) {
-		if (this.inflate) {
+		if (this.compress === "zlib-stream") {
 			try {
-				buffer = this.inflate.process(buffer);
+				buffer = this.inflate!.process(buffer);
+			} catch {
+				buffer = buffer.toString();
+			}
+		} else if (this.compress === "zstd-stream") {
+			try {
+				buffer = await this.zstdDecoder!.decode(buffer);
 			} catch {
 				buffer = buffer.toString();
 			}
diff --git a/src/gateway/util/Send.ts b/src/gateway/util/Send.ts
index 9b1eef6a..cee3aa8c 100644
--- a/src/gateway/util/Send.ts
+++ b/src/gateway/util/Send.ts
@@ -43,22 +43,20 @@ const recurseJsonReplace = (json: any) => {
 	return json;
 };
 
-export function Send(socket: WebSocket, data: Payload) {
+export async function Send(socket: WebSocket, data: Payload) {
 	if (process.env.WS_VERBOSE)
 		console.log(`[Websocket] Outgoing message: ${JSON.stringify(data)}`);
 
 	if (process.env.WS_DUMP) {
 		const id = socket.session_id || "unknown";
 
-		(async () => {
-			await fs.mkdir(path.join("dump", id), {
-				recursive: true,
-			});
-			await fs.writeFile(
-				path.join("dump", id, `${Date.now()}.out.json`),
-				JSON.stringify(data, null, 2),
-			);
-		})();
+		await fs.mkdir(path.join("dump", id), {
+			recursive: true,
+		});
+		await fs.writeFile(
+			path.join("dump", id, `${Date.now()}.out.json`),
+			JSON.stringify(data, null, 2),
+		);
 	}
 
 	let buffer: Buffer | string;
@@ -71,9 +69,15 @@ export function Send(socket: WebSocket, data: Payload) {
 	else if (socket.encoding === "json")
 		buffer = JSON.stringify(data, JSONReplacer);
 	else return;
+
 	// TODO: compression
-	if (socket.deflate) {
-		buffer = socket.deflate.process(buffer) as Buffer;
+	if (socket.compress === "zlib-stream") {
+		buffer = socket.deflate!.process(buffer) as Buffer;
+	} else if (socket.compress === "zstd-stream") {
+		if (typeof(buffer) === "string")
+			buffer = Buffer.from(buffer as string);
+
+		buffer = await socket.zstdEncoder!.encode(buffer as Buffer) as Buffer;
 	}
 
 	return new Promise((res, rej) => {
diff --git a/src/gateway/util/WebSocket.ts b/src/gateway/util/WebSocket.ts
index 2f8fa7cf..09bae2d0 100644
--- a/src/gateway/util/WebSocket.ts
+++ b/src/gateway/util/WebSocket.ts
@@ -20,19 +20,24 @@ import { Intents, ListenEventOpts, Permissions } from "@spacebar/util";
 import WS from "ws";
 import { Deflate, Inflate } from "fast-zlib";
 import { Capabilities } from "./Capabilities";
+import { ZstdCompress } from "zlib";
+import { ZstdDecompress } from "node:zlib";
+import { Decoder, Encoder } from "@toondepauw/node-zstd";
 
 export interface WebSocket extends WS {
 	version: number;
 	user_id: string;
 	session_id: string;
 	encoding: "etf" | "json";
-	compress?: "zlib-stream";
+	compress?: "zlib-stream" | "zstd-stream";
 	ipAddress?: string;
 	userAgent?: string; // for cdn request signing
 	shard_count?: bigint;
 	shard_id?: bigint;
 	deflate?: Deflate;
 	inflate?: Inflate;
+	zstdEncoder?: Encoder;
+	zstdDecoder?: Decoder;
 	heartbeatTimeout: NodeJS.Timeout;
 	readyTimeout: NodeJS.Timeout;
 	intents: Intents;