From 1569fec0cf23442b176b2a9a775208349aa719ed Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 02:37:38 +1100 Subject: [PATCH 01/10] boilerplate stuff --- bundle/src/Server.ts | 28 ++++++++++----------- util/src/util/Constants.ts | 3 +++ webrtc/.vscode/launch.json | 23 +++++++++++++++++ webrtc/package-lock.json | Bin 25101 -> 31537 bytes webrtc/package.json | 4 ++- webrtc/src/Server.ts | 36 +++++++++++---------------- webrtc/src/opcodes/Connect.ts | 5 ++++ webrtc/src/opcodes/Heartbeat.ts | 7 ++++++ webrtc/src/opcodes/Identify.ts | 20 +++++++++++++++ webrtc/src/opcodes/Resume.ts | 5 ++++ webrtc/src/opcodes/SelectProtocol.ts | 16 ++++++++++++ webrtc/src/opcodes/Speaking.ts | 6 +++++ webrtc/src/opcodes/index.ts | 35 ++++++++++++++++++++++++++ webrtc/src/start.ts | 1 + webrtc/src/util/Heartbeat.ts | 18 ++++++++++++++ webrtc/src/util/index.ts | 1 + webrtc/tsconfig.json | 19 +++++++++++--- 17 files changed, 187 insertions(+), 40 deletions(-) create mode 100644 webrtc/.vscode/launch.json create mode 100644 webrtc/src/opcodes/Connect.ts create mode 100644 webrtc/src/opcodes/Heartbeat.ts create mode 100644 webrtc/src/opcodes/Identify.ts create mode 100644 webrtc/src/opcodes/Resume.ts create mode 100644 webrtc/src/opcodes/SelectProtocol.ts create mode 100644 webrtc/src/opcodes/Speaking.ts create mode 100644 webrtc/src/opcodes/index.ts create mode 100644 webrtc/src/util/Heartbeat.ts create mode 100644 webrtc/src/util/index.ts diff --git a/bundle/src/Server.ts b/bundle/src/Server.ts index 71a60d49..bc1d7cbc 100644 --- a/bundle/src/Server.ts +++ b/bundle/src/Server.ts @@ -50,20 +50,20 @@ async function main() { endpointPublic: `ws://localhost:${port}`, }), }, - // regions: { - // default: "fosscord", - // useDefaultAsOptimal: true, - // available: [ - // { - // id: "fosscord", - // name: "Fosscord", - // endpoint: "127.0.0.1:3001", - // vip: false, - // custom: false, - // deprecated: false, - // }, - // ], - // }, + regions: { + default: "fosscord", + useDefaultAsOptimal: true, + available: [ + { + id: "fosscord", + name: "Fosscord", + endpoint: "127.0.0.1:3004", + vip: false, + custom: false, + deprecated: false, + }, + ], + }, } as any); //Sentry diff --git a/util/src/util/Constants.ts b/util/src/util/Constants.ts index 5fdf5bc0..a1892105 100644 --- a/util/src/util/Constants.ts +++ b/util/src/util/Constants.ts @@ -73,7 +73,10 @@ export const VoiceOPCodes = { HEARTBEAT: 3, SESSION_DESCRIPTION: 4, SPEAKING: 5, + HEARTBEAT_ACK: 6, + RESUME: 7, HELLO: 8, + RESUMED: 9, CLIENT_CONNECT: 12, CLIENT_DISCONNECT: 13, }; diff --git a/webrtc/.vscode/launch.json b/webrtc/.vscode/launch.json new file mode 100644 index 00000000..92403164 --- /dev/null +++ b/webrtc/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "ts-node", + "type": "node", + "request": "launch", + "args": [ + "src/start.ts" + ], + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "cwd": "${workspaceRoot}", + "protocol": "inspector", + "internalConsoleOptions": "openOnSessionStart" + } + ] +} \ No newline at end of file diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index a5db2de13632936e54cc30ed1e8df396443d47a0..6c3726dc425b0a2cc59c41eb82b859538beb60c6 100644 GIT binary patch literal 31537 zcmeHQ*OKa5dcNN0DNgm}U>YQpaQTdfBt#G(M0R<*6hT4>CBk^-*}awmh(ulT*U()_T4~%ryN1r3hU_9pMUxG?G3yI|K3~&oXNWqIBjT>VtLU$b@&Fn zJUA&Uj$+A*C242U;H;_0n&8=i1Ml{G<-Pij$Daq6vt^~|2X3^G?_eG8MEaKRNw%eG z>!Kt08+h;Vs|Xq|6<%x~pCsQg@FjTpzoa)2HgoUePyW$Syb`=-L=M57^V<#F5N1|_^rtA|2gZ17&ganSY7-RYCry3`|v)KRi$6};Qo*G{$d zZ@(p%l$8)%EMZXqu41h%7=4^>lsr6W)pW%0>g=X=7vp@1eafb9FL>j}6qLPzB) zIY8$!;_-{Je?S7vHdusE`-6?oteo*ikyl-3GwANjRUW6`Q4`lUkKV(i`Zr;MrkJ1+ zCe)Qp|!Xh8up|VhSFDExQ^*c@EjF>Q1m(){nL31a*&&D(LKLHNfT^ajLi1 z^`3)Q20e4vsSta5*%T{BfgJ^IXR^=X%uiv0-NPhN=$}@W*HB|ACRl_C@&psSTy@dH zu{NevhG~%#ViW|Zu5az5r22NaS&Vn{3QCQdi^;6r(>8{PjHp%BWLHcXQ^cd7tK%gp zE3(=?T82wA-6_}j-F!ma!vy`C_=HF?AtFrBCzvR!gCTaP_R5olHcxK1QptCj$pk;F zU7e|v5DV=#_kq`_YQkpJo3rC^tatg^05=_pmN`EKIEiMOaipQn25k zn~>)rTsbK>hAQi`e4fS6eowNK#hA>4dxz$pY;|0r-a3w%mO?g|0&OCTaJ`!n!G1NM zI8t|wS%t&J><7QELV4qo2K$RWGI z0eXFjfA%5_dk=uH_lgmPCkV(Ta*kHIwZ+Jmxq&!b4Hlt+_6p*z#<>l4Z){3dzzr)_(T+r}=u50|LHupwINx3_fV zZ{Tcv%h{OVfp-%7^FSEC{Uu)_#vJ)K;pVRM0K?vq_sWZ$2!a~Ghwi!coj^IX-@r@Kh*AsZSF}9y{WWll{dNBgg%h)w}#ws>GwvU8A zXk&c8L=U%(+JG3JbWPZmUBfdLp*_j^CV9%rb6gNPszp5t&i@J*{4i1z1y3Qc_~SWn;c0ySS=l<%18|*R z1H1sCCu~#eIl>b79h8spGjYf?fFi;Rz1}(;W1;IcsZb1EQRBLfN27 zvODeSjq>&+4wO-Ue>&_a?daGbSXSm2wowC1)+{Iy{{eN)w^(h(83b_hZLRs6K#W-f z#J+)Rz_V$TAHVx9*=7kO5G>Djy^^Z9w&z7!E=hgd5Cy3dR{>d~h~aLj?)UrmYZqU9Q3k)6;xW>CKF~fK5sE zxG1BHJk!fPvQe1}^@<*l^u==hbMvFkkM&-D%Hb@mknO(%6OnHFw>M2u6cUF4AAAG+ zR35oNk}KsgQswnL8lVKY2>dQaUw*fCd7M_1r5?Sg=vJ9^I>JKNJbN4}hfa6a z>Yaogc|zS)@1Pb~-mOlyvdNg<6f;pn>f5@jmuSYI%q>1Uu;vn2Fn_!ql`3{Gi(Hyk z%vU}#L`Ah-Q^-vXR)7h6(PU>R0z@)LfRSL-_!ti5IFX#*P!ddeGw&{RoM<{6Ya>U4 z=Nzf+w|fg|TAqkive3hcQiH|?v$osq270;B3u|Rq<}#2_-L{AkCcXL2)-Yc ztZv+U8125uoC!66dGe|l5qZL*F1~Lc$fYve59R}-Q7i8`ia7O-%|6Syh0?fA9gZZ% zVw;JxQBB>@^a;-|+5tnkR@hrv>ZGa_Nc?2VlrHsTc8m=P9byjf$w}d@!(7~Hx>nAWg6?G3&0l=jwLnQMgG-bOl6zOicKOr2B5 z%5rtYh~d63Eek76D4V{_ctg8-ENw=G#(^D_C(|l-u!nvddlvt#T|rXRz$mJ5br!p% zN4~wmyeX_tLuUI9b<4Sq8dw^vc>-osqk;KK^Dd< zq}*T;vm#DvGrHB3WV{yCFoBrk%WZpwXoCfFTAio~F{$_ZND6T)9C(ob9+a z+<>B?fHGlGgO%hPARuSGs0wpw14weu$JvV~;4~{^!z$^$f!=G*@zta|=~XO^w}K&!W=>N_h(l%5I~5A2S*VQ%V{;MUk~1hgv-YWj z1DO-JtuZq_Wj|1#Pi=la@|XA}2W#M4yk?&hY5?8j6~GfZD36taEFE{7X1G8a7-1|{ z(rnjodIj2N$J|WrumhshRB_!yPj)c%c5Sjlc=SQ@JBH2rL)2gRQ*t}L>e2_~p4WlpCvPmS%!ppag5+7-bdM2T0@M zDCpu6OOl~|0?fp2-*9lojDOZ_rfxt2X=IQhG7mTyDQ+t59mbgAswTU#$&HTd`N*m_ z^^xbL4Mb_wlnuC2sr&YpvCk1iP-K$Ev3wpf>@^nUv2PYlfJo!ToIp$rEIEErj7U6K zFf72b5~U9Up6-|&sF?Xmo!-ksPBI$=hnvE*Je!4b1053!RwRhh%$gL)Igd3<<(4vF zR{^pdHqEYr4>}gQQ2oZKWlPibGd9YkDDq=7C4J#{P*BayD1Zk6Hf)EBKAFF1rm$6titAp&>1>z> z^NKGzc^86U^DYNK4**h6HqfJ{8)Kl5;izYu+@^u-XvJXdty|)yN^Lz5J-cF&m^mBO z@cv}A-`1-GW1(V;kzn=@-4?YeuQz1HWvkdB+>a+{W7e^<8~8Z0m2JGd;o(dl=4S=> zOb;TFHgy8)0~$J?06`fY-WuSN=<}hxW0c@{*fb{egb8GOR|3g`zSOLj<+3xu$8Cf% zR|Z4l^?j$#Z`+4@$(kRPi8^VIoOWFs_&qxi#qr|eeryDec$FCaKMl%@|{D(A)Z=Ct6Oe(UyBZbg+Pr9K$Erkg=D1^z&CQS^Se}_tqP;*VcVQb>&~JNmC>D5!vw(1 zdgD}T%%Y!6NSd%=$Mis^2ZHrD%<LP?>;fh28b;UYR-bOHRv3>hc1nA= zXoLjiPYSwN3Ad&3QRvM_<focm^jL1 zwZT^PKIjoq>I}ZeaY8guk*2pNMr(BG&Z0cTtkY`zYSOKop|&D7Rlk|&LJ;2R(? zIjhvOKlS$4*nxUr#@!DJG#A|$6~+}!fbi9Egz_`Rp*ii!cH9`Zw9=AuG_;0uZ7FP? zke*`N8#|nv46$pgqCGnv+wvIoOXP_=l|%aAss~48xbbUz^}xMc7r@?4%J4`(P?j(noUp+=IWxu6R?yV#<`zF9a7ItnMVY)~gP zNhifEanewlYPV|#Z7o?%+UYXl;5d-h$8J*_6)0=yE^%qqcJHT5ui8d()(cYmuLa2$ zf?(B`gD84?RB1wr^`XFv=+ZVOGq72;S*N)?1+?U-+WwxZ`{2*s+<)xezNGZBB;g7& z9y3scz*~Nz2#0O5x-%=aK(DdYj%n}K^VSkMv@vE;=K-(>6tK|8^gi1^?H8aq$l%1GJQ{MdaY#0w=FhGwsnD&5N!%nx$U0Ooxt5$xQ`(Ua z6=%mIwGgBA`{RHtV%N^db%2wzS?8PntVPaH1bdY^ifq<1Q4^P3(C#brVY599hI9pX zWofHJ*r?ZbLHo2a340Ej23F-YI$@?KEfmI^kP3zy*+oG8%(g{-*ta{igEH-*V_9UW z4n4W3?gtP;-TXiY0|+&zDB5P8AMjDaKxZP**Psjn_lUK!< z$`i=fW!e~pVPnB}hDf(b6Fgc$N4P#(?&{ldzq&Jy6tY^^hWwOt_normJDbK@Mz?*f z;ZCbfool#VcHY`a177F)Wz-{N>LQ093)%TDRsQ5v+DHd7DbWmBsyzR6J-h;Er(mij zlV<(!2{=jg_!DWHRi-%ifd-s1iD@Lo+f4H{2w2bg?BsMY>70haY)0>c%y2{pX5r2@ z;<}qCxlZPN*p!fbD-&#Ak}~ISoXyKInk)r=D6lvgyK>q!NBGL0sL~rf|59Vk*`L{o$?07v5`^Vo< zknVqQE7!d++1rWl@J<^{;y7gQ@Zsm9I{?3OD)p)OoM()0^bg=1m`IPkkxvG!*Hj}l zlpSS;RV@8y z8B4Pe$vbDfK%Oh(LBR3tCxy~0{<+zTc)0qtoq&T+kwn5>I3IxWJV>iO*?`ju+wdF9 z1hr19#yB%(?o6bE9Fp>UM3i}y_Kq^wpHVZ=EzehIO$DZw9%~FgcbZ*c1B?YV8Xsr4 zuT1#9y~C6?Y4-ARxABTTXrw4>4$c7+77hG^+Sr@#d5R0#Bp}Z5Tgh=o#vT~Oo@dgoEU~i zRa6$c+5uafoGD!%^OcpTm+5wA*r-^NNtyQBvXp<3ba0bL&0Qp>gW+sSq@qX(c>xuiM`LQr|Cxe-h26FDZ83oKW>nC7BW+Z5g9 zBcJ;Iz}}15ub2@6Um`HnH6RO(ddlxP0Q z8;i>J&w8%!;AE-!qcvkJ>trHjieE8f*F&Kmwfo82a*suSvPVJ zJ6$iy!O;RzaI~O>1V4`rMsUd4X;U! z3a5(gfy0t&P+n4E>4Xr@qHocM)iRuK}hQ+Ucmzn$P|%(iM))w z=b*ghd-nf)^atcC09cAPIE)>&e7r7^&<($3BBReniVv>=STXaAmgK^~YW;N2@W?#h zCzjynyq0}_#|81!aZR3-0hG`U$S}D4;!ooE5bnZ|ITwZ(s}CQ^0``LC9#D>im_DmK z{No?K@!@`9ULY5S#DaZV69KYQR&ObWdTK%#}V2OegMU-#9!+CfW_Z( zA3l*CGtWWiD?D%F%<^$UAWz!A##0q;86dm6Fi8#a2x)ENGK2%)06!leBJf3@8;}Fi z<7Wok)ZlW&1Xj~tO6*({1)Ajg1kfk&=dn8Z$3G?@)5(!1MZcp3B0~{?kU4tV1_Aq4 zf7p`&&MG5q(z)65XhQ$BJ&$O?RsJj^{LJh1kdG(V_qG$T)ZkG9VhR<(B`F6T1 zQ-@9`&Y)ZP`N8uu=Umgif%OYcj^7(|xmkM@P`qXra%lrO$~_-P8c%e`4;~i3Wwz8z zrekn`Q!-Cxxwx7+aGj@oJ{x}hWb@uee2-RCZ_&n`0|2lya)1&e^NIqb37 z_`y@nw_Bfvz>z9}-2ZH%IRn_8Cldb-8T#o)l5c^^=?c0QlZ;Zn;h?zr;-wHHB^Pl} z{G5@<@pzP__|#PK?b5F8L97HaMPPW_Z|7l`o3(w7;o)0UGSN&V9GURwA2B$5mb0sN z&>Qr>`~>i2PBW)c^_7NxZ#nR~V*G{XW}`I3%@V)FsBHSroo-h5m8N5F zHPPR9GWKdvH7`p8!F;hD<%%ZI16RjC$uR7fo`1b6R^5mPj0t|Sk=N`*WgZiLy(!mQ zx&G+m{_PX4`NseHIuUnV8~^NuKW+IlRDr~2jy}$*-m(5aY*G~lh994Ao1GoG5syr4 z@maU$Ob;ym{Q7Cx<;nEvIq0|^@yrvCuQo_}UixKx?q-|5?)2uZYIxX73ncU1UTy~r;Pu4){@SOestbRy0B%TEXO5*wiOh>{p(c^UF z51!h~;kr1Oi}Hbx$$tFAUUb29%JoL#zv__QnfbCKY> mEcy$Mx}6tyze)E0?z~!7QC$9;!`GhEf*T6vuYUQ-v14G{uAWI&nIXn?%w+1I2e&Cy!q$>xQ-kLRIh_i?X&6ugLzDS_n-qYurgbb zTHR8&1o+gEal+|>n-Acoowuk1z1!l@K5GphPn$D|{D9%y8EHK&^#koXMOC^?B-Qq< z5#0*kr+i^wJW%cXQt~nh^S!m4QwpstCy2vi4b#Cu`pQ5xyk`0>1jr6kq9pX z<6VHO7K{4zgVjm(qVW`>t{#(h5S*VvP`EU68V(sy_+axeTv?pY6vcpC?A6J2e<@6c zT?r;e6yW{X#=$yL6E7t=kO0%ui$j z{pwl@W$>_&+9*iPCb#P4%E>q;g)`k;THv{88>g$SRJR_jxxM9{E3lfZmC|%8mmY|l z%fXa)CAI7)V{tKr=9IK6(5pq)nISfa2VOOq)Q>F6xcbbAYh(A-M2-|XdkxLbSPdL_ z%dQ8yJUy=Q4VV%JPnC_M`qugROlg_UtO+H-Hr%Z;k9K#dK12)?qqF=cS&kJGjF|co2XuHT8EE1pn2gKDBQt^8YQVCwJQy7)P`w%);8wOjC-{0Kjao* zzukmj(Dckf(Ejp-HiXvnfWhO2Bf82?wlxdp|yIdyUDsjeRe&c3Mrd7Ej3bn zjbM|>%Ag&Tc!sVobt9piZ?h8Oxd25KghVbCq^d%$mu)Ns;_LBfO(+U-zFbq_Y4oUu zrV*ZDrKFofBRJ9yC*>~KmJLkn^ywUVw9 zLaWura+0!o)(TfAEaRBdJ)-MN=yy8sP&4q{jA<_qMo~w#gMANA&{j=2TNwi0qvuC| zdTY^=i8j-L7!enXQhK2D!c2`O3h5x7V5mT4r4#BUJDCtxVV4S2BEJ!e5dKxtPY0Uo zNl!ZETW!`FwLz{D>t-moYzuZ{O5Y{*iIIC8-u8qM$4DpU#ID#+j%d|Ze|qfo$w`C$ zvfTiTWeTyv&-&k0S8M@Mv4rV=`*C)6{P^rytLjhM#-YFQtIFnrlklWP2j7UsiBi7H z0m4sDlA45YF!R{ZmJu8T-g16-j9FV|D0Z;KrxrEE*E>Ua2`BC(tRoN5Umo2wWJiur z?+o{Y&z$$eB7YkGa2i44jbKC1!H@i*|LE!2-SPAxzHeMy7*oH>hsWScaqeEn^)R62 zhyN}L(z_A`&|*z+iaWGBX>e}a^3DA6cB#?3x-0H4Yh3sXe~b6J#PmCv@oy4k?jI9H peOq`$xBrf^?|Ee4wbmXc64E<%*X`b6XVn*aH^y~Z1Q{+H{six0 { - socket.on("message", (message) => { - socket.emit( - JSON.stringify({ - op: 2, - d: { - ssrc: 1, - ip: "127.0.0.1", - port: 3004, - modes: [ - "xsalsa20_poly1305", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305_lite", - ], - heartbeat_interval: 1, - }, - }) - ); + this.ws.on("connection", async (socket: WebSocket) => { + await setHeartbeat(socket); + + socket.on("message", async (message: string) => { + const payload: Payload = JSON.parse(message); + + if (OPCodeHandlers[payload.op]) + await OPCodeHandlers[payload.op](socket, payload); + else + console.error(`Unimplemented`, payload) }); }); } async listen(): Promise { // @ts-ignore - await (db as Promise); + await initDatabase(); await Config.init(); console.log("[DB] connected"); console.log(`[WebRTC] online on 0.0.0.0:${port}`); diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts new file mode 100644 index 00000000..5cc66506 --- /dev/null +++ b/webrtc/src/opcodes/Connect.ts @@ -0,0 +1,5 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; + +export async function onConnect(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts new file mode 100644 index 00000000..04150e36 --- /dev/null +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -0,0 +1,7 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { setHeartbeat } from "./../util"; + +export async function onHeartbeat(socket: WebSocket, data: Payload) { + await setHeartbeat(socket); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts new file mode 100644 index 00000000..2026d7c9 --- /dev/null +++ b/webrtc/src/opcodes/Identify.ts @@ -0,0 +1,20 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index" +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onIdentify(socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.READY, + d: { + ssrc: 1, + ip: "127.0.0.1", + port: 3005, + modes: [ + "xsalsa20_poly1305", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305_lite", + ], + heartbeat_interval: 1, + }, + })); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts new file mode 100644 index 00000000..de21eba6 --- /dev/null +++ b/webrtc/src/opcodes/Resume.ts @@ -0,0 +1,5 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; + +export async function onResume(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts new file mode 100644 index 00000000..f1732dd9 --- /dev/null +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -0,0 +1,16 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onSelectProtocol(socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.SESSION_DESCRIPTION, + d: { + video_codec: "H264", + secret_key: new Array(32).fill(null).map(x => Math.random() * 256), + mode: "aead_aes256_gcm_rtpsize", + media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + audio_codec: "opus", + } + })); +} \ No newline at end of file diff --git a/webrtc/src/opcodes/Speaking.ts b/webrtc/src/opcodes/Speaking.ts new file mode 100644 index 00000000..14f86b3c --- /dev/null +++ b/webrtc/src/opcodes/Speaking.ts @@ -0,0 +1,6 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index" +import { VoiceOPCodes } from "@fosscord/util"; + +export async function onSpeaking(socket: WebSocket, data: Payload) { +} \ No newline at end of file diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts new file mode 100644 index 00000000..2fe69c38 --- /dev/null +++ b/webrtc/src/opcodes/index.ts @@ -0,0 +1,35 @@ +import { WebSocket } from "@fosscord/gateway"; +import { VoiceOPCodes } from "@fosscord/util"; + +export interface Payload { + op: number; + d?: any; + s?: number; + t?: string; +} + +import { onIdentify } from "./Identify"; +import { onSelectProtocol } from "./SelectProtocol"; +import { onHeartbeat } from "./Heartbeat"; +import { onSpeaking } from "./Speaking"; +import { onResume } from "./Resume"; +import { onConnect } from "./Connect"; + +export type OPCodeHandler = (this: WebSocket, data: Payload) => any; + +export default { + [VoiceOPCodes.IDENTIFY]: onIdentify, //op 0 + [VoiceOPCodes.SELECT_PROTOCOL]: onSelectProtocol, //op 1 + //op 2 voice_ready + [VoiceOPCodes.HEARTBEAT]: onHeartbeat, //op 3 + //op 4 session_description + [VoiceOPCodes.SPEAKING]: onSpeaking, //op 5 + //op 6 heartbeat_ack + [VoiceOPCodes.RESUME]: onResume, //op 7 + //op 8 hello + //op 9 resumed + //op 10? + //op 11? + [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 + //op 13? +}; \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 68867a2c..5614982d 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,3 +1,4 @@ import { Server } from "./Server"; const server = new Server(); +server.listen(); \ No newline at end of file diff --git a/webrtc/src/util/Heartbeat.ts b/webrtc/src/util/Heartbeat.ts new file mode 100644 index 00000000..7b5ed9cd --- /dev/null +++ b/webrtc/src/util/Heartbeat.ts @@ -0,0 +1,18 @@ +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; +import { VoiceOPCodes } from "@fosscord/util"; + +export async function setHeartbeat(socket: WebSocket) { + if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout); + + socket.heartbeatTimeout = setTimeout(() => { + return socket.close(CLOSECODES.Session_timed_out); + }, 1000 * 45); + + socket.send(JSON.stringify({ + op: VoiceOPCodes.HEARTBEAT_ACK, + d: { + v: 6, + heartbeat_interval: 13750, + } + })); +} \ No newline at end of file diff --git a/webrtc/src/util/index.ts b/webrtc/src/util/index.ts new file mode 100644 index 00000000..e8557452 --- /dev/null +++ b/webrtc/src/util/index.ts @@ -0,0 +1 @@ +export * from "./Heartbeat" \ No newline at end of file diff --git a/webrtc/tsconfig.json b/webrtc/tsconfig.json index 77353db0..fb93b0bd 100644 --- a/webrtc/tsconfig.json +++ b/webrtc/tsconfig.json @@ -1,5 +1,8 @@ { "include": ["src/**/*.ts"], + "ts-node": { + "require": ["tsconfig-paths/register"], + }, "compilerOptions": { /* Visit https://aka.ms/tsconfig.json to read more about this file */ @@ -18,7 +21,7 @@ "sourceMap": true /* Generates corresponding '.map' file. */, // "outFile": "./", /* Concatenate and emit output to single file. */ "outDir": "./dist/" /* Redirect output structure to the directory. */, - "rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, + "rootDir": "../" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */, // "composite": true, /* Enable project compilation */ // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ // "removeComments": true, /* Do not emit comments to output. */ @@ -62,11 +65,19 @@ // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ /* Experimental Options */ - // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ - // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ /* Advanced Options */ "skipLibCheck": true /* Skip type checking of declaration files. */, - "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ + "forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */ + + "baseUrl": "../", + "paths": { + "@fosscord/api": ["api/src/index"], + "@fosscord/gateway": ["gateway/src/index"], + "@fosscord/cdn": ["cdn/src/index"], + "@fosscord/util": ["util/src/index"] + }, } } From aa0357e06015a900aa6757ab9b0eef968eb02052 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Sun, 16 Jan 2022 03:35:30 +1100 Subject: [PATCH 02/10] messing around with things I don't understand --- webrtc/src/Server.ts | 70 ++++++++++++++++++++++++++-- webrtc/src/opcodes/Connect.ts | 3 +- webrtc/src/opcodes/Heartbeat.ts | 3 +- webrtc/src/opcodes/Identify.ts | 7 ++- webrtc/src/opcodes/Resume.ts | 3 +- webrtc/src/opcodes/SelectProtocol.ts | 5 +- webrtc/src/opcodes/Speaking.ts | 3 +- 7 files changed, 81 insertions(+), 13 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 06a36df9..cdda10ec 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,15 +1,19 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, CLOSECODES, Payload, OPCODES } from "@fosscord/gateway"; +import { WebSocket, Payload, } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; -import { setHeartbeat } from "./util" -import mediasoup from "mediasoup"; +import { setHeartbeat } from "./util"; +import * as mediasoup from "mediasoup"; +import { types as MediasoupTypes } from "mediasoup"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; export class Server { public ws: WebSocketServer; + public mediasoupWorkers: MediasoupTypes.Worker[] = []; + public mediasoupRouters: MediasoupTypes.Router[] = []; + public mediasoupTransports: MediasoupTypes.Transport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -23,9 +27,9 @@ export class Server { const payload: Payload = JSON.parse(message); if (OPCodeHandlers[payload.op]) - await OPCodeHandlers[payload.op](socket, payload); + await OPCodeHandlers[payload.op].call(this, socket, payload); else - console.error(`Unimplemented`, payload) + console.error(`Unimplemented`, payload); }); }); } @@ -34,7 +38,63 @@ export class Server { // @ts-ignore await initDatabase(); await Config.init(); + await this.createWorkers(); console.log("[DB] connected"); console.log(`[WebRTC] online on 0.0.0.0:${port}`); } + + async createWorkers(): Promise { + const numWorkers = 1; + for (let i = 0; i < numWorkers; i++) { + const worker = await mediasoup.createWorker(); + if (!worker) return; + + worker.on("died", () => { + console.error("mediasoup worker died"); + }); + + worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { + console.log("new router"); + + this.mediasoupRouters.push(router); + + router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { + console.log("new transport"); + + this.mediasoupTransports.push(transport); + }) + + await router.createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true + }); + }); + + await worker.createRouter({ + mediaCodecs: [ + { + kind: "audio", + mimeType: "audio/opus", + clockRate: 48000, + channels: 2 + }, + { + kind: "video", + mimeType: "video/H264", + clockRate: 90000, + parameters: + { + "packetization-mode": 1, + "profile-level-id": "42e01f", + "level-asymmetry-allowed": 1 + } + } + ] + }); + + this.mediasoupWorkers.push(worker); + } + } } diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5cc66506..5db11638 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onConnect(socket: WebSocket, data: Payload) { +export async function onConnect(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 04150e36..06d6bcb1 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,7 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { setHeartbeat } from "./../util"; +import { Server } from "../Server" -export async function onHeartbeat(socket: WebSocket, data: Payload) { +export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { await setHeartbeat(socket); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 2026d7c9..6043a460 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,14 +1,17 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onIdentify(socket: WebSocket, data: Payload) { +export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { ssrc: 1, ip: "127.0.0.1", - port: 3005, + + //@ts-ignore + port: this.mediasoupTransports[0].iceCandidates.port, modes: [ "xsalsa20_poly1305", "xsalsa20_poly1305_suffix", diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index de21eba6..dcd4f4cd 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,5 +1,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; +import { Server } from "../Server" -export async function onResume(socket: WebSocket, data: Payload) { +export async function onResume(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index f1732dd9..fcc45855 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,15 +1,16 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSelectProtocol(socket: WebSocket, data: Payload) { +export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { video_codec: "H264", secret_key: new Array(32).fill(null).map(x => Math.random() * 256), mode: "aead_aes256_gcm_rtpsize", - media_session_id: "d8eb5c84d987c6642ec4ce72ffa97f00", + media_session_id: this.mediasoupTransports[0].id, audio_codec: "opus", } })); diff --git a/webrtc/src/opcodes/Speaking.ts b/webrtc/src/opcodes/Speaking.ts index 14f86b3c..861a7c3d 100644 --- a/webrtc/src/opcodes/Speaking.ts +++ b/webrtc/src/opcodes/Speaking.ts @@ -1,6 +1,7 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index" import { VoiceOPCodes } from "@fosscord/util"; +import { Server } from "../Server" -export async function onSpeaking(socket: WebSocket, data: Payload) { +export async function onSpeaking(this: Server, socket: WebSocket, data: Payload) { } \ No newline at end of file From 7c87f8d60a15359a129ab076c212cbf4f1cd8884 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 17 Jan 2022 02:59:26 +1100 Subject: [PATCH 03/10] more fuckery --- webrtc/package-lock.json | Bin 31537 -> 32074 bytes webrtc/package.json | 1 + webrtc/src/Server.ts | 31 ++++++++---- webrtc/src/opcodes/Connect.ts | 4 ++ webrtc/src/opcodes/Identify.ts | 68 ++++++++++++++++++++++++-- webrtc/src/opcodes/SelectProtocol.ts | 70 +++++++++++++++++++++++++-- webrtc/src/start.ts | 6 +++ 7 files changed, 161 insertions(+), 19 deletions(-) diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 6c3726dc425b0a2cc59c41eb82b859538beb60c6..43bb8cd83ccd4da91b13dac0631ab7a60e07109f 100644 GIT binary patch delta 321 zcmdn^jq%hk#t8?6Q}Rnv^U9R06qMo&jr0ukOq6ss?&_CfgNRRd^c7_V3s0UHsV@pv ztFNmMk<^7~n0!!7J}f^xr7$VDvOF!V%EK_nBs{`9z$4tzAS67pps>K)v(hK6Fe|el z#H6$!-_tGFz$3u7u*5IFq%u3Oz{MofGu^-m$j?nl4{**f3imDZEitbQ$t<57=%_OJ b0E^saPiYBz5}g!D8zD-t3gxTq`4Czqu;rf&u`Gr3(iD diff --git a/webrtc/package.json b/webrtc/package.json index 8c66245d..d5a994a1 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -18,6 +18,7 @@ "typescript": "^4.3.2" }, "dependencies": { + "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", "tsconfig-paths": "^3.12.0", diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index cdda10ec..1d2e73e7 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -54,21 +54,32 @@ export class Server { }); worker.observer.on("newrouter", async (router: MediasoupTypes.Router) => { - console.log("new router"); + console.log("new router created [id:%s]", router.id); this.mediasoupRouters.push(router); - router.observer.on("newtransport", (transport: MediasoupTypes.Transport) => { - console.log("new transport"); + router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + console.log("new transport created [id:%s]", transport.id); + + await transport.enableTraceEvent(); + + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { + console.log("new producer created [id:%s]", producer.id); + }); + + transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { + console.log("new consumer created [id:%s]", consumer.id); + }); + + transport.observer.on("newdataproducer", (dataProducer) => { + console.log("new data producer created [id:%s]", dataProducer.id); + }); + + transport.on("trace", (trace) => { + console.log(trace); + }); this.mediasoupTransports.push(transport); - }) - - await router.createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], - enableUdp: true, - enableTcp: true, - preferUdp: true }); }); diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index 5db11638..b312d6f2 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -3,4 +3,8 @@ import { Payload } from "./index"; import { Server } from "../Server" export async function onConnect(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 15, + d: { any: 100 } + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6043a460..6bbed04c 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,9 +1,67 @@ import { WebSocket } from "@fosscord/gateway"; -import { Payload } from "./index" +import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; + +const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { + var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + listenIps: [{ ip: "127.0.0.1" }], + enableUdp: true, + enableTcp: true, + preferUdp: true, + }); + + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + var producer = await transport.produce( + { + kind: "audio", + rtpParameters: + { + mid: "1", + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + id: i + 1, + uri: x, + })) + } + }); + + const consumer = await transport.consume( + { + producerId: producer.id, + rtpCapabilities: + { + codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { + return { + mimeType: x.mimeType, + kind: x.kind, + clockRate: x.clockRate, + channels: x.channels, + payloadType: x.preferredPayloadType as number + }; + }), + headerExtensions: test.split("\na=").map((x, i) => ({ + kind: "audio", + preferredId: i + 1, + uri: x, + })) + } + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { @@ -11,11 +69,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) ip: "127.0.0.1", //@ts-ignore - port: this.mediasoupTransports[0].iceCandidates.port, + port: transport.iceCandidates[0].port, modes: [ "xsalsa20_poly1305", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305_lite", + // "xsalsa20_poly1305_suffix", + // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, }, diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index fcc45855..24e8ef5f 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -1,17 +1,79 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; -import { Server } from "../Server" +import { Server } from "../Server"; + +/* + { + op: 1, + d: { + protocol: "webrtc", + data: " + a=extmap-allow-mixed + a=ice-ufrag:ilWh + a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-options:trickle + a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level + a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time + a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 + a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid + a=rtpmap:111 opus/48000/2 + a=extmap:14 urn:ietf:params:rtp-hdrext:toffset + a=extmap:13 urn:3gpp:video-orientation + a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay + a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type + a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing + a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space + a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id + a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id + a=rtpmap:96 VP8/90000 + a=rtpmap:97 rtx/90000 + ", + sdp: "same data as in d.data? also not documented by discord", + codecs: [ + { + name: "opus", + type: "audio", + priority: 1000, + payload_type: 111, + rtx_payload_type: null, + }, + { + name: "H264", + type: "video", + priority: 1000, + payload_type: 102, + rtx_payload_type: 121, + }, + { + name: "VP8", + type: "video", + priority: 2000, + payload_type: 96, + rtx_payload_type: 97, + }, + { + name: "VP9", + type: "video", + priority: 3000, + payload_type: 98, + rtx_payload_type: 99, + }, + ], + rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", + }, + } +*/ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: "H264", + video_codec: data.d.codecs.find((x: any) => x.type === "video").name, secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "aead_aes256_gcm_rtpsize", + mode: "xsalsa20_poly1305", media_session_id: this.mediasoupTransports[0].id, - audio_codec: "opus", + audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, } })); } \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 5614982d..299bfce8 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,4 +1,10 @@ +import { config } from "dotenv"; +config(); + import { Server } from "./Server"; +//testing +process.env.DATABASE = "../bundle/database.db"; + const server = new Server(); server.listen(); \ No newline at end of file From 9d86d10a85dabca86acc357f8cba15646da1a552 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 21 Jan 2022 21:04:45 +1100 Subject: [PATCH 04/10] mmmm --- webrtc/package-lock.json | Bin 32074 -> 33275 bytes webrtc/package.json | 2 + webrtc/src/opcodes/Identify.ts | 79 ++++++++++----------------- webrtc/src/opcodes/SelectProtocol.ts | 37 +++++++++++++ webrtc/src/opcodes/index.ts | 2 + 5 files changed, 69 insertions(+), 51 deletions(-) diff --git a/webrtc/package-lock.json b/webrtc/package-lock.json index 43bb8cd83ccd4da91b13dac0631ab7a60e07109f..afba7e761c8a7595e1dff36c4faee723403a8d91 100644 GIT binary patch delta 723 zcmX^0i}80e(*}Mc!^!N7_2SqhP4rARmoqNl zJ_lxker- z0mYG_e#w=Y8Bwmr?v>7w7L~=mkshUi?rA<5?oovnmVUV=Il)z-KH(u%QTde~CF$B> z{`#INMR^rb9szj)!R~pElOM9`Y!>2X)8s++8OVK;1yzmYQCx!QvdMzUDv?21QI!$K zd12*VmCmIh`KBfA0TtQ-QQ_GxzRv09`Tjly!8sXOE-6uI<)Qh0h38*PNaAzqeRl$llu3hv~L)a2~c nBE78Q$%!&Do5fASr3Z1y%5F~g?^GZ=WW#6^vYQ|0r}F{;8%F!G delta 38 ucmey}%yjA(;|6}l$@+}to6j&#GA=r&*ZvX4T*(h0O~~qIdx?K@QRY diff --git a/webrtc/package.json b/webrtc/package.json index d5a994a1..b9bac356 100644 --- a/webrtc/package.json +++ b/webrtc/package.json @@ -13,6 +13,7 @@ "license": "ISC", "devDependencies": { "@types/node": "^15.6.1", + "@types/sdp-transform": "^2.4.5", "@types/ws": "^7.4.4", "ts-node": "^10.4.0", "typescript": "^4.3.2" @@ -21,6 +22,7 @@ "dotenv": "^12.0.4", "mediasoup": "^3.9.5", "node-turn": "^0.0.6", + "sdp-transform": "^2.14.1", "tsconfig-paths": "^3.12.0", "ws": "^7.4.6" } diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 6bbed04c..c31870c8 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -2,10 +2,6 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; -import * as mediasoup from "mediasoup"; -import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; - -const test = "extmap-allow-mixed\na=ice-ufrag:ilWh\na=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ\na=ice-options:trickle\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\na=rtpmap:111 opus/48000/2\na=extmap:14 urn:ietf:params:rtp-hdrext:toffset\na=extmap:13 urn:3gpp:video-orientation\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\na=rtpmap:96 VP8/90000\na=rtpmap:97 rtx/90000"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ @@ -15,52 +11,31 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) preferUdp: true, }); - const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; - const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; - - var producer = await transport.produce( - { - kind: "audio", - rtpParameters: - { - mid: "1", - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - id: i + 1, - uri: x, - })) - } - }); - - const consumer = await transport.consume( - { - producerId: producer.id, - rtpCapabilities: - { - codecs: codecs.filter(x => x.kind === "audio").map((x: RtpCodecCapability) => { - return { - mimeType: x.mimeType, - kind: x.kind, - clockRate: x.clockRate, - channels: x.channels, - payloadType: x.preferredPayloadType as number - }; - }), - headerExtensions: test.split("\na=").map((x, i) => ({ - kind: "audio", - preferredId: i + 1, - uri: x, - })) - } - }); + /* + //discord proper sends: + { + "streams": [ + { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, + { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 1311884, + "port": 50008, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.214.158", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] + } + */ socket.send(JSON.stringify({ op: VoiceOPCodes.READY, @@ -71,11 +46,13 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) //@ts-ignore port: transport.iceCandidates[0].port, modes: [ - "xsalsa20_poly1305", + "aead_aes256_gcm_rtpsize", + // "xsalsa20_poly1305", // "xsalsa20_poly1305_suffix", // "xsalsa20_poly1305_lite", ], heartbeat_interval: 1, + experiments: [], }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 24e8ef5f..08335ade 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -2,6 +2,9 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { VoiceOPCodes } from "@fosscord/util"; import { Server } from "../Server"; +import * as mediasoup from "mediasoup"; +import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; +import * as sdpTransform from 'sdp-transform'; /* { @@ -66,6 +69,40 @@ import { Server } from "../Server"; */ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { + const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; + const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; + + const transport = this.mediasoupTransports[0]; //whatever + + const res = sdpTransform.parse(data.d.sdp); + + /* + res.media.map(x => x.rtp).flat(1).map(x => ({ + codec: x.codec, + payloadType: x.payload, + clockRate: x.rate as number, + mimeType: `audio/${x.codec}`, + })), + */ + + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 2fe69c38..36d30e7d 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -32,4 +32,6 @@ export default { //op 11? [VoiceOPCodes.CLIENT_CONNECT]: onConnect, //op 12 //op 13? + //op 15? + //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} }; \ No newline at end of file From 9bdc7933a8ffbb7cbb55f70909b2073bba30bde8 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Fri, 4 Feb 2022 18:46:09 +1100 Subject: [PATCH 05/10] ;jondfgsk --- webrtc/src/Server.ts | 19 +++--------- webrtc/src/opcodes/Identify.ts | 7 ++--- webrtc/src/opcodes/SelectProtocol.ts | 46 ++++++++++++++++++---------- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d2e73e7..dcbf216a 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,5 +1,5 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, Payload, } from "@fosscord/gateway"; +import { WebSocket, Payload, CLOSECODES } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; import OPCodeHandlers from "./opcodes"; import { setHeartbeat } from "./util"; @@ -28,8 +28,10 @@ export class Server { if (OPCodeHandlers[payload.op]) await OPCodeHandlers[payload.op].call(this, socket, payload); - else + else { console.error(`Unimplemented`, payload); + socket.close(CLOSECODES.Unknown_opcode); + } }); }); } @@ -46,7 +48,7 @@ export class Server { async createWorkers(): Promise { const numWorkers = 1; for (let i = 0; i < numWorkers; i++) { - const worker = await mediasoup.createWorker(); + const worker = await mediasoup.createWorker({ logLevel: "debug" }); if (!worker) return; worker.on("died", () => { @@ -91,17 +93,6 @@ export class Server { clockRate: 48000, channels: 2 }, - { - kind: "video", - mimeType: "video/H264", - clockRate: 90000, - parameters: - { - "packetization-mode": 1, - "profile-level-id": "42e01f", - "level-asymmetry-allowed": 1 - } - } ] }); diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index c31870c8..82f327be 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -5,7 +5,7 @@ import { Server } from "../Server"; export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { var transport = await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "127.0.0.1" }], + listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, preferUdp: true, @@ -40,10 +40,9 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { + streams: [], ssrc: 1, - ip: "127.0.0.1", - - //@ts-ignore + ip: transport.iceCandidates[0].ip, port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 08335ade..36527a8b 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -68,6 +68,8 @@ import * as sdpTransform from 'sdp-transform'; } */ +var test_hasMadeProducer = false; + export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; const codecs = rtpCapabilities.codecs as RtpCodecCapability[]; @@ -85,23 +87,33 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })) - }, - paused: false, - }); + if (!test_hasMadeProducer) { + const producer = await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: 48000, + payloadType: 111, + mimeType: "audio/opus", + channels: 2, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, + })) + }, + paused: false, + }); + + const consumer = await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }) + + test_hasMadeProducer = true; + } socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, From d491bcd8bed78703331bc6550cf3d52e3647e764 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Thu, 17 Feb 2022 18:54:16 +1100 Subject: [PATCH 06/10] Discord.js gateway connection when connecting to voice will close 4002 decode error due to not sending self_video field. temp solution: remove it from server. this commit will probably be reverted later; I'm just trying to see if a proper ( not self signed ) SSL cert on slowcord.maddy.k.vu will fix this SSL error I'm receiving. --- bundle/package-lock.json | Bin 576191 -> 589262 bytes bundle/package.json | 1 + gateway/src/schema/VoiceStateUpdateSchema.ts | 6 +-- webrtc/.vscode/launch.json | 4 +- webrtc/src/Server.ts | 14 ++++-- webrtc/src/opcodes/Identify.ts | 45 ++++++++++++++++--- webrtc/src/opcodes/index.ts | 6 +-- 7 files changed, 59 insertions(+), 17 deletions(-) diff --git a/bundle/package-lock.json b/bundle/package-lock.json index 898c041c76670ef3d9676afe6c64b3b6ac5f60e9..9464d164300482e180a4b3eb3e45b12b157e2b16 100644 GIT binary patch delta 17728 zcmcJ1XLuFW*8eko(i^0Z-smBrh;%~l5PFA1dIxw%n7VDvDZ@@pBghV6nyawwBDPb$igql#&U5JEZgOpW9)@l{TwNm{dI#mic@>U;V zqam+{5L!HMOTZ-O!$DM>SC?Ecu5QTYC_61egYIlw4etr+En z5ynr&8E!+6ncWuEOgzck0}S6i?m~OKrUE>jge?BGNvMKH>jMQlm0%dy7+EhLrKA{- z5BfpG6y=7|VBa$*rN+?he$Ysg0-!8Gh&7(u03z?`tF<*ktj`#Q*bj@L(Zprn1+S8HgUOgd3v|8*F~D z|B(uUoq%K*I$QCCZH)yFs4+)b0>5q&f|*{vdmxz-vcY0^`CMf_ZrKeI=P4`sYyTsf zabs@}q_Mk^dQvyKeUtB`(Y+X&?P_^W^DcTCz~FbLn{^EIJ<_=F3vmup~*9| zy1etXi)UTfVq3Z-AB`w?nG$6@yOM9zy0p!0J^H;k`SM`*DMgo+5M$|$)zCaoDdML< zw`a?(TcpMpAG+4vea`6*l_+?=NY#s%-e~Fmaa{``(s<`@FXPO$Jmc!EkNE+bzMwj! zHWmgzUJJn&=KUkoft+@NXK~gKZe+D%OvL=?3CVAvoE~gZf{ci7qa>cSwLo`kPrt2Tl;215-htm)z+<*RRRNdU0Humxci5&&U4+qc z85H>np3rp{DpOjKQs6?v2KwIgOU|`+cnWKozzDj~g`y=pwo}dP=uuDj3N}%(_p4e9 z=$+%yDb>T^ym^l@gzuDho+;ZEBkj6MN%4X;bcMeHa0M%rIyW!at8B9|h>T!}O(H`p z2oqdSf-$}k9`1LrM0sB4B~&DHLWFTI8M^@vh6zP(+~gcmba;LP1AWp#w|JyM`6R`M zghvQl?W>uh)d?5MAu{;xGBvz>N-c#q~g3AL+`Z1V`%U-@-lm+T-hkNQqRcVi^Psn zf=O}}Ayojst>^-r^8US@Pyw!lp*vbqO|aPsJJYAP6kjN-&RI!-wRaV7SemSRliV6Y z2UjKE06UU|t!~8JY>%;U&?$#t>82P4y3t$*%DjbfkXMSQXL0AGYWZp9LpMduD^p~+ zURN+PHRn)#zs{j)wT)R9x&WF~xFx_ZItYQ_bJp$?E$;t0>(G7_EJ2?*(|$M($4$_2 z`vh3Lo%@&a8Em^G2OFtZ+rX-yludA|t?tj0LP}Wn9VMGw>ngP5#zWK5>QFvJ@h20y z3qH0j2!3hJ2J!3-@_Gx~t*nhc>M)>>@B!bPS_4f53StB=(zLH23bwJS{W0C>`dC?+ z-(OIvM0jQdvHgW_ElVTX03nF)NNNodvIH)Enm*_6mLj2G5>CD{*z^F9^^ceUseg(A z@Z49$MD#aNfg6IFmLI07P%;QzVal&!dp@gZxFEy&p+dg38{|F~6w)nSsGx8TavlYc zv`69~iBGvd8hsbd@7X|j`a~Q9U1VuI|EhYa>)Fv#GNRyumkLlmUC4y~;~n&|_5Cvi zGc_g)CkPj0C-Y7DI{s790gj)IieCPypg_rFVGi9ifpniDOtEq}bF}`t3r$FT4ERS$ z(XjSY!3Xjxsg=O5q8bm;puZr3ebWSFy z6Ff-tTw!`8#wVnKEW^(Ch3}ZG$*NUCsASWeq;=>syk1e4yT5yTt>E@p1Y=g{Bg{8*|}9?NyetBvFancFM+w ze(-=1oiWugiBrk&(0V&E~$2{6qsL8jO~-ws$-q;?OKg&(0X$BDU;gNt6e*-e~qraCTETB zRIkDKdL1Tooltw+u$EaVV|tGt-)TVqcAY1VnAopP-*zpMJ9M3tIW1#Cr(s$B$Mvq4 z-Ev0HiM`u)?%ujhRtxyHr`!a*mZ*`!D^P8T8VJ=y>NLvLwP~{CN8Om;HUhwla zty)~x@DWKF!-r>0NXp0_gJe!{d687FR$YiaBSjiFJIH-rtvk6*!y0vaXLRo0zinpg z$!!NrnAobrgqdkGM%V2!vBR_h-5L&?_DX{p9VWIIFuGgUNs}^Oo78_s{}FY2P3!U=da!&Zc>HN^~^%bL`^LK(LNjonz_Ouz!jZ8Iy z9J($vwp>Lxq>L3CL0*BVLwT4~mCU;-wBpSMa^VM|kLG43aA&I=0p*ue1s+UOXOZjo zg*9w=B0)#d&5QuU=ctt-?6z!ku$Wi3$L~*+eN4KcUNu?U% zPcQ5koFu;BwT>r=(^Y3x8$f#06BpW@iK9>shk~O*3`wdlPV?ffh}OA~w@mT_|7og1 zPPGvCn5S2=L)T#5A+;_kYb{nY<0@0U<_mF7Br*s^7vc;z}E7NnORiZBD0IH*prfHN^emnMLop`OVmxlqmQ_mpWD$s;-|9Ha9Gwrb`GTS zb2Q>2A#JdD*m+JoMjVrFd#X0_XO0n9JJ$&(XmqJ6 zSF?(@v`|=+Ep{T(MDab-GOx>GMY4U8s92PTlT%YfQF5MjIHYAual~u7i0uVu{Xr9q zNEm-d-EK77XXZ3N%@oaU7EO*Y4VIe2!rMY^5q)7O@hbG(ii2XHzhfp(YFS zj*F-HDQBDz?@N~GsdYw@g{Q?rn}>y_-->IjG3b*tcs^fv4F;bT2U!Y5&YTtJ+I%PT zUF)ylcBkj6KCu40XnHuB9QOTA?7`el0y%I@Q+Py(s>z1W5S2{E%J&-`&%G~Z0o z+QHDtX5NbHRW@=zVTrPtOXI80EU5q;0eM%^88~v|ru3Z$Z0&&Ug264M2-tB=%&^HV z{<;{(T_8_SAKeo}NahW(60`V3t~8D6Yjq)Kyv($w3;gphF&r{)qZea@hyNh-;UY9v))<=!89K|gp&rq1UrEy=Zi?*A7Zlm z7lL2?J(T&X?crG?Es*T&qG8js;DQ`Yi;XUDT#yR6baO<>p%#p9TAq-nk~GXDkzNCk zWs3B>`HYmXY6d$YTd)uqQh>SlvOmR<@TxAcO2aIfP}w!|M3>rFT5EDu=qd5u2tRz3 z)>9Wzc7ZG25-)@Kh$307LazHrv39AkngUtv0FNW3R9k)3Btz&yn)pjAM1EeZ`%h~e zpi@ZZAT3Zbk|&tNqy(bD-3^s|T@`M<0FEzKwcwVq+;-KvW6tfWB%*9G2-k#@0TfSgSEEv{)l#bWoF>5W6wX5rYihZaem#Yv5< zlZb@XW3^>a@*B#3aH6!^Mv+T$l?1CQJB042suCat6%n3{*CxQs@eak9Ta|aVUtv5n zn4qt#ws9F_aZy1OVw>L5FXdSEm&IE1awa*%hoCbj!_{w8U57L zBKZ{vOOjSQL#a0*6SXi2(n6(b#@uf!!MNIz2WeDWiWK0%epF;D<2+ag<;0eLdzAbh z%t}O8k*4V~_>9Jcp-DYy3itDL202hq`p6#lBfh}Wd*^gbBYE|uZ~?B(*ZrXDKj=2o za?xpbt$}rM!C@&7qHjtbP}RWXq+tVTkki?RksS@B>K-=P=CzTcNI*+zxMYPFf-kYh zjt7?j9ok9vZC*34BkI`b#ads>p98+r$=B_rS&D67->y;ygl|D-eX6SjwkSbnbeC!= zHVRlMxzR`ZRfji4^xD{kCfJ4kP#}jgBn+SSmvaK0`}&eev!n%H)-!l7mPR=_ZAyw3 zOKE;iihSYGaw!DLi^Uky6{Hnbs?n50tdi!M5gW6uzic@bIanaQVipaoC(>7pfm(&q zMlPJlW^y=rUMTH1XP15?fzCdW=|xf#`^g>KB|Ws5CS|*&1W&+RWdUG9;z8CNkXGri za^j2USAGDS26#ys2N50QBS0U?rEjFD(3g%*`4MxI@js)Mmw0PYR!jRMh)9 z17CYceJ=T|ydrP6w91w;1&VS{u3uz^B#-d4b|t_1$Rl*yb5^i?@`ds#ElS=EkJ{kH zD^_};u>P*1k?3f7m+78px8N!f@me-Tr#%I(Fc=>a&y8&1^LFLL9- z_7zL4U~{~@(!EaUo*=V|DgmC<)x#ksQT~jp8Z&G((@^qPqMTz3y<~YcxxTqw=LUKN z+*p83qr8Dw)m*D8YtXQU{JzTpL=umh@;x&V)9T3O{Hs+myjoYTquI`)><2XT+(z=x zmW=b;UpO;#p>i}C)kl6$=6dfpP`+$SY3cWPsFIG-bCd(Dw8H{>5SBS#R@MXI&}({U z=yg<#hnzujJozX?ZtBMtbNFW!)=kh8A!VX`!KN7zljQTB)RR^Qj|?n7yUoz!ohKQ` zGtV<~+pZPi{#zJ&zFVjx zN`R%?V}B7nVNkvtLDnvlbMdmz5jhCiLxdxhXF~ zNzFoeV-TOm^ytQZ`E^L$CkMk{>-G2e2TUp=N$6gAufY5@{J2kkWT_s(DfMj3(p{j_ z7xEr@cbEX9H|g0

y-Wk0EgRfXuG9Oe6-K^w~k2Kd@QvPnwj- z>8dmJKrXR^Nw9;7cMHyWkn*DtMS2{gM6m#cb#@rQ4&4Vzj>~7^Yub5t_9Nlr6Y^$D zd*SO8sS^|>iotO7q&%4Od%Q@GWlo~wBB?U=4fPq;RXxO&%3tz*{wkGES=t9zPs=Qy zO(3~vNOW(`?va8jz%W|ImdDAtyjgO5N8v5Y92YB0uJ-~|46k}Gd;0yUo=vW0lzw$0rESjp*M5!#(wB5++ur%V%9^BZBb6#h34o?|l$f(gkk*jMrcD#^8#0R}yUe_EjCIKLW2R*)cju)d@mk9B1p; z&&No=SDRkk)nZ^6f?$|wO!L{UJ|ty~BADvHrWBpSJ1L5OA&7p#0Pz3D>5JeOiv!LX zCBfhD-^K1Vt&ctgF)^6DyNZF{=oPxZbHHD0;-a_~Cjcha5+i-jZV=hkm3f#{m@fgs z5WHI4XqSL>809zJ0WqF^)e3Mw6Hn#FS?pPuU>Ms5OxiA}7$_T#kOsUU3@}m;xO30X z1Q9Z+FQcFqWY1KHs~j%b&fKE3Xm1F-K3iF0L>*Ou5$QDtFM6?OQHhc3#K$;D{alHI zp`-BH)PJbblad%|#H^K}+B|-l=op52X{ZHSh+d*>=4a}77m4?(*^qrk34y$&cx;wc;|aH*=#D#1C1`49)mi#QZ77pKw4w1>!d^qF8I>*9bc$ugYvc8d)Ttk{aq~V{79+tU$8gUIoN1>ITXiY z&s@;~#(DV~Hf)aD%u`Oav+e1E(0SR4E`Xhv4S=+(N+jZ_80KbW3dcoQTQf1ng9FiH ztzdrmFPI*;aS>3q1Cb)7SJCmh!+Tf=VXLuG3;|-sLyR10Hie`y#T-cSWKKNK3!Aa8 z4rq_X3k$ylW1lKO^G}syu1a8E*#-DxEnpw=atx433Yx|Y$RivgYG;s9=D_`ME-B^* z5cAWR+D$BUYJ-mL={}C(cwy>;{mKT5Y*O&KQqhXncJdtfpO+K=A{5AyA&bi<|b911f|fi)m%|Q76BX^L=YQAJFYr zhL~FzFb*Yfl#aEu#d(+$5X3p6QGp?X01-hRT*POp;0+=|gI+e@6JA6Ky?mStV2BL~ zv`|BUs3Bjdc|tL-^c+FjBGl6fB6|7qE_fo&%brpNo5K#t*_U!)h^$kze|3Qqy?i$Z zvdCP(9t;`C=*CO8bwG;vEP}IwP5PWch~kNU^n=GgD!VMbrG^|s5Lz@=plBilBIBgMfd7c~!9Faa;H51hlk zWyy9;HU@2rfS>vb{dum}@UgO$;geVnh)=92_Js8Q93K)(c9ttweVt+nBx|25Sot|n zC!Q;>s0_kr5>Z}3j|Bg_*b8<*lNbUy9`FML9`%qQRaHNCTBlp7E?nyoWqai z_+SP-pe@3tbH}=v6y1t^=cTr^9Ru5jUXSzy|L&Scz89jRR`{s4l8Pddf+Q;AU?{ky z7E(g|c>xv{sBW?th4OZoYWAKY^y0Og+u%9Dss``)sy_J@REC?k1i&u`UV>3N?_52I z63X#N1o23kodu?rIU@vY?fiE@lD7yqA85LbdPY#IIUJhC3@FSEfdq&^nhI&7u-jQR zT#dyR1jVGGxC3EeZS{onF$fTlRF$OHQ4Ll*heOBu*x+^mlFUa37KuHsb}EEF41A(^ zlIQhRyu)^E!0KlXMj{ACinAO9tM$Ao_9zy@H(X7idtL4Oo@!^U`G4(DRd zMGR6P`${S`QTLh%2ILB{SK$)7S@PwdJ=>}8vmR4SetYD{0ZU>#F3Qz(BC>2m)zOb!#SG7Q+))qs)=&jhZ2$Uyei)?rZV?Ybc~AawwUZuFf)hK8QG4Lp~U$&aoU8hF6L>augJSQ2CtD z2)sukl+oeSJjnKuYNXlNW9D%sQ~lQJ)R2-1>J;;Sm3}5|C#thmYr2#%X%zU$=$-CQ zQw!X@nhHC7)l_&~fVX;2*W*7iIjRhI9;5QyJb=M}{cN?^X*Yp@5G1#nA%-C6sJFXq zVbSdeh%uTBd)~m?^QJ!{x_oX9NAH=Yh@GeyIAvzB4dol0?!QePV zv(UdtuTAPA-UueQKU6~LKc4;2wqEsV}V*jK0AS7z_ z=jhYC&SIo;$`3v~i$mXT)(S6;1rN^kJR3FS`pzBLHhinrd!T55D@FrV9y_FU5%cLKZW}ELE3W7 zgLHbPeq-J)w){#DBuS#yNCe+?4n5oyi03{29AZC@&%oQe&txrHwuvmI2op8`O=1q2 z>!D@ZcSHctVb0$-e%c&At`CUwsmHC_h}H&ho_~+lzOspo0)8wr3nT@JTBZmIaatu3 zQdO(Y|AbD}wUz88+^en)WD5a;c!t6!HMHmagI7(>+ZDfKCehUFjns=CfJmP4@Hk1^ z$+f7Ys#XCGT@Z)yT&PB}w!wCEtV?|}S-WUcxgo!y5=4m6kSD3&&gZrbUMRKwT9HySR9XopnBSE=N>FAQh?%m0C7F9(j?9o#=L^0IlKrP z5($X%x#LCwX_u}YK_z=fkA~pav~`yLfL~`#fzKOiCX^9%mG4BsJkcUX1iY(fV?Jo2 zSq{q_gVbPw&-(d@Z&6^NiW#q^tP3Ivxi*_h}mV zwAK!|Gea>xG+!RT`SomvN0~AJv%6(&v=?$qC-w&r`x6QOwADUYpqL$|*jdVjftw$Ys`%;Ggt+);aK3l*$mZp>Nh|9?aW z-LPh0X4XwJ@7-CfM4?1mpMz+z6bc9lDu+uww9PM}0n3)2HWs~MZZGX4W@^yW1JRlf zs^TSb!B{lR8=jKAfy(Lw6m7A6JNgnR1_%%X6vpL~-0ua&VDKIae)_M7ptl6O+6+W# zva}S}x3EE9OYVB|e^Jp=sQMtA_xJZ@zCLUAuCf;;osxj1;k1h4*DSo`oW6Yu^&r+ z0|3E3&)t}ZPFdQQ{NO^fQC^N2fzTMRpN|k9tJxI@7$|)xrspek*VLllo{Ab;(i5|W zov&-}T0WU{oTjzp21eFR*G3AE+dxv`QVTTk4FoSpA2*O@k?k|JEiVnlIB3rOV>n@LcU*Jk*dLh87;qr@p>J%l6<79DnUDL$EJ~$j@axAb%wyPtq1k z@tkz}kqp_ZE%suoi4(Hma3RJf3ZDUl&kTiGpJ15(AQvw|&H#F)r7>9Q@A?wu>WuKD z_g7jMTe?P58b3I&2Av(Inx5psG3|*-?8TGXMthEoX{SFlE0h=zlm(JNS`UYlZl|@+ z?BN#yH=(4~w^~2*7f#g1;r)x+KJEe$bQ1!NiHu!{@A1@j?nbGzgs)yDjJ_$Cw3F4i z9A!y=%K!3|b>I@*D(ih1VmA&D^%4&^=ixfde09r<2Lgg%LU>8*>8GEwB61KxORQqk zWKttoZ*4zf#J$vp2tSnC=_p+#heP!DIc9^r7Nf_QY3i5N@OYj##@dHBA{~X%O(BJS5JnNnsOy)P&mod0=9K^KIOKdvza~u6_6Ku_3Ei7pd z?xGJ-L(r1KDco0-hTrX$M0GUqL?>1AFvX)KwbUt&CqQ!Ii~Il~(ZQHvNn zhDCKmp(G)B)J^s{AXkR0FXTGlb4VV-i@Ls>^|MT!u)|{hD6R)Qk3o5$F2uo_Z_uzG zZO2U=lJSK}pXh%%B@sw#x1ViaXYJHCaSwxlk~p4Y4=U0Ja#xIik`hi#uM|XRHRy@? zPg*#pK8s3aGefwkX+iLJsT>DmcI&(hf-m=I_$#<#eTlU%IZ&)`G+i5;B7ZM0B7f}H a&7Y2tj$h~who5y&U&qgY1RvM82>%1K4BHw2 delta 14236 zcmaJ|d3a4%*MIhzZ<3oyMwxC5F%v_Tm@ZYNl&ID$qODQ{ZBcWgv_+K5A&W{R=9yH~ zZevK4qDrKO(h^c?s1hmqwra??_TG00ulM`MlY7qIYp;2&y?#g0!|DY~s^>P9^7>2L zg3YExcOTKVpfDvGpFIQp8 z8zj<B%@XQ+M6oG zW?tRRgNd0nfgR}!J*KO{pwCht=M`;@ATGIlq0xFR%sjt63;b=E zrN(UagJmD0)f{z6EX;AJgO@BvBd6_a3@eVKPoy408SHygnX;Ku+M2Bo`NLs<5(>!^ zRX?+EcOP^6zBO*~Tne7eWHhH2u6CMvlsW2DEvUMrl62WK)Ch>oz$d8Uyy62v1O}`@ z7Lix8?RRUSn<?=xI_ zS(G0MUYTlpu)V`+*TDrLX3^0cRBRAzFC&4_u7=?S$!{n~V|0XB{KFZf83cuucns@rgmV|Ku zxiYaq;w|-6X!sfl23xM`1@?c`NEj4C-Zm3XUt>0Qud14tdM1-^%(I_eOSt{*XB+3W z`gtP(-EFD*gO}@P7TQxdw@#R;aecrt@i8Avmyv-A(4)f7(8@P_05m*<>+mrra?@WAQ z-j`}09#_dFBh19>Te$OZEg<<%rr)27 zTyif2yCGT3#NChExUSHB|!4WA|oy^x0*P>YbN;-=Ajzp|iqp0*orNKmhdL zpjLrgLZacjy&m=#ntml=ux_8a$R$vwuAX>ZYqan{BfVRqYow5+nLP2K96P^_WzwD&Ef(k$GNbJ!MkUbI81-9!2m~D`C_l z%xn$QF_R?UiH58nJTf{Lq=RatK8Ktt#0#Djp_h~&S68{Dbne0DtDku^ z-H&836;Vu7vm}qQkkyrN?6K+(`Bex{H*Q<5gf3l)AKa`>LLfN-OJBxmHJJ&CKF59O zyqWl`#yIhY)=!XvS13|&$1A6t`;)rW#XVN326RC<8SU0LXLl7u{VXIvx>i<;g6Z){ zGVM z3!#1uh`mxW;~Um>Cqd9EhBSdGjWEL9*^Ix;BAPE0$Kxp_i9s;P(0pLf3yKfTZcMrm zbN+yF8X3U8qqpEiHdZFT)KWXkWT3baii z+0Sr?&vO`M)KO!z)z7a1#!f^f9ga%Gay0EvXMXeOXzE?`b*EEOOmLIA-)tERgaJaqw0XiU&$jkTw}zuk?AESO9-Gl?Uhd>W|@{?jmbZJC9xnmkUAg=16DRhdg3ZmRWVTf4&8rZQ@bNr!8{ za&P_K!zL34{4H_}OuJ}AQ+q0DBZ2|>(ZF}O9tB4m%hjOmr+9+nU#SX>m_Y)>5|q6< zb+piO3F~~q!7&K~#&hK7Gmz!QvnsJ%2h2~+CM7`dNNosoZ!QIaZ5z><)-JneiCC5C$LAf+fmX$7Qf4ji9y#NCudX- z<)70uo0}e-KSjRq?1j%B8v)>};C-cZhBh6hpOLK4>kRpd7MvsKxa?f|&v`Oj;*Bag z7kLD?t17{;;u5(cLT}%8C6*p9B`negXfaB;FmuL<-I%D zCn+(I)>7^YHy$X_5TB%B9=xSR!h4Igesu3`6q`i>{k)v?)|oeiP|H80h2rvg4ce{L z?n3buB!eFNx|z5>5MIp3_8~qTucX!rRJ-*FHVe!f9YZAzEeD;#@kP9#&Ke!RXa)_dBdQa=rG$xW7~J@iWUfCyGefZ zMh^)jEMLb9Bjff!$pE>Y9s=2|jA#fRh-PKix-SMw z4+Yb^2T3b9LxbDPR=99S>qv`|q%Hi&5So)LnJ9gd976lOCD|l*60Qb^-a*xl?kWdS zbhDj`ry$XJBbZ9qECgDOk`AD8DrHwXCPiw-O$?=vwP1Q-tmNff%&Hk}_<__*6QR0( zqO@Eb=!%*MIPsTcr8_4{4_K%N!Dl~Wp&Mk;-=gi(q&WmOEyWDkwoGb6>wYX9A>1#@ zPRTK}^(^U%#4Hv8gZj(W=%~*myhE}+1xrN~eRYmh;N*gzl{-$t2CFR=k-Z^GW4b0? z8sTJWuqmmusC?n^ zd$JE4T#NY&CALE0&r%V8{-C>F1k}J zFG~aA;7{mdHSUwDkk&v~XzCRVW1)WDW;kx)ar5J`N3~ydhkO9E?xT-lvnA1s@I{!< zmvfa&zb*|RV4X{L0=@`?ihlYM$m)V&nmtSPg&+FsuRv;hsRer*XZG0kg|nx+C4J@W zsqi^uO(t1_kJ3w04aoi-19|o!J(KIG;~gp7UGh~ZAEt-W73ES@2^kQ!h4f^-DU|k> zJWhx{@V68J+wMuMSqq}0?n`}~uI=))M7CtNNDv1bpp%XKN6JJ+bDAn(t5Xw_8)-El zEl!T+{T3~JAT4pWi>w<$RuTz4@mRXW`D4A7kHUHOSL8GTTQ6c;(5{W9(%q_zjU~JK zGGAPhB!}|e4JyXyZ?ZmX82I0nRQT6|manLW$;#W1#RZnNU5u9+Q066n0%;?#@|M1@ z2h;f$InYf~ETOohFPE~iBKBD2Y?oLLmin&K5E3e(UsZvijw(-)~&f+(efj=;89RK2kl&5 z3l$epTMmMTRtwHC{){jcGu(lk`h;mQKCa|=g zyyNM_l*pb1!-M*Aj@xIBFoGF}v7ZZ{iDmp^Lm6ib?2h7pH-K(+=#mT-11dG1$6&!c zk}24`-K>&cD7N#dV#2~41RiDRpF?Vh{4UID@hl^^UB=|?Z!1V0PF|yU6To=|9?iI+ zS*>N>B?p;>#a0-*!tjO)A0<|_Ca%O+ESj$shcSuY$kaa;?N5&*iVu_>mGGu)mzTrw zqf#(UZZFSQ+(J`!t$*;=Q z2sAj4p|E_NULA6Vpz)`k#E`;gjp1Hbc_2&L;Slp13Vx(J8uD==I#mDl`V@2dgb@1b zYjR&>$vnIy@_HZN2gBo~?fMe)d}0XA>Mie(L?L2tDfE+PW8nRa)$&oE9tr~n$fraV zd}E-TWah<%K(|4%K}!e9Q&drdX1pa2F}p?u!`^&70{clTZS=N0TH-J1fuV9kRX75t z-*TjU-+;24x(4qb(PP9!6(?=9|2TQt|9Y{Oz_wI55VlU1b*McN>*-I^<<0(ZB|{FS z8y3hjz1g1;(0jT3HiSQr+d%v(OfTMR`KvH^GwLMq651 zPkC3g^02*$;sq0$Dr}a=hc6b|wVCpzNDJ&<_QbCH!sfvSyTMuM8_ktOo_E-s#saw1 zhnBpkY;yLaMG1;GeBDx6!ugx|wt=EP#v;8eLD>L(Qxz-4m|H6I_K-%jQ(h3XEW?dR zHc}`Nn&n;BSM8OXqNDmD#lS-SfrRa+t1pI>Uf2zFZHG6>cAb@Rp5_#=ISrw57o~|V zy4~pUNSZrc+627aeEkno)Kt9Pq>VS;7J_mi;AKix>i>cAvLY0MTMl@V29&?AD0IO` z%CCNmf-7fgFgV+&38@!T9SE zAqB1=9~>#zfgujoz_YVy5|d}iX!sV`W;43$h}M=p>#g}({KeQ;5pgKRwj2O zw0nftgqb)U9*$G5VzO1Edx!w=p&1HS9)@ z8C}RegL8p~rxn&<1i^s=O|u60cF=aOqd&k_f%uE-UMJY;C)u?1OqqnCbgnG1wr$3j2Oj)?%fI0>@S5 zTkcA=c5A_qeodjm6}MegbQp78S;{+_dDo0YsF+PO26}mdf#%;(e&jtn-CnL-m7tRq z{Wkxe;&57Y?0=M(txOJDDyuWzhlkw|yKu}_Ifnc#44R;B5YG}V8pnq-Bq;SUnnBD( zJrZ(`8`GdQ%K{a1am$6tD)zpNLg8YZR=AMhTacb0o;AgJ`aXF0k~qU^`mKEpaxI!a z_|3%aIQ9&IxOr+5$e6AU7R&HhgCZi_g}}(Vu5Hi9e<_ZbZA6@f5?E>B$0~7dv){uG zoS{_{}RC$IbvY%Lbnu6*>K-~(T;`uc#3ozI*&jRo4P^Xg&q+Z%)6t~PXuxq z(pz~2PWBnt>pX?=`W(2IBHX*m$v!*F^#AP{Y_}hfIS2%%;yA*I>rShrTU651;4Z{Z zSEJb+KEN#8C*)z4wsI7mB8qOu3zXA%f4Bu`201fue2>7cb4ziCdWIjoDBlYPbilde zB~a4^E3FT$6lN;@S*-?_($qi(?y`fU%)~4O9xp>G-25mJ$9UPhF!sHIH#AtzUON4c z&!Z9OeF#&};)LpAoS{M8IKzv!sbRcCz_CJI>wdEcF4QrGL%&AGdGOCfXFRbJF_x(- zRTFxyRYRf8UuY=?OhRIB5@T*TiN3{-iNB?ecoxHLa~^@*#FtR23msSnOGgy`$z(Os zEI2(_=md8qpw5$_ZF8J%7q7+xVqU_O&)`N5yPNqxIS)XuL*V%<^+PCFr>+y*UC*Kr zFc=GkWw_nGLEQ|kFXPmu_?#MNrjfQ#Eud0Yeh1%rWMz*e;;V!-Rfa~5^%JB&&w zFhvwN799j*m*Tei;!mc-cg@6`?HPME2I>ZyE&p(uk%>xCeB8zp0CP6{cH_8^nk)GU zRW}KtaPcItAH@{{-*vd9u|ctn_OR!x)XlZ$S@ct2?+mxnV&T<2Y7uN{i-tyi@&3D4 zJ?Pe-bN|?V>Y+-{CKce%dE;6cLc5Q_Tn3%6pv919P-qJu`A!~De^{h6(iKdV9@ifJ*}84S;?L8RE@ z9hF6R3NNmd9~nFx&XC*TaH2&0&coXO!i{eTDOnIwgooEv3Q9LEjsiyl*ERycPJ5wZ zngb@rI2Y!NY&gBo!)Z7JeVzQV6v+W!K)}}`RJ6STSpj>3Dn+C7oTu>L!B6|2bBPND z=HvsxHpH-iqcLW!`oF6Oo@;o-f}MTKGkSI*!a@X!oW?)jAP56{lB7pW9L+ny;H>Ae zif>m4?Xc{}l0QrI92vg!T<1V6Skz348$z7CxbrWAg0bu`P_!?(oEdAh`V^tyB+(d9 ze_e~hDH+BD#XUZr?RBBRq5FgXTD1!QHVy-&A?sG+0l(8i$kiWlQiO;b!)kw)1~AMmo&3YRiPGnFqPhVnDRG8a!#KTVSoPwnFo=Sfohhg zq^Y)skqD%S3R{SH_@8E)(~sPodN5$HHP=c69Xz!zGQi?WSU-hCFu-?+B-+WzLFPs#i-P-g*FU^!7wYRp8{SyWr zK$ri#wT{fEEBk6&om`-qp9OL$0=d{xySjOf-tHFq8QWiL4*r9*uRU++<3ZYwUT}Vj z)`G*Y6yewD1on;4b_qqh1gQh%NtzBtBXDZ4;8Uysw!P>hxG$WEDG*nRi0dke4jH3? zL5+!8pzL-Ms~$nSPSJ*{Vhcj4`{||`+CV~Qexgm0sPAm8ngsT_`27Sk4rE;-3Gi~d z2J}|C7Kb-Wrmztv;?xL+eh1C%dJwI*SR2RVDi9FcY)Svk&|cE$YgyX&vd2j%f}7j8 zwgSO#wN}K)ILC5|uX1(@}Cnt005fFNegIyc(Mijpe z-K=nfhS|hb@z|2u?Cl=m2+TrH1CphiG{PASgu}9CrT*Kr3V$IW!zH1>qK80Pfp!am zI%`20`!(P%DFT$BL-P0NVtz-pjr7P-Z9bvz9@D;5q1H)l9W6PjO;_p2bDFO^IAh~* z^iuYcl+H<*-HLo6tE1#YlYY_qTbLIFLifv>AN~D`)*n^Llxw0tGjO zmj-f-5D+7LMeuq5cNCt*Fa7$ahS$!d+ZbM)F5-a^(5_6|B0?kJj)o{I%N{50XuF6- zBq58X8YUa*7!=LHqsLTW{LUVNuntdFtM6)WQsbUhf>C;3+X|1~#5HI`4IX|i`_P~M z(i+J$_<{B+fieGTYhAvU+zdYqNK26%wCJI>1I^0(>F`*z3Qs!-5v~@ZZG58P*Hrst zeFYt*=<5|`=Rl7CQN;i7d*g&Ph9B6jkRZC%OK(Q#pWb>iiN4^gk9NkFI|~7#eOq9G zzrI6|9u}bAQe6x!*pFCeYN*~pI4ivpu1{6PVTkax5P9L-7`?6zzwFk-XizQvuuT7` ztG_AH#0GkbLXS7rLs4pG9z^wGU~eZZt_@r0%jnq_`lo(E;i;W_6 z%$p)6hYDc|qLcdSn1vs=)`OryJIn@8+F}sYuZurZ2VsdR@29U4I^WyHFkrfY95EET3B-eu}Qbroku+pE7uqk8s>| z;e<+4@UR_jfd$Na4ej8%|Xn~ zTl((`)m9c9m)&FKG6f5DZADJ1#OGZ_&V+hxFg+p`7Ygx*vB%={y~pTF1$~zQ4aIj= zh|L+?TlvX#p2pt}yIJBd=6X(|=^yGn+yVc7Ei9Wytn#zfn4vZ}Tpq7KcIG*r_YjE- z#!nX9vd2w!Itu(gS-&l;=#tPX5JZN>LD#AJF`-c`g93}Ah{eru(sl_J%1*^^Kz5*d zxa{3h(d9;#86nWDCf=k!or+FlAFW5j3z2d^*q5gB>@TW2#U|pYQy(6J&!BI%* zF_YQXZ81dNHdhwv?a%b(%vesR!q*-oZo2&c_|1j+`U8e5@8P_a^rf`=&x-fb->{?NL)Y=Lh3Fk-lBY#Dmy(|#RB zA7|*TogE4~a*ohAQmqcgP`ChVzp@lm%3w2cJJv6=xh-Q2vg8YJJ?4T=j<60fMxUqqE*-cH19NNFB zG1yr{5gj@t5+*`Hp(3GQX=9vHEA^%RsfeC2Xb2D*>ME+I>;F4gU`!_Z=iIrHH#j=t zP@7eL8uzl%kw;V@MWpDQ;n4;lG_;r3Oxi8c=&8}Iy^VJ@_;4U9x9mkFgw44e;4K+J zyONA?;u^xnhyo3whz9lIkrhbsnl=|d2zkp`&y@lQ`OJa0H(__x`uUy42x`rI) zhF_*bdI?4bhh8$LViy_H4DV2^Pp8X2G5%5nJ;Yw>^4e`)I~EF)ql{R%)X)fmw&_Mc zp62P^bmNI=Uns&cn`KzK$k<4Gn#RxGLac7t_@}$&7&ip|Off=fpB!VEh23`p;lx5i zh0Ngy3&(FlAc7SbiU><@snV_#Xgm$0ci2*8y6}*POYzHr9@~sB;K4RygiAulG6_Gn z+iv_Vv8ziUUAMzf@&9W@;&Ak0yx|K!<{2RNH~7T}pv!j}{hb#@yle$Z(E4j*gKz+^ zZ;U%$)N;_MaJm74DSf@2=a@wp3<-spU-c788q*a1uL!(#P()HN&>q!6zrK3P*e`1A zvD3!aE(WuQ7`5QiS)&Q0oK})VFd&rDfI#k9;~P5ZoN-?D&=a>91WI0qm{;)HW^bZZ z1KTBI66X;LE*bv{9lLN7oRc%0)Nusi@aSKSb)pjA{MB$ahPjuGPvLhzRC?blMj_Px z&Dbo|;lfUQA?&J=S;^b|zCiT;kE=#1_dkj-$|}(f(Zt(EF`@ByjGC2l1t4CsXxClC U#|!p9GD5*yQGp(PWGLkS02YkeZU6uP diff --git a/bundle/package.json b/bundle/package.json index 8915665d..4dec60ee 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -64,6 +64,7 @@ "@fosscord/gateway": "file:../gateway", "@sentry/node": "^6.16.1", "@sentry/tracing": "^6.16.1", + "@yukikaze-bot/erlpack": "^1.0.1", "ajv": "8.6.2", "ajv-formats": "^2.1.1", "amqplib": "^0.8.0", diff --git a/gateway/src/schema/VoiceStateUpdateSchema.ts b/gateway/src/schema/VoiceStateUpdateSchema.ts index 9efa191e..c046600d 100644 --- a/gateway/src/schema/VoiceStateUpdateSchema.ts +++ b/gateway/src/schema/VoiceStateUpdateSchema.ts @@ -3,7 +3,7 @@ export const VoiceStateUpdateSchema = { $channel_id: String, self_mute: Boolean, self_deaf: Boolean, - self_video: Boolean, + $self_video: Boolean, //required in docs but bots don't always send it }; export interface VoiceStateUpdateSchema { @@ -11,5 +11,5 @@ export interface VoiceStateUpdateSchema { channel_id?: string; self_mute: boolean; self_deaf: boolean; - self_video: boolean; -} + self_video?: boolean; +} \ No newline at end of file diff --git a/webrtc/.vscode/launch.json b/webrtc/.vscode/launch.json index 92403164..49584172 100644 --- a/webrtc/.vscode/launch.json +++ b/webrtc/.vscode/launch.json @@ -17,7 +17,9 @@ ], "cwd": "${workspaceRoot}", "protocol": "inspector", - "internalConsoleOptions": "openOnSessionStart" + "internalConsoleOptions": "openOnSessionStart", + "sourceMaps": true, + "resolveSourceMapLocations": null, } ] } \ No newline at end of file diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index dcbf216a..0145a221 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -1,7 +1,7 @@ import { Server as WebSocketServer } from "ws"; -import { WebSocket, Payload, CLOSECODES } from "@fosscord/gateway"; +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { Config, initDatabase } from "@fosscord/util"; -import OPCodeHandlers from "./opcodes"; +import OPCodeHandlers, { Payload } from "./opcodes"; import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; @@ -26,8 +26,16 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); + console.log(payload); + if (OPCodeHandlers[payload.op]) - await OPCodeHandlers[payload.op].call(this, socket, payload); + try { + await OPCodeHandlers[payload.op].call(this, socket, payload); + } + catch (e) { + console.error(e); + socket.close(CLOSECODES.Unknown_error); + } else { console.error(`Unimplemented`, payload); socket.close(CLOSECODES.Unknown_opcode); diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index 82f327be..e965e3de 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -1,9 +1,38 @@ -import { WebSocket } from "@fosscord/gateway"; +import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { Payload } from "./index"; -import { VoiceOPCodes } from "@fosscord/util"; +import { VoiceOPCodes, Session, User, Guild } from "@fosscord/util"; import { Server } from "../Server"; -export async function onIdentify(this: Server, socket: WebSocket, data: Payload) { +export interface IdentifyPayload extends Payload { + d: { + server_id: string, //guild id + session_id: string, //gateway session + streams: Array<{ + type: string, + rid: string, //number + quality: number, + }>, + token: string, //voice_states token + user_id: string, + video: boolean, + }; +} + +export async function onIdentify(this: Server, socket: WebSocket, data: IdentifyPayload) { + + const session = await Session.findOneOrFail( + { session_id: data.d.session_id, }, + { + where: { user_id: data.d.user_id }, + relations: ["user"] + } + ); + const user = session.user; + const guild = await Guild.findOneOrFail({ id: data.d.server_id }); + + if (!guild.members.find(x => x.id === user.id)) + return socket.close(CLOSECODES.Invalid_intent); + var transport = await this.mediasoupRouters[0].createWebRtcTransport({ listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, @@ -40,15 +69,17 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Payload) socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { - streams: [], + streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))], ssrc: 1, ip: transport.iceCandidates[0].ip, port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", - // "xsalsa20_poly1305", - // "xsalsa20_poly1305_suffix", - // "xsalsa20_poly1305_lite", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" ], heartbeat_interval: 1, experiments: [], diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 36d30e7d..9b1eb270 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -3,9 +3,9 @@ import { VoiceOPCodes } from "@fosscord/util"; export interface Payload { op: number; - d?: any; - s?: number; - t?: string; + d: any; + s: number; + t: string; } import { onIdentify } from "./Identify"; From b529a372644fab22b34007353cc69caf3de5331c Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 19:15:33 +1100 Subject: [PATCH 07/10] augh --- bundle/package.json | 2 +- util/src/util/Constants.ts | 5 +- webrtc/src/Server.ts | 12 +++-- webrtc/src/opcodes/Heartbeat.ts | 4 +- webrtc/src/opcodes/Identify.ts | 37 ++++++++++++--- webrtc/src/opcodes/SelectProtocol.ts | 68 ++++++++++++++++++++++------ webrtc/src/opcodes/Version.ts | 14 ++++++ webrtc/src/opcodes/index.ts | 3 ++ webrtc/src/start.ts | 6 +-- webrtc/src/util/Heartbeat.ts | 21 +++++---- 10 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 webrtc/src/opcodes/Version.ts diff --git a/bundle/package.json b/bundle/package.json index 0b00b325..aedd963b 100644 --- a/bundle/package.json +++ b/bundle/package.json @@ -112,4 +112,4 @@ "typescript-json-schema": "^0.50.1", "ws": "^7.4.2" } -} +} \ No newline at end of file diff --git a/util/src/util/Constants.ts b/util/src/util/Constants.ts index d5315767..42a2c274 100644 --- a/util/src/util/Constants.ts +++ b/util/src/util/Constants.ts @@ -77,8 +77,9 @@ export const VoiceOPCodes = { RESUME: 7, HELLO: 8, RESUMED: 9, - CLIENT_CONNECT: 12, - CLIENT_DISCONNECT: 13, + CLIENT_CONNECT: 12, // incorrect, op 12 is probably used for video + CLIENT_DISCONNECT: 13, // incorrect + VERSION: 16, //not documented }; export const Events = { diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 0145a221..1d18d6d1 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,6 +6,8 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; +import Net from "net"; + var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -13,7 +15,7 @@ export class Server { public ws: WebSocketServer; public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; - public mediasoupTransports: MediasoupTypes.Transport[] = []; + public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; constructor() { this.ws = new WebSocketServer({ @@ -26,7 +28,7 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - console.log(payload); + // console.log(payload); if (OPCodeHandlers[payload.op]) try { @@ -68,9 +70,13 @@ export class Server { this.mediasoupRouters.push(router); - router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => { + router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); + transport.observer.on("sctpstatechange", (state) => { + console.log(state) + }); + await transport.enableTraceEvent(); transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { diff --git a/webrtc/src/opcodes/Heartbeat.ts b/webrtc/src/opcodes/Heartbeat.ts index 06d6bcb1..47f33f76 100644 --- a/webrtc/src/opcodes/Heartbeat.ts +++ b/webrtc/src/opcodes/Heartbeat.ts @@ -1,8 +1,8 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; -import { setHeartbeat } from "./../util"; +import { setHeartbeat } from "../util"; import { Server } from "../Server" export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) { - await setHeartbeat(socket); + await setHeartbeat(socket, data.d); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index e965e3de..d7da5c7c 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -28,12 +28,12 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } ); const user = session.user; - const guild = await Guild.findOneOrFail({ id: data.d.server_id }); + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); if (!guild.members.find(x => x.id === user.id)) return socket.close(CLOSECODES.Invalid_intent); - var transport = await this.mediasoupRouters[0].createWebRtcTransport({ + var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], enableUdp: true, enableTcp: true, @@ -66,13 +66,39 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify } */ + + + /* + { + "streams": [ + { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } + ], + "ssrc": 129860, + "port": 50003, + "modes": [ + "aead_aes256_gcm_rtpsize", + "aead_aes256_gcm", + "xsalsa20_poly1305_lite_rtpsize", + "xsalsa20_poly1305_lite", + "xsalsa20_poly1305_suffix", + "xsalsa20_poly1305" + ], + "ip": "109.200.213.251", + "experiments": [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ]; + }; + */ + socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { - streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))], - ssrc: 1, + streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], + ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: transport.iceCandidates[0].port, + port: "50001", modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -81,7 +107,6 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - heartbeat_interval: 1, experiments: [], }, })); diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index 36527a8b..a957e14f 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -87,42 +87,82 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa })), */ + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") + if (!test_hasMadeProducer) { const producer = await transport.produce({ kind: "audio", rtpParameters: { mid: "audio", codecs: [{ - clockRate: 48000, - payloadType: 111, - mimeType: "audio/opus", - channels: 2, + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, }], headerExtensions: res.ext?.map(x => ({ id: x.value, uri: x.uri, - })) + })), }, paused: false, }); - + const consumer = await transport.consume({ producerId: producer.id, - paused: false, + paused: true, rtpCapabilities, - }) - + }); + test_hasMadeProducer = true; } + /* server sends sdp: + + m=audio 50021 ICE/SDP //same port as sent in READY + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.213.132 //same IP as sent in READY + a=rtcp:50021 //same port? + a=ice-ufrag:rTmX + a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT + + */ + + + var test = { + "video_codec": "H264", + "sdp": ` + m=audio 50011 ICE/SDP\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + c=IN IP4 109.200.214.156\n + a=rtcp:50011\n + a=ice-ufrag:d0aZ\n + a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n + a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, + "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", + "audio_codec": "opus" + }; + + socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: data.d.codecs.find((x: any) => x.type === "video").name, - secret_key: new Array(32).fill(null).map(x => Math.random() * 256), - mode: "xsalsa20_poly1305", - media_session_id: this.mediasoupTransports[0].id, - audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name, + video_codec: videoCodec?.substring(6) || undefined, + // mode: "xsalsa20_poly1305", + media_session_id: transport.id, + audio_codec: audioCodec?.mimeType.substring(6), + sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` + + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + + `a=ice-pwd:${transport.iceParameters.password}\n` + + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` + + `a=candidate:1 1 ${transport.iceCandidates[0].protocol} ${transport.iceCandidates[0].priority} ${transport.iceCandidates[0].ip} ${transport.iceCandidates[0].port} typ ${transport.iceCandidates[0].type}` } })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Version.ts b/webrtc/src/opcodes/Version.ts new file mode 100644 index 00000000..0ea6eb4d --- /dev/null +++ b/webrtc/src/opcodes/Version.ts @@ -0,0 +1,14 @@ +import { WebSocket } from "@fosscord/gateway"; +import { Payload } from "./index"; +import { setHeartbeat } from "../util"; +import { Server } from "../Server" + +export async function onVersion(this: Server, socket: WebSocket, data: Payload) { + socket.send(JSON.stringify({ + op: 16, + d: { + voice: "0.8.31", //version numbers? + rtc_worker: "0.3.18", + } + })) +} \ No newline at end of file diff --git a/webrtc/src/opcodes/index.ts b/webrtc/src/opcodes/index.ts index 9b1eb270..d0f40bc2 100644 --- a/webrtc/src/opcodes/index.ts +++ b/webrtc/src/opcodes/index.ts @@ -15,6 +15,8 @@ import { onSpeaking } from "./Speaking"; import { onResume } from "./Resume"; import { onConnect } from "./Connect"; +import { onVersion } from "./Version"; + export type OPCodeHandler = (this: WebSocket, data: Payload) => any; export default { @@ -34,4 +36,5 @@ export default { //op 13? //op 15? //op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"} + [VoiceOPCodes.VERSION]: onVersion, }; \ No newline at end of file diff --git a/webrtc/src/start.ts b/webrtc/src/start.ts index 299bfce8..98f06ad5 100644 --- a/webrtc/src/start.ts +++ b/webrtc/src/start.ts @@ -1,10 +1,10 @@ +//testing +process.env.DATABASE = "../bundle/database.db"; + import { config } from "dotenv"; config(); import { Server } from "./Server"; -//testing -process.env.DATABASE = "../bundle/database.db"; - const server = new Server(); server.listen(); \ No newline at end of file diff --git a/webrtc/src/util/Heartbeat.ts b/webrtc/src/util/Heartbeat.ts index 7b5ed9cd..8c5e3a7a 100644 --- a/webrtc/src/util/Heartbeat.ts +++ b/webrtc/src/util/Heartbeat.ts @@ -1,18 +1,23 @@ import { WebSocket, CLOSECODES } from "@fosscord/gateway"; import { VoiceOPCodes } from "@fosscord/util"; -export async function setHeartbeat(socket: WebSocket) { +export async function setHeartbeat(socket: WebSocket, nonce?: Number) { if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout); socket.heartbeatTimeout = setTimeout(() => { return socket.close(CLOSECODES.Session_timed_out); }, 1000 * 45); - socket.send(JSON.stringify({ - op: VoiceOPCodes.HEARTBEAT_ACK, - d: { - v: 6, - heartbeat_interval: 13750, - } - })); + if (!nonce) { + socket.send(JSON.stringify({ + op: VoiceOPCodes.HELLO, + d: { + v: 5, + heartbeat_interval: 13750, + } + })); + } + else { + socket.send(JSON.stringify({ op: VoiceOPCodes.HEARTBEAT_ACK, d: nonce })); + } } \ No newline at end of file From e731a369e56fc02a667b9b28043ff23ea6c433be Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Mon, 7 Mar 2022 22:57:37 +1100 Subject: [PATCH 08/10] VOICE CONNECTS!!! Dtls stuck on "connecting" state + currently no way to edit/inspect packets received or use own packet format in mediasoup ( fork? ) --- webrtc/src/Server.ts | 23 ++- webrtc/src/opcodes/Connect.ts | 32 ++++- webrtc/src/opcodes/Identify.ts | 63 ++------- webrtc/src/opcodes/Resume.ts | 20 ++- webrtc/src/opcodes/SelectProtocol.ts | 200 ++++++++++++--------------- 5 files changed, 165 insertions(+), 173 deletions(-) diff --git a/webrtc/src/Server.ts b/webrtc/src/Server.ts index 1d18d6d1..42b82c6a 100644 --- a/webrtc/src/Server.ts +++ b/webrtc/src/Server.ts @@ -6,7 +6,7 @@ import { setHeartbeat } from "./util"; import * as mediasoup from "mediasoup"; import { types as MediasoupTypes } from "mediasoup"; -import Net from "net"; +import udp from "dgram"; var port = Number(process.env.PORT); if (isNaN(port)) port = 3004; @@ -16,6 +16,8 @@ export class Server { public mediasoupWorkers: MediasoupTypes.Worker[] = []; public mediasoupRouters: MediasoupTypes.Router[] = []; public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = []; + public mediasoupProducers: MediasoupTypes.Producer[] = []; + public mediasoupConsumers: MediasoupTypes.Consumer[] = []; constructor() { this.ws = new WebSocketServer({ @@ -28,8 +30,6 @@ export class Server { socket.on("message", async (message: string) => { const payload: Payload = JSON.parse(message); - // console.log(payload); - if (OPCodeHandlers[payload.op]) try { await OPCodeHandlers[payload.op].call(this, socket, payload); @@ -44,6 +44,7 @@ export class Server { } }); }); + } async listen(): Promise { @@ -73,18 +74,26 @@ export class Server { router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => { console.log("new transport created [id:%s]", transport.id); - transport.observer.on("sctpstatechange", (state) => { - console.log(state) - }); - await transport.enableTraceEvent(); + transport.on("connect", () => { + console.log("transport connect") + }) + transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => { console.log("new producer created [id:%s]", producer.id); + + this.mediasoupProducers.push(producer); }); transport.observer.on("newconsumer", (consumer: MediasoupTypes.Consumer) => { console.log("new consumer created [id:%s]", consumer.id); + + this.mediasoupConsumers.push(consumer); + + consumer.on("rtp", (rtpPacket) => { + console.log(rtpPacket); + }); }); transport.observer.on("newdataproducer", (dataProducer) => { diff --git a/webrtc/src/opcodes/Connect.ts b/webrtc/src/opcodes/Connect.ts index b312d6f2..1f874a44 100644 --- a/webrtc/src/opcodes/Connect.ts +++ b/webrtc/src/opcodes/Connect.ts @@ -2,8 +2,38 @@ import { WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +/* +Sent by client: + +{ + "op": 12, + "d": { + "audio_ssrc": 0, + "video_ssrc": 0, + "rtx_ssrc": 0, + "streams": [ + { + "type": "video", + "rid": "100", + "ssrc": 0, + "active": false, + "quality": 100, + "rtx_ssrc": 0, + "max_bitrate": 2500000, + "max_framerate": 20, + "max_resolution": { + "type": "fixed", + "width": 1280, + "height": 720 + } + } + ] + } +} +*/ + export async function onConnect(this: Server, socket: WebSocket, data: Payload) { - socket.send(JSON.stringify({ + socket.send(JSON.stringify({ //what is op 15? op: 15, d: { any: 100 } })) diff --git a/webrtc/src/opcodes/Identify.ts b/webrtc/src/opcodes/Identify.ts index d7da5c7c..9baa16e3 100644 --- a/webrtc/src/opcodes/Identify.ts +++ b/webrtc/src/opcodes/Identify.ts @@ -34,71 +34,20 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify return socket.close(CLOSECODES.Invalid_intent); var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({ - listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }], + listenIps: [{ ip: "10.22.64.69" }], enableUdp: true, enableTcp: true, preferUdp: true, + enableSctp: true, }); - /* - //discord proper sends: - { - "streams": [ - { "type": "video", "ssrc": 1311885, "rtx_ssrc": 1311886, "rid": "50", "quality": 50, "active": false }, - { "type": "video", "ssrc": 1311887, "rtx_ssrc": 1311888, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 1311884, - "port": 50008, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.214.158", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ] - } - */ - - - - /* - { - "streams": [ - { "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false } - ], - "ssrc": 129860, - "port": 50003, - "modes": [ - "aead_aes256_gcm_rtpsize", - "aead_aes256_gcm", - "xsalsa20_poly1305_lite_rtpsize", - "xsalsa20_poly1305_lite", - "xsalsa20_poly1305_suffix", - "xsalsa20_poly1305" - ], - "ip": "109.200.213.251", - "experiments": [ - "bwe_conservative_link_estimate", - "bwe_remote_locus_client", - "fixed_keyframe_interval" - ]; - }; - */ - socket.send(JSON.stringify({ op: VoiceOPCodes.READY, d: { streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))], ssrc: Math.floor(Math.random() * 10000), ip: transport.iceCandidates[0].ip, - port: "50001", + port: transport.iceCandidates[0].port, modes: [ "aead_aes256_gcm_rtpsize", "aead_aes256_gcm", @@ -107,7 +56,11 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify "xsalsa20_poly1305_suffix", "xsalsa20_poly1305" ], - experiments: [], + experiments: [ + "bwe_conservative_link_estimate", + "bwe_remote_locus_client", + "fixed_keyframe_interval" + ] }, })); } \ No newline at end of file diff --git a/webrtc/src/opcodes/Resume.ts b/webrtc/src/opcodes/Resume.ts index dcd4f4cd..856b550c 100644 --- a/webrtc/src/opcodes/Resume.ts +++ b/webrtc/src/opcodes/Resume.ts @@ -1,6 +1,24 @@ -import { WebSocket } from "@fosscord/gateway"; +import { CLOSECODES, WebSocket } from "@fosscord/gateway"; import { Payload } from "./index"; import { Server } from "../Server" +import { Guild, Session, VoiceOPCodes } from "@fosscord/util"; export async function onResume(this: Server, socket: WebSocket, data: Payload) { + const session = await Session.findOneOrFail( + { session_id: data.d.session_id, }, + { + where: { user_id: data.d.user_id }, + relations: ["user"] + } + ); + const user = session.user; + const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] }); + + if (!guild.members.find(x => x.id === user.id)) + return socket.close(CLOSECODES.Invalid_intent); + + socket.send(JSON.stringify({ + op: VoiceOPCodes.RESUMED, + d: null, + })) } \ No newline at end of file diff --git a/webrtc/src/opcodes/SelectProtocol.ts b/webrtc/src/opcodes/SelectProtocol.ts index a957e14f..dc9d2b88 100644 --- a/webrtc/src/opcodes/SelectProtocol.ts +++ b/webrtc/src/opcodes/SelectProtocol.ts @@ -6,15 +6,18 @@ import * as mediasoup from "mediasoup"; import { RtpCodecCapability } from "mediasoup/node/lib/RtpParameters"; import * as sdpTransform from 'sdp-transform'; + /* - { - op: 1, - d: { - protocol: "webrtc", - data: " + + Sent by client: +{ + "op": 1, + "d": { + "protocol": "webrtc", + "data": " a=extmap-allow-mixed - a=ice-ufrag:ilWh - a=ice-pwd:Mx7TDnPKXDnTgYWC+qMaqspQ + a=ice-ufrag:vNxb + a=ice-pwd:tZvpbVPYEKcnW0gGRPq0OOnh a=ice-options:trickle a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time @@ -32,43 +35,63 @@ import * as sdpTransform from 'sdp-transform'; a=rtpmap:96 VP8/90000 a=rtpmap:97 rtx/90000 ", - sdp: "same data as in d.data? also not documented by discord", - codecs: [ - { - name: "opus", - type: "audio", - priority: 1000, - payload_type: 111, - rtx_payload_type: null, - }, - { - name: "H264", - type: "video", - priority: 1000, - payload_type: 102, - rtx_payload_type: 121, - }, - { - name: "VP8", - type: "video", - priority: 2000, - payload_type: 96, - rtx_payload_type: 97, - }, - { - name: "VP9", - type: "video", - priority: 3000, - payload_type: 98, - rtx_payload_type: 99, - }, + "codecs": [ + { + "name": "opus", + "type": "audio", + "priority": 1000, + "payload_type": 111, + "rtx_payload_type": null + }, + { + "name": "H264", + "type": "video", + "priority": 1000, + "payload_type": 102, + "rtx_payload_type": 121 + }, + { + "name": "VP8", + "type": "video", + "priority": 2000, + "payload_type": 96, + "rtx_payload_type": 97 + }, + { + "name": "VP9", + "type": "video", + "priority": 3000, + "payload_type": 98, + "rtx_payload_type": 99 + } ], - rtc_connection_id: "b3c8628a-edb5-49ae-b860-ab0d2842b104", - }, + "rtc_connection_id": "3faa0b80-b3e2-4bae-b291-273801fbb7ab" } +} + +Sent by server: + +{ + "op": 4, + "d": { + "video_codec": "H264", + "sdp": " + m=audio 50001 ICE/SDP + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + c=IN IP4 109.200.214.158 + a=rtcp:50001 + a=ice-ufrag:CLzn + a=ice-pwd:qEmIcNwigd07mu46Ok0XCh + a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 + a=candidate:1 1 UDP 4261412862 109.200.214.158 50001 typ host + ", + "media_session_id": "807955cb953e98c5b90704cf048e81ec", + "audio_codec": "opus" + } +} + */ -var test_hasMadeProducer = false; export async function onSelectProtocol(this: Server, socket: WebSocket, data: Payload) { const rtpCapabilities = this.mediasoupRouters[0].rtpCapabilities; @@ -78,87 +101,46 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa const res = sdpTransform.parse(data.d.sdp); - /* - res.media.map(x => x.rtp).flat(1).map(x => ({ - codec: x.codec, - payloadType: x.payload, - clockRate: x.rate as number, - mimeType: `audio/${x.codec}`, + const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video"); + const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio"); + + const producer = this.mediasoupProducers[0] || await transport.produce({ + kind: "audio", + rtpParameters: { + mid: "audio", + codecs: [{ + clockRate: audioCodec!.clockRate, + payloadType: audioCodec!.preferredPayloadType as number, + mimeType: audioCodec!.mimeType, + channels: audioCodec?.channels, + }], + headerExtensions: res.ext?.map(x => ({ + id: x.value, + uri: x.uri, })), - */ + }, + paused: false, + }); - const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType - const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio") - - if (!test_hasMadeProducer) { - const producer = await transport.produce({ - kind: "audio", - rtpParameters: { - mid: "audio", - codecs: [{ - clockRate: audioCodec!.clockRate, - payloadType: audioCodec!.preferredPayloadType as number, - mimeType: audioCodec!.mimeType, - channels: audioCodec?.channels, - }], - headerExtensions: res.ext?.map(x => ({ - id: x.value, - uri: x.uri, - })), - }, - paused: false, - }); - - const consumer = await transport.consume({ - producerId: producer.id, - paused: true, - rtpCapabilities, - }); - - test_hasMadeProducer = true; - } - - /* server sends sdp: - - m=audio 50021 ICE/SDP //same port as sent in READY - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - c=IN IP4 109.200.213.132 //same IP as sent in READY - a=rtcp:50021 //same port? - a=ice-ufrag:rTmX - a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87 - a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host //same IP and PORT - - */ - - - var test = { - "video_codec": "H264", - "sdp": ` - m=audio 50011 ICE/SDP\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - c=IN IP4 109.200.214.156\n - a=rtcp:50011\n - a=ice-ufrag:d0aZ\n - a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n - a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n - a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`, - "media_session_id": "9e18c981687f2de5399edd5cb3f3babf", - "audio_codec": "opus" - }; + console.log("can consume: " + this.mediasoupRouters[0].canConsume({ producerId: producer.id, rtpCapabilities: rtpCapabilities })); + const consumer = this.mediasoupConsumers[0] || await transport.consume({ + producerId: producer.id, + paused: false, + rtpCapabilities, + }); socket.send(JSON.stringify({ op: VoiceOPCodes.SESSION_DESCRIPTION, d: { - video_codec: videoCodec?.substring(6) || undefined, - // mode: "xsalsa20_poly1305", + video_codec: videoCodec?.mimeType?.substring(6) || undefined, + mode: "xsalsa20_poly1305_lite", media_session_id: transport.id, audio_codec: audioCodec?.mimeType.substring(6), sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n` + `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n` + `c=IN IPV4 ${transport.iceCandidates[0].ip}\n` - + `a=rtcp:${transport.iceCandidates[0].port}\n` + + `a=rtcp: ${transport.iceCandidates[0].port}\n` + `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n` + `a=ice-pwd:${transport.iceParameters.password}\n` + `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n` From c79d464186f1b206ae2634201591a91d49edb40c Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:22:02 +1100 Subject: [PATCH 09/10] Changing Member.premium_since back from Date to number fixes an error in the Discord electron client related to rendering premium status. Client throws "Invalid time value", so I'm guessing it's something to do with premium_since not being the date format they want ( allegedly ISO8601, but works with a plain number, so wtf ) --- util/src/entities/Member.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/src/entities/Member.ts b/util/src/entities/Member.ts index 3c5f9db0..b7406881 100644 --- a/util/src/entities/Member.ts +++ b/util/src/entities/Member.ts @@ -86,7 +86,7 @@ export class Member extends BaseClassWithoutId { joined_at: Date; @Column({ nullable: true }) - premium_since?: Date; + premium_since?: number; @Column() deaf: boolean; From f5f007e81fd2a7ab3215f3de743fe1ba040333e9 Mon Sep 17 00:00:00 2001 From: Madeline <46743919+MaddyUnderStars@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:57:20 +1100 Subject: [PATCH 10/10] Added preferred_region optional property of VoiceStateUpdateSchema to allow electron client to connect to voice without crashing --- gateway/src/schema/VoiceStateUpdateSchema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gateway/src/schema/VoiceStateUpdateSchema.ts b/gateway/src/schema/VoiceStateUpdateSchema.ts index c046600d..f6480414 100644 --- a/gateway/src/schema/VoiceStateUpdateSchema.ts +++ b/gateway/src/schema/VoiceStateUpdateSchema.ts @@ -4,6 +4,7 @@ export const VoiceStateUpdateSchema = { self_mute: Boolean, self_deaf: Boolean, $self_video: Boolean, //required in docs but bots don't always send it + $preferred_region: String, }; export interface VoiceStateUpdateSchema { @@ -12,4 +13,5 @@ export interface VoiceStateUpdateSchema { self_mute: boolean; self_deaf: boolean; self_video?: boolean; + preferred_region?: string; } \ No newline at end of file