Further admin api work

extra/admin-api/Spacebar.AdminAPI/Controllers/Media/UserMediaController.cs

@@ -0,0 +1,27 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.EntityFrameworkCore;
+using Spacebar.AdminApi.Models;
+using Spacebar.Db.Contexts;
+using Spacebar.Db.Models;
+using Spacebar.RabbitMqUtilities;
+
+namespace Spacebar.AdminAPI.Controllers.Media;
+
+[ApiController]
+[Route("/media/user")]
+public class UserMediaController(ILogger<UserMediaController> logger, SpacebarDbContext db, RabbitMQService mq, IServiceProvider sp) : ControllerBase {
+
+    [HttpGet("{userId}/attachments")]
+    public async IAsyncEnumerable<Attachment> GetAttachmentsByUser(string userId) {
+        var db2 = sp.CreateScope().ServiceProvider.GetService<SpacebarDbContext>();
+        var attachments = db.Attachments
+            // .IgnoreAutoIncludes()
+            .Where(x => x.Message!.AuthorId == userId)
+            .AsAsyncEnumerable();
+        await foreach (var attachment in attachments) {
+            attachment.Message = await db2.Messages.FindAsync(attachment.MessageId);
+            // attachment.Message.Author = await db2.Users.FindAsync(attachment.Message.AuthorId);
+            yield return attachment;
+        }
+    }
+}

extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs

@@ -146,6 +146,13 @@ public class UserController(ILogger<UserController> logger, SpacebarDbContext db
             yield break;
         }
 
+        user.Data = "{}";
+        user.Deleted = true;
+        user.Disabled = true;
+        user.Rights = 0;
+        db.Users.Update(user);
+        await db.SaveChangesAsync();
+
         var factory = new ConnectionFactory {
             Uri = new Uri("amqp://guest:guest@127.0.0.1/")
         };

extra/admin-api/Spacebar.AdminAPI/Middleware/AuthenticationMiddleware.cs

@@ -2,6 +2,7 @@ using System.IdentityModel.Tokens.Jwt;
 using System.Security.Cryptography;
 using ArcaneLibs.Extensions;
 using Microsoft.IdentityModel.Tokens;
+using Spacebar.AdminAPI.Services;
 using Spacebar.Db.Contexts;
 using Spacebar.Db.Models;
 
@@ -54,7 +55,8 @@ public class AuthenticationMiddleware(RequestDelegate next) {
 
         if (!_userCache.ContainsKey(token)) {
             var db = sp.GetRequiredService<SpacebarDbContext>();
-            user = await db.Users.FindAsync(res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value)
+            var config = sp.GetRequiredService<Configuration>();
+            user = await db.Users.FindAsync(config.OverrideUid ?? res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value)
                    ?? throw new InvalidOperationException();
             _userCache[token] = user;
             _userCacheExpiry[token] = DateTime.Now.AddMinutes(5);

extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs

@@ -7,7 +7,7 @@ using Spacebar.Db.Models;
 
 namespace Spacebar.AdminAPI.Services;
 
-public class AuthenticationService(SpacebarDbContext db) {
+public class AuthenticationService(SpacebarDbContext db, Configuration config) {
     private static Dictionary<string, User> _userCache = new();
     private static Dictionary<string, DateTime> _userCacheExpiry = new();
     
@@ -37,6 +37,6 @@ public class AuthenticationService(SpacebarDbContext db) {
             throw new UnauthorizedAccessException();
         }
 
-        return await db.Users.FindAsync(res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) ?? throw new InvalidOperationException();
+        return await db.Users.FindAsync(config.OverrideUid ?? res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) ?? throw new InvalidOperationException();
     }
 }

extra/admin-api/Spacebar.AdminAPI/Services/Configuration.cs

@@ -4,4 +4,6 @@ public class Configuration {
     public Configuration(IConfiguration configuration) {
         configuration.GetRequiredSection("SpacebarAdminApi").Bind(this);
     }
+    
+    public string? OverrideUid { get; set; }
 }

extra/admin-api/Spacebar.AdminApi.Models/FileMetadataModel.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+
+namespace Spacebar.AdminApi.Models;
+
+public class FileMetadataModel {
+    public string UserId { get; set; } = null!;
+    public string Id { get; set; } = null!;
+    
+    [JsonConverter(typeof(JsonStringEnumConverter<FileUploadType>))]
+    public FileUploadType Type { get; set; }
+
+
+    public enum FileUploadType {
+        Attachment,
+        Avatar,
+        Banner,
+        GuildIcon,
+        GuildSplash,
+        GuildCover,
+        Emoji,
+        Sticker
+    }
+}

extra/admin-api/Spacebar.Db/Models/User.cs

@@ -92,7 +92,7 @@ public partial class User
     [Column("email", TypeName = "character varying")]
     public string? Email { get; set; }
 
-    [Column("flags")]
+    [Column("flags", TypeName = "text")]
     public ulong Flags { get; set; }
 
     [Column("public_flags")]

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor

@@ -24,6 +24,11 @@
                 <span class="bi bi-list-nested-nav-menu" aria-hidden="true"></span> Guilds
             </NavLink>
         </div>
+        <div class="nav-item px-3">
+            <NavLink class="nav-link" href="Media">
+                <span class="bi bi-list-nested-nav-menu" aria-hidden="true"></span> Media
+            </NavLink>
+        </div>
     </nav>
 </div>
 

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Index.razor

@@ -0,0 +1,7 @@
+@page "/Media"
+<h3>Index of /Media</h3>
+<hr/>
+
+@code {
+    
+}

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Users.razor

@@ -0,0 +1,106 @@
+@page "/Media/ByUser"
+@using System.Net.Http.Headers
+@using System.Reflection
+@using Spacebar.AdminApi.Models
+@using Spacebar.AdminAPI.TestClient.Services
+@using ArcaneLibs.Blazor.Components
+@inject Config Config
+@inject ILocalStorageService LocalStorage
+
+<PageTitle>Uploaded media by user</PageTitle>
+
+<details>
+    <summary>Displayed columns</summary>
+    @foreach (var column in DisplayedColumns) {
+        var value = column.Value;
+        <span>
+            <InputCheckbox @bind-Value:get="@(value)" @bind-Value:set="@(b => {
+                                                                           DisplayedColumns[column.Key] = b;
+                                                                           StateHasChanged();
+                                                                       })"/>
+            @column.Key.Name
+        </span>
+        <br/>
+    }
+</details>
+
+<InputSelect @bind-Value="@SelectedUserId">
+    <option value="">All users</option>
+    @if (UserList is { Count: > 0 }) {
+        @foreach (var user in UserList.OrderByDescending(u => u.Id).Where(x => !x.Deleted)) {
+            <option value="@user.Id">@user.Username</option>
+        }
+    }
+</InputSelect>
+
+
+<table class="table table-bordered">
+    @{
+        var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList();
+    }
+    <thead>
+        <tr>
+            @foreach (var column in columns) {
+                <th>@column.Name</th>
+            }
+            <th>Actions</th>
+        </tr>
+    </thead>
+    <tbody>
+        @foreach (var user in UserMedia) {
+            <tr>
+                @foreach (var column in columns) {
+                    <td>@column.GetValue(user)</td>
+                }
+                <td>
+                    <LinkButton href="@($"/Users/Delete/{user.Id}")" Color="#ff0000">Delete</LinkButton>
+                </td>
+            </tr>
+        }
+    </tbody>
+</table>
+
+@code {
+
+    private Dictionary<PropertyInfo, bool> DisplayedColumns { get; set; } = typeof(FileMetadataModel).GetProperties()
+        .ToDictionary(p => p, p => p.Name == "Username" || p.Name == "Id" || p.Name == "MessageCount");
+
+    private List<UserModel> UserList { get; set; } = new();
+    private List<FileMetadataModel> UserMedia { get; set; } = new();
+
+    [SupplyParameterFromQuery(Name = "UserId")]
+    public string? SelectedUserId {
+        get;
+        set {
+            field = value;
+            if (string.IsNullOrWhiteSpace(field))
+                UserMedia.Clear();
+            else _ = GetMediaForUser(value!);
+        }
+    }
+
+    protected override async Task OnInitializedAsync() {
+        using var hc = new HttpClient();
+        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
+        var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/");
+        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
+        await foreach (var user in content) {
+            UserList.Add(user!);
+            StateHasChanged();
+        }
+    }
+
+    private async Task GetMediaForUser(string userId) {
+        using var hc = new HttpClient();
+        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
+        var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/media/user/{userId}/attachments");
+        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        var content = response.Content.ReadFromJsonAsAsyncEnumerable<FileMetadataModel>();
+        await foreach (var media in content) {
+            UserMedia.Add(media!);
+            StateHasChanged();
+        }
+    }
+
+}

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Users.razor

@@ -4,6 +4,7 @@
 @using Spacebar.AdminApi.Models
 @using Spacebar.AdminAPI.TestClient.Services
 @using ArcaneLibs.Blazor.Components
+@using ArcaneLibs.Extensions
 @inject Config Config
 @inject ILocalStorageService LocalStorage
 
@@ -24,6 +25,7 @@
     }
 </details>
 
+<p>Got @UserList.Count users.</p>
 <table class="table table-bordered">
     @{
         var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList();
@@ -37,7 +39,7 @@
         </tr>
     </thead>
     <tbody>
-        @foreach (var user in UserList) {
+        @foreach (var user in UserList.Where(x => !x.Deleted).OrderByDescending(x=>x.MessageCount)) {
             <tr>
                 @foreach (var column in columns) {
                     <td>@column.GetValue(user)</td>
@@ -58,15 +60,21 @@
     private List<UserModel> UserList { get; set; } = new();
 
     protected override async Task OnInitializedAsync() {
-        using var hc = new HttpClient();
+        var hc = new StreamingHttpClient();
         hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
-        var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/");
+
-        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        // var request = new HttpRequestMessage(HttpMethod.Get, Config.AdminUrl + "/_spacebar/admin/users/");
-        var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
+        
-        await foreach (var user in content) {
+        var response = hc.GetAsyncEnumerableFromJsonAsync<UserModel>(Config.AdminUrl + "/_spacebar/admin/users/");
-            UserList.Add(user);
+        // if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
-            StateHasChanged();
+        // var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
+        await foreach (var user in response) {
+            // Console.WriteLine(user.ToJson(indent: false, ignoreNull: true));
+            UserList.Add(user!);
+            if(UserList.Count % 1000 == 0)
+                StateHasChanged();
         }
+        StateHasChanged();
     }
 
 }

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/UsersDelete.razor

@@ -14,15 +14,21 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far!
     <progress max="@progress.Total" value="@progress.Deleted"></progress>
 }
 
+@if (Done) {
+    <h1>Done!</h1>
+}
+
 @code {
 
     [Parameter]
     public required string Id { get; set; }
 
     private Dictionary<string, DeleteProgress> ChannelDeleteProgress { get; set; } = new();
+    
+    private bool Done { get; set; }
 
     protected override async Task OnInitializedAsync() {
-        using var hc = new HttpClient();
+        var hc = new StreamingHttpClient();
         hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
         var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/Users/{Id}/delete?messageDeleteChunkSize=100");
         if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
@@ -47,10 +53,13 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far!
                     break;
                 }
             }
-
+            
             StateHasChanged();
             await Task.Delay(1);
         }
+
+        Done = true;
+        StateHasChanged();
     }
 
     private class DeleteProgress {

extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Services/StreamingHttpClient.cs

@@ -0,0 +1,296 @@
+#define SINGLE_HTTPCLIENT // Use a single HttpClient instance for all MatrixHttpClient instances
+// #define SYNC_HTTPCLIENT   // Only allow one request as a time, for debugging
+using System.Data;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Net.Http.Headers;
+using System.Reflection;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using ArcaneLibs;
+using ArcaneLibs.Extensions;
+
+namespace Spacebar.AdminAPI.TestClient.Services;
+
+#if SINGLE_HTTPCLIENT
+// TODO: Add URI wrapper for
+public class StreamingHttpClient {
+    private static readonly HttpClient Client;
+
+    static StreamingHttpClient() {
+        try {
+            var handler = new SocketsHttpHandler {
+                PooledConnectionLifetime = TimeSpan.FromMinutes(15),
+                MaxConnectionsPerServer = 4096,
+                EnableMultipleHttp2Connections = true
+            };
+            Client = new HttpClient(handler) {
+                DefaultRequestVersion = new Version(3, 0),
+                Timeout = TimeSpan.FromDays(1)
+            };
+        }
+        catch (PlatformNotSupportedException e) {
+            Console.WriteLine("Failed to create HttpClient with connection pooling, continuing without connection pool!");
+            Console.WriteLine("Original exception (safe to ignore!):");
+            Console.WriteLine(e);
+
+            Client = new HttpClient {
+                DefaultRequestVersion = new Version(3, 0)
+            };
+        }
+        catch (Exception e) {
+            Console.WriteLine("Failed to create HttpClient:");
+            Console.WriteLine(e);
+            throw;
+        }
+    }
+
+#if SYNC_HTTPCLIENT
+    internal SemaphoreSlim _rateLimitSemaphore { get; } = new(1, 1);
+#endif
+
+    public static bool LogRequests = true;
+    public Dictionary<string, string> AdditionalQueryParameters { get; set; } = new();
+
+    public Uri? BaseAddress { get; set; }
+
+    // default headers, not bound to client
+    public HttpRequestHeaders DefaultRequestHeaders { get; set; } =
+        typeof(HttpRequestHeaders).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, [], null)?.Invoke([]) as HttpRequestHeaders ??
+        throw new InvalidOperationException("Failed to create HttpRequestHeaders");
+
+    private static JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) {
+        options ??= new JsonSerializerOptions();
+        // options.Converters.Add(new JsonFloatStringConverter());
+        // options.Converters.Add(new JsonDoubleStringConverter());
+        // options.Converters.Add(new JsonDecimalStringConverter());
+        options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+        return options;
+    }
+
+    public async Task<HttpResponseMessage> SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) {
+        if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
+        // if (!request.RequestUri.IsAbsoluteUri)
+        request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!);
+        var swWait = Stopwatch.StartNew();
+#if SYNC_HTTPCLIENT
+        await _rateLimitSemaphore.WaitAsync(cancellationToken);
+#endif
+
+        if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
+        if (!request.RequestUri.IsAbsoluteUri)
+            request.RequestUri = new Uri(BaseAddress ?? throw new InvalidOperationException("Relative URI passed, but no BaseAddress is specified!"), request.RequestUri);
+        swWait.Stop();
+        var swExec = Stopwatch.StartNew();
+        
+        foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value);
+        foreach (var (key, value) in DefaultRequestHeaders) {
+            if (request.Headers.Contains(key)) continue;
+            request.Headers.Add(key, value);
+        }
+
+        request.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse"), true);
+
+        if (LogRequests)
+            Console.WriteLine("Sending " + request.Summarise(includeHeaders: true, includeQuery: true, includeContentIfText: false, hideHeaders: ["Accept"]));
+
+        HttpResponseMessage? responseMessage;
+        try {
+            responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+        }
+        catch (Exception e) {
+            if (e is TaskCanceledException or TimeoutException) {
+                if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) {
+                    await Task.Delay(Random.Shared.Next(500, 2500), cancellationToken);
+                    request.ResetSendStatus();
+                    return await SendAsync(request, cancellationToken);
+                }
+            }
+            else if (!e.ToString().StartsWith("TypeError: NetworkError"))
+                Console.WriteLine(
+                    $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
+            throw;
+        }
+#if SYNC_HTTPCLIENT
+        finally {
+            _rateLimitSemaphore.Release();
+        }
+#endif
+
+        // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)");
+        if (LogRequests)
+            Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [
+                "Server",
+                "Date",
+                "Transfer-Encoding",
+                "Connection",
+                "Vary",
+                "Content-Length",
+                "Access-Control-Allow-Origin",
+                "Access-Control-Allow-Methods",
+                "Access-Control-Allow-Headers",
+                "Access-Control-Expose-Headers",
+                "Cache-Control",
+                "Cross-Origin-Resource-Policy",
+                "X-Content-Security-Policy",
+                "Referrer-Policy",
+                "X-Robots-Tag",
+                "Content-Security-Policy"
+            ]));
+
+        return responseMessage;
+    }
+
+    public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) {
+        var responseMessage = await SendUnhandledAsync(request, cancellationToken);
+        if (responseMessage.IsSuccessStatusCode) return responseMessage;
+
+        //retry on gateway timeout
+        // if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) {
+            // request.ResetSendStatus();
+            // return await SendAsync(request, cancellationToken);
+        // }
+
+        //error handling
+        var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken);
+        if (content.Length == 0)
+            throw new DataException("Content was empty");
+            // throw new MatrixException() {
+                // ErrorCode = "M_UNKNOWN",
+                // Error = "Unknown error, server returned no content"
+            // };
+        
+        // if (!content.StartsWith('{')) throw new InvalidDataException("Encountered invalid data:\n" + content);
+        if (!content.TrimStart().StartsWith('{')) {
+            responseMessage.EnsureSuccessStatusCode();
+            throw new InvalidDataException("Encountered invalid data:\n" + content);
+        }
+        //we have a matrix error
+
+        throw new Exception("Unknown http exception");
+        // MatrixException? ex;
+        // try {
+        //     ex = JsonSerializer.Deserialize<MatrixException>(content);
+        // }
+        // catch (JsonException e) {
+        //     throw new LibMatrixException() {
+        //         ErrorCode = "M_INVALID_JSON",
+        //         Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken)
+        //     };
+        // }
+        //
+        // Debug.Assert(ex != null, nameof(ex) + " != null");
+        // ex.RawContent = content;
+        // // Console.WriteLine($"Failed to send request: {ex}");
+        // if (ex.RetryAfterMs is null) throw ex!;
+        // //we have a ratelimit error
+        // await Task.Delay(ex.RetryAfterMs.Value, cancellationToken);
+        request.ResetSendStatus();
+        return await SendAsync(request, cancellationToken);
+    }
+
+    // GetAsync
+    public Task<HttpResponseMessage> GetAsync([StringSyntax("Uri")] string? requestUri, CancellationToken? cancellationToken = null) =>
+        SendAsync(new HttpRequestMessage(HttpMethod.Get, requestUri), cancellationToken ?? CancellationToken.None);
+
+    // GetFromJsonAsync
+    public async Task<T?> TryGetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
+        try {
+            return await GetFromJsonAsync<T>(requestUri, options, cancellationToken);
+        }
+        catch (JsonException e) {
+            Console.WriteLine($"Failed to deserialize response from {requestUri}: {e.Message}");
+            return default;
+        }
+        catch (HttpRequestException e) {
+            Console.WriteLine($"Failed to get {requestUri}: {e.Message}");
+            return default;
+        }
+    }
+
+    public async Task<T> GetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
+        options = GetJsonSerializerOptions(options);
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        var response = await SendAsync(request, cancellationToken);
+        response.EnsureSuccessStatusCode();
+        await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+
+        return await JsonSerializer.DeserializeAsync<T>(responseStream, options, cancellationToken) ??
+               throw new InvalidOperationException("Failed to deserialize response");
+    }
+
+    // GetStreamAsync
+    public async Task<Stream> GetStreamAsync(string requestUri, CancellationToken cancellationToken = default) {
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        var response = await SendAsync(request, cancellationToken);
+        response.EnsureSuccessStatusCode();
+        return await response.Content.ReadAsStreamAsync(cancellationToken);
+    }
+
+    public async Task<HttpResponseMessage> PutAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
+        CancellationToken cancellationToken = default) where T : notnull {
+        options = GetJsonSerializerOptions(options);
+        var request = new HttpRequestMessage(HttpMethod.Put, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
+            Encoding.UTF8, "application/json");
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async Task<HttpResponseMessage> PostAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
+        CancellationToken cancellationToken = default) where T : notnull {
+        options ??= new JsonSerializerOptions();
+        // options.Converters.Add(new JsonFloatStringConverter());
+        // options.Converters.Add(new JsonDoubleStringConverter());
+        // options.Converters.Add(new JsonDecimalStringConverter());
+        options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+        var request = new HttpRequestMessage(HttpMethod.Post, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
+            Encoding.UTF8, "application/json");
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async IAsyncEnumerable<T?> GetAsyncEnumerableFromJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, JsonSerializerOptions? options = null) {
+        options = GetJsonSerializerOptions(options);
+        var res = await GetAsync(requestUri);
+        options.PropertyNameCaseInsensitive = true;
+        var result = JsonSerializer.DeserializeAsyncEnumerable<T>(await res.Content.ReadAsStreamAsync(), options);
+        await foreach (var resp in result) yield return resp;
+    }
+
+    public static async Task<bool> CheckSuccessStatus(string url) {
+        //cors causes failure, try to catch
+        try {
+            var resp = await Client.GetAsync(url);
+            return resp.IsSuccessStatusCode;
+        }
+        catch (Exception e) {
+            Console.WriteLine($"Failed to check success status: {e.Message}");
+            return false;
+        }
+    }
+
+    public async Task<HttpResponseMessage> PostAsync(string uri, HttpContent? content, CancellationToken cancellationToken = default) {
+        var request = new HttpRequestMessage(HttpMethod.Post, uri) {
+            Content = content
+        };
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async Task DeleteAsync(string url) {
+        var request = new HttpRequestMessage(HttpMethod.Delete, url);
+        await SendAsync(request);
+    }
+
+    public async Task<HttpResponseMessage> DeleteAsJsonAsync<T>(string url, T payload) {
+        var request = new HttpRequestMessage(HttpMethod.Delete, url) {
+            Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
+        };
+        return await SendAsync(request);
+    }
+}
+#endif

extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Constants.cs

@@ -0,0 +1,7 @@
+namespace Spacebar.AdminApi.PrepareTestData;
+
+public class Constants {
+    public const string ApiBaseUrl = "http://localhost:3001/api/v9";
+    public const string CdnBaseUrl = "http://localhost:3003/";
+    // public const string ApiBaseUrl = "http://localhost:3001/api/v9";
+}

extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Program.cs

@@ -1,8 +1,17 @@
 // See https://aka.ms/new-console-template for more information
 
 using ArcaneLibs;
+using Spacebar.AdminApi.PrepareTestData;
 using Spacebar.AdminApi.PrepareTestData.TestDataTypes;
 
+await Utils.PostFileWithDataAsync("http://localhost:3001/api/v9/channels/1324497120836834414/messages",
+    "eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjExODM1Njg3NTA5MzEwOTk2NzkiLCJpYXQiOjE3NDQyMTMxNzcsImtpZCI6IjdiMWM5OTBhMWQ1ZWI3MDVjMWFjNmIxOWYwNTVmMTM5Y2FiZDhhOTZmMzg3YTU1NDM3MDRhZDY0OTMyMzViYTMifQ.AUf87OS5DsWLfBR9VVF7emOE8cG8B4JLvktr2WxF9_XQsPd0X8da2s9f9Lq5pTmYe9zOaI7DrHMuggih3uZ9NmZzAfeasEgew4gCBKIcvhxSaKWcU9DMVHgZl-ZH5HnB0yk8l5IKzIV3z6wt9Ght-F_g5SRZiNlpthva0jU2QhRro3IB",
+    new {
+        content = "Hello world",
+        nonce = Random.Shared.NextInt64()
+    },
+    File.ReadAllBytes("/home/Rory/Documents/kuromi_smug.png"), "test.png", "image/png");
+return;
 Console.WriteLine("Hello, World!");
 var tests = ClassCollector<ITestData>.ResolveFromAllAccessibleAssemblies();
 foreach (var test in tests) {
@@ -19,6 +28,7 @@ if (runMethod != null) {
     var task = runMethod.Invoke(testToRun, null) as Task;
     await task!;
     Console.WriteLine($"Test {testToRun.FullName} completed.");
-} else {
+}
+else {
     Console.WriteLine("Test not found.");
 }

extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/TestDataTypes/AttachmentSpamTestData.cs

@@ -0,0 +1,87 @@
+using System.Net.Http.Json;
+using System.Text.Json.Nodes;
+
+namespace Spacebar.AdminApi.PrepareTestData.TestDataTypes;
+
+public class AttachmentSpamTestData : ITestData {
+    public static async Task Run() {
+        using var hc = new HttpClient();
+        var token = await Utils.CreateUser();
+        hc.DefaultRequestHeaders.Authorization = new("Bearer", token);
+
+        int guildCount = 1;
+        int channelCount = 100;  // per guild
+        int messageCount = 1000; // per channel
+
+        for (int guild = 0; guild < guildCount; guild++) {
+            var guildId = await Utils.CreateGuild(token);
+            // for (int channel = 0; channel < channelCount; channel++) {
+            //     Console.WriteLine($"> Creating channel {channel} in guild {guild}...");
+            //     var channelRequest = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/guilds/{guildId}/channels", new {
+            //         name = Guid.NewGuid().ToString()[0..30],
+            //         type = 0
+            //     });
+            //     var channelResponse = await channelRequest.Content.ReadFromJsonAsync<JsonObject>();
+            //     var channelId = channelResponse!["id"]!.ToString();
+            //     await SendMessages(hc, channelId, messageCount);
+            // }
+            
+            var ss = new SemaphoreSlim(16,16);
+            var tasks = Enumerable.Range(0, channelCount).Select(async channel => {
+                await ss.WaitAsync();
+                Console.WriteLine($"> Creating channel {channel} in guild {guildId}...");
+                var channelRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/guilds/{guildId}/channels", new {
+                    name = Guid.NewGuid().ToString()[0..30],
+                    type = 0
+                });
+                var channelResponse = await channelRequest.Content.ReadFromJsonAsync<JsonObject>();
+                var channelId = channelResponse!["id"]!.ToString();
+                await SendMessages(hc, channelId, messageCount);
+                ss.Release();
+            });
+            await Task.WhenAll(tasks);
+        }
+    }
+    
+    private static async Task CreateChannels(HttpClient hc, string guildId, int channelCount) {
+        var tasks = Enumerable.Range(0, channelCount).Select(async channel => {
+            Console.WriteLine($"> Creating channel {channel} in guild {guildId}...");
+            var channelRequest = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/guilds/{guildId}/channels", new {
+                name = Guid.NewGuid().ToString()[0..30],
+                type = 0
+            });
+            var channelResponse = await channelRequest.Content.ReadFromJsonAsync<JsonObject>();
+        });
+        await Task.WhenAll(tasks);
+    }
+
+    private static async Task SendMessages(HttpClient hc, string channelId, int maxMessageCount) {
+        // var ss = new SemaphoreSlim(32, 32);
+        // var tasks = Enumerable.Range(0, Random.Shared.Next((int)(0.75 * maxMessageCount), maxMessageCount)).Select(async message => {
+        //     var success = false;
+        //     while (!success) {
+        //         await ss.WaitAsync();
+        //         Console.WriteLine($"> Sending message {message} in channel {channelId}...");
+        //         var messageReq = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/channels/{channelId}/messages", new {
+        //             content = Guid.NewGuid().ToString()
+        //         });
+        //         var messageResponse = await messageReq.Content.ReadFromJsonAsync<JsonObject>();
+        //         if (messageResponse.ContainsKey("id")) {
+        //             success = true;
+        //             Console.WriteLine(messageResponse!["id"]!.ToString());
+        //         }
+        //     }
+        //
+        //     ss.Release();
+        // });
+        // await Task.WhenAll(tasks);
+        
+        var messageReq = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/channels/{channelId}/messages", new {
+            content = Guid.NewGuid().ToString()
+        });
+        var messageResponse = await messageReq.Content.ReadFromJsonAsync<JsonObject>();
+        if (messageResponse.ContainsKey("id")) {
+            await hc.GetAsync($"http://localhost:5112/Users/duplicate/{messageResponse!["id"]!.ToString()}?count={maxMessageCount}");
+        }
+    }
+}

extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Utils.cs

@@ -1,5 +1,9 @@
+using System.Net;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using System.Text.Json;
 using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
 using ArcaneLibs.Extensions;
 
 namespace Spacebar.AdminApi.PrepareTestData;
@@ -8,7 +12,7 @@ public static class Utils {
     public static async Task<string> CreateUser() {
         Console.WriteLine("> Creating user...");
         using var hc = new HttpClient();
-        var registerRequest = await hc.PostAsJsonAsync("http://localhost:3001/api/v9/auth/register", new {
+        var registerRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/auth/register", new {
             username = Guid.NewGuid().ToString()[0..30],
             password = "password",
             email = $"{Guid.NewGuid()}@example.com",
@@ -22,14 +26,36 @@ public static class Utils {
     public static async Task<string> CreateGuild(string token) {
         using var hc = new HttpClient();
         hc.DefaultRequestHeaders.Authorization = new("Bearer", token);
-        
+
         Console.WriteLine("> Creating guild...");
-        var guildRequest = await hc.PostAsJsonAsync("http://localhost:3001/api/v9/guilds", new {
+        var guildRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/guilds", new {
-            name = Guid.NewGuid().ToString()[0..30]
+            name = Guid.NewGuid().ToString()[..30]
         });
-        
+
         var guildResponse = await guildRequest.Content.ReadFromJsonAsync<JsonObject>();
         var guildId = guildResponse!["id"]!.ToString();
         return guildId;
     }
+
+    public static async Task PostFileWithDataAsync(string url, string token, object data, byte[] file, string filename, string contentType) {
+        try {
+            using var hc = new HttpClient();
+            hc.DefaultRequestHeaders.Authorization = new("Bearer", token);
+            var f = new MultipartFormDataContent();
+            // f.Add(new StringContent());
+            f.Add(JsonContent.Create(data), "payload_json");
+            f.Add(new ByteArrayContent(file) {
+                Headers = { ContentType = new MediaTypeHeaderValue(contentType) }
+            }, "files[0]", filename);
+
+            var _resp = await hc.PostAsync(url, f);
+            var resp = await _resp.Content.ReadAsStringAsync();
+
+            Console.WriteLine(resp);
+        }
+        catch (Exception e) {
+            Console.WriteLine(e);
+            throw;
+        }
+    }
 }