From e1483e9f90ef506444a75499768ec0d7c27bfdc8 Mon Sep 17 00:00:00 2001 From: "Emma [it/its]@Rory&" Date: Thu, 10 Apr 2025 17:12:53 +0200 Subject: [PATCH] Further admin api work --- .../Controllers/Media/UserMediaController.cs | 27 ++ .../Controllers/UserController.cs | 7 + .../Middleware/AuthenticationMiddleware.cs | 4 +- .../Services/AuthenticationService.cs | 4 +- .../Services/Configuration.cs | 2 + .../FileMetadataModel.cs | 23 ++ extra/admin-api/Spacebar.Db/Models/User.cs | 2 +- .../Layout/NavMenu.razor | 5 + .../Pages/Media/Index.razor | 7 + .../Pages/Media/Users.razor | 106 +++++++ .../Pages/Users.razor | 24 +- .../Pages/UsersDelete.razor | 13 +- .../Services/StreamingHttpClient.cs | 296 ++++++++++++++++++ .../Constants.cs | 7 + .../Program.cs | 12 +- .../TestDataTypes/AttachmentSpamTestData.cs | 87 +++++ .../Utils.cs | 36 ++- 17 files changed, 642 insertions(+), 20 deletions(-) create mode 100644 extra/admin-api/Spacebar.AdminAPI/Controllers/Media/UserMediaController.cs create mode 100644 extra/admin-api/Spacebar.AdminApi.Models/FileMetadataModel.cs create mode 100644 extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Index.razor create mode 100644 extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Users.razor create mode 100644 extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Services/StreamingHttpClient.cs create mode 100644 extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Constants.cs create mode 100644 extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/TestDataTypes/AttachmentSpamTestData.cs diff --git a/extra/admin-api/Spacebar.AdminAPI/Controllers/Media/UserMediaController.cs b/extra/admin-api/Spacebar.AdminAPI/Controllers/Media/UserMediaController.cs new file mode 100644 index 00000000..a06d110b --- /dev/null +++ b/extra/admin-api/Spacebar.AdminAPI/Controllers/Media/UserMediaController.cs @@ -0,0 +1,27 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Spacebar.AdminApi.Models; +using Spacebar.Db.Contexts; +using Spacebar.Db.Models; +using Spacebar.RabbitMqUtilities; + +namespace Spacebar.AdminAPI.Controllers.Media; + +[ApiController] +[Route("/media/user")] +public class UserMediaController(ILogger logger, SpacebarDbContext db, RabbitMQService mq, IServiceProvider sp) : ControllerBase { + + [HttpGet("{userId}/attachments")] + public async IAsyncEnumerable GetAttachmentsByUser(string userId) { + var db2 = sp.CreateScope().ServiceProvider.GetService(); + var attachments = db.Attachments + // .IgnoreAutoIncludes() + .Where(x => x.Message!.AuthorId == userId) + .AsAsyncEnumerable(); + await foreach (var attachment in attachments) { + attachment.Message = await db2.Messages.FindAsync(attachment.MessageId); + // attachment.Message.Author = await db2.Users.FindAsync(attachment.Message.AuthorId); + yield return attachment; + } + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs index 3b753ebd..620382e1 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs @@ -146,6 +146,13 @@ public class UserController(ILogger logger, SpacebarDbContext db yield break; } + user.Data = "{}"; + user.Deleted = true; + user.Disabled = true; + user.Rights = 0; + db.Users.Update(user); + await db.SaveChangesAsync(); + var factory = new ConnectionFactory { Uri = new Uri("amqp://guest:guest@127.0.0.1/") }; diff --git a/extra/admin-api/Spacebar.AdminAPI/Middleware/AuthenticationMiddleware.cs b/extra/admin-api/Spacebar.AdminAPI/Middleware/AuthenticationMiddleware.cs index c87d568f..2dc4c977 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Middleware/AuthenticationMiddleware.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Middleware/AuthenticationMiddleware.cs @@ -2,6 +2,7 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Cryptography; using ArcaneLibs.Extensions; using Microsoft.IdentityModel.Tokens; +using Spacebar.AdminAPI.Services; using Spacebar.Db.Contexts; using Spacebar.Db.Models; @@ -54,7 +55,8 @@ public class AuthenticationMiddleware(RequestDelegate next) { if (!_userCache.ContainsKey(token)) { var db = sp.GetRequiredService(); - user = await db.Users.FindAsync(res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) + var config = sp.GetRequiredService(); + user = await db.Users.FindAsync(config.OverrideUid ?? res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) ?? throw new InvalidOperationException(); _userCache[token] = user; _userCacheExpiry[token] = DateTime.Now.AddMinutes(5); diff --git a/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs b/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs index 2ec7fab1..53d2cf21 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs @@ -7,7 +7,7 @@ using Spacebar.Db.Models; namespace Spacebar.AdminAPI.Services; -public class AuthenticationService(SpacebarDbContext db) { +public class AuthenticationService(SpacebarDbContext db, Configuration config) { private static Dictionary _userCache = new(); private static Dictionary _userCacheExpiry = new(); @@ -37,6 +37,6 @@ public class AuthenticationService(SpacebarDbContext db) { throw new UnauthorizedAccessException(); } - return await db.Users.FindAsync(res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) ?? throw new InvalidOperationException(); + return await db.Users.FindAsync(config.OverrideUid ?? res.ClaimsIdentity.Claims.First(x => x.Type == "id").Value) ?? throw new InvalidOperationException(); } } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminAPI/Services/Configuration.cs b/extra/admin-api/Spacebar.AdminAPI/Services/Configuration.cs index a580e6c6..a988a316 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Services/Configuration.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Services/Configuration.cs @@ -4,4 +4,6 @@ public class Configuration { public Configuration(IConfiguration configuration) { configuration.GetRequiredSection("SpacebarAdminApi").Bind(this); } + + public string? OverrideUid { get; set; } } \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminApi.Models/FileMetadataModel.cs b/extra/admin-api/Spacebar.AdminApi.Models/FileMetadataModel.cs new file mode 100644 index 00000000..dfc577a9 --- /dev/null +++ b/extra/admin-api/Spacebar.AdminApi.Models/FileMetadataModel.cs @@ -0,0 +1,23 @@ +using System.Text.Json.Serialization; + +namespace Spacebar.AdminApi.Models; + +public class FileMetadataModel { + public string UserId { get; set; } = null!; + public string Id { get; set; } = null!; + + [JsonConverter(typeof(JsonStringEnumConverter))] + public FileUploadType Type { get; set; } + + + public enum FileUploadType { + Attachment, + Avatar, + Banner, + GuildIcon, + GuildSplash, + GuildCover, + Emoji, + Sticker + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.Db/Models/User.cs b/extra/admin-api/Spacebar.Db/Models/User.cs index 114572fb..94fc5fa6 100644 --- a/extra/admin-api/Spacebar.Db/Models/User.cs +++ b/extra/admin-api/Spacebar.Db/Models/User.cs @@ -92,7 +92,7 @@ public partial class User [Column("email", TypeName = "character varying")] public string? Email { get; set; } - [Column("flags")] + [Column("flags", TypeName = "text")] public ulong Flags { get; set; } [Column("public_flags")] diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor index 8a96bc59..732d46e7 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor @@ -24,6 +24,11 @@ Guilds +
+ + Media + +
diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Index.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Index.razor new file mode 100644 index 00000000..76621e74 --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Index.razor @@ -0,0 +1,7 @@ +@page "/Media" +

Index of /Media

+
+ +@code { + +} \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Users.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Users.razor new file mode 100644 index 00000000..81008a25 --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Media/Users.razor @@ -0,0 +1,106 @@ +@page "/Media/ByUser" +@using System.Net.Http.Headers +@using System.Reflection +@using Spacebar.AdminApi.Models +@using Spacebar.AdminAPI.TestClient.Services +@using ArcaneLibs.Blazor.Components +@inject Config Config +@inject ILocalStorageService LocalStorage + +Uploaded media by user + +
+ Displayed columns + @foreach (var column in DisplayedColumns) { + var value = column.Value; + + + @column.Key.Name + +
+ } +
+ + + + @if (UserList is { Count: > 0 }) { + @foreach (var user in UserList.OrderByDescending(u => u.Id).Where(x => !x.Deleted)) { + + } + } + + + + + @{ + var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList(); + } + + + @foreach (var column in columns) { + + } + + + + + @foreach (var user in UserMedia) { + + @foreach (var column in columns) { + + } + + + } + +
@column.NameActions
@column.GetValue(user) + Delete +
+ +@code { + + private Dictionary DisplayedColumns { get; set; } = typeof(FileMetadataModel).GetProperties() + .ToDictionary(p => p, p => p.Name == "Username" || p.Name == "Id" || p.Name == "MessageCount"); + + private List UserList { get; set; } = new(); + private List UserMedia { get; set; } = new(); + + [SupplyParameterFromQuery(Name = "UserId")] + public string? SelectedUserId { + get; + set { + field = value; + if (string.IsNullOrWhiteSpace(field)) + UserMedia.Clear(); + else _ = GetMediaForUser(value!); + } + } + + protected override async Task OnInitializedAsync() { + using var hc = new HttpClient(); + hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken); + var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/"); + if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync()); + var content = response.Content.ReadFromJsonAsAsyncEnumerable(); + await foreach (var user in content) { + UserList.Add(user!); + StateHasChanged(); + } + } + + private async Task GetMediaForUser(string userId) { + using var hc = new HttpClient(); + hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken); + var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/media/user/{userId}/attachments"); + if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync()); + var content = response.Content.ReadFromJsonAsAsyncEnumerable(); + await foreach (var media in content) { + UserMedia.Add(media!); + StateHasChanged(); + } + } + +} \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Users.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Users.razor index 13c34661..c0e67867 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Users.razor +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/Users.razor @@ -4,6 +4,7 @@ @using Spacebar.AdminApi.Models @using Spacebar.AdminAPI.TestClient.Services @using ArcaneLibs.Blazor.Components +@using ArcaneLibs.Extensions @inject Config Config @inject ILocalStorageService LocalStorage @@ -24,6 +25,7 @@ } +

Got @UserList.Count users.

@{ var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList(); @@ -37,7 +39,7 @@ - @foreach (var user in UserList) { + @foreach (var user in UserList.Where(x => !x.Deleted).OrderByDescending(x=>x.MessageCount)) { @foreach (var column in columns) { @@ -58,15 +60,21 @@ private List UserList { get; set; } = new(); protected override async Task OnInitializedAsync() { - using var hc = new HttpClient(); + var hc = new StreamingHttpClient(); hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken); - var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/"); - if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync()); - var content = response.Content.ReadFromJsonAsAsyncEnumerable(); - await foreach (var user in content) { - UserList.Add(user); - StateHasChanged(); + + // var request = new HttpRequestMessage(HttpMethod.Get, Config.AdminUrl + "/_spacebar/admin/users/"); + + var response = hc.GetAsyncEnumerableFromJsonAsync(Config.AdminUrl + "/_spacebar/admin/users/"); + // if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync()); + // var content = response.Content.ReadFromJsonAsAsyncEnumerable(); + await foreach (var user in response) { + // Console.WriteLine(user.ToJson(indent: false, ignoreNull: true)); + UserList.Add(user!); + if(UserList.Count % 1000 == 0) + StateHasChanged(); } + StateHasChanged(); } } \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/UsersDelete.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/UsersDelete.razor index 084a372f..98a3e0fc 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/UsersDelete.razor +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Pages/UsersDelete.razor @@ -14,15 +14,21 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far! } +@if (Done) { +

Done!

+} + @code { [Parameter] public required string Id { get; set; } private Dictionary ChannelDeleteProgress { get; set; } = new(); + + private bool Done { get; set; } protected override async Task OnInitializedAsync() { - using var hc = new HttpClient(); + var hc = new StreamingHttpClient(); hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken); var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/Users/{Id}/delete?messageDeleteChunkSize=100"); if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync()); @@ -47,10 +53,13 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far! break; } } - + StateHasChanged(); await Task.Delay(1); } + + Done = true; + StateHasChanged(); } private class DeleteProgress { diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Services/StreamingHttpClient.cs b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Services/StreamingHttpClient.cs new file mode 100644 index 00000000..67dc673e --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Services/StreamingHttpClient.cs @@ -0,0 +1,296 @@ +#define SINGLE_HTTPCLIENT // Use a single HttpClient instance for all MatrixHttpClient instances +// #define SYNC_HTTPCLIENT // Only allow one request as a time, for debugging +using System.Data; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http.Headers; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using ArcaneLibs; +using ArcaneLibs.Extensions; + +namespace Spacebar.AdminAPI.TestClient.Services; + +#if SINGLE_HTTPCLIENT +// TODO: Add URI wrapper for +public class StreamingHttpClient { + private static readonly HttpClient Client; + + static StreamingHttpClient() { + try { + var handler = new SocketsHttpHandler { + PooledConnectionLifetime = TimeSpan.FromMinutes(15), + MaxConnectionsPerServer = 4096, + EnableMultipleHttp2Connections = true + }; + Client = new HttpClient(handler) { + DefaultRequestVersion = new Version(3, 0), + Timeout = TimeSpan.FromDays(1) + }; + } + catch (PlatformNotSupportedException e) { + Console.WriteLine("Failed to create HttpClient with connection pooling, continuing without connection pool!"); + Console.WriteLine("Original exception (safe to ignore!):"); + Console.WriteLine(e); + + Client = new HttpClient { + DefaultRequestVersion = new Version(3, 0) + }; + } + catch (Exception e) { + Console.WriteLine("Failed to create HttpClient:"); + Console.WriteLine(e); + throw; + } + } + +#if SYNC_HTTPCLIENT + internal SemaphoreSlim _rateLimitSemaphore { get; } = new(1, 1); +#endif + + public static bool LogRequests = true; + public Dictionary AdditionalQueryParameters { get; set; } = new(); + + public Uri? BaseAddress { get; set; } + + // default headers, not bound to client + public HttpRequestHeaders DefaultRequestHeaders { get; set; } = + typeof(HttpRequestHeaders).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, [], null)?.Invoke([]) as HttpRequestHeaders ?? + throw new InvalidOperationException("Failed to create HttpRequestHeaders"); + + private static JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) { + options ??= new JsonSerializerOptions(); + // options.Converters.Add(new JsonFloatStringConverter()); + // options.Converters.Add(new JsonDoubleStringConverter()); + // options.Converters.Add(new JsonDecimalStringConverter()); + options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + return options; + } + + public async Task SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) { + if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); + // if (!request.RequestUri.IsAbsoluteUri) + request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!); + var swWait = Stopwatch.StartNew(); +#if SYNC_HTTPCLIENT + await _rateLimitSemaphore.WaitAsync(cancellationToken); +#endif + + if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); + if (!request.RequestUri.IsAbsoluteUri) + request.RequestUri = new Uri(BaseAddress ?? throw new InvalidOperationException("Relative URI passed, but no BaseAddress is specified!"), request.RequestUri); + swWait.Stop(); + var swExec = Stopwatch.StartNew(); + + foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value); + foreach (var (key, value) in DefaultRequestHeaders) { + if (request.Headers.Contains(key)) continue; + request.Headers.Add(key, value); + } + + request.Options.Set(new HttpRequestOptionsKey("WebAssemblyEnableStreamingResponse"), true); + + if (LogRequests) + Console.WriteLine("Sending " + request.Summarise(includeHeaders: true, includeQuery: true, includeContentIfText: false, hideHeaders: ["Accept"])); + + HttpResponseMessage? responseMessage; + try { + responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + } + catch (Exception e) { + if (e is TaskCanceledException or TimeoutException) { + if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) { + await Task.Delay(Random.Shared.Next(500, 2500), cancellationToken); + request.ResetSendStatus(); + return await SendAsync(request, cancellationToken); + } + } + else if (!e.ToString().StartsWith("TypeError: NetworkError")) + Console.WriteLine( + $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}"); + throw; + } +#if SYNC_HTTPCLIENT + finally { + _rateLimitSemaphore.Release(); + } +#endif + + // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)"); + if (LogRequests) + Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [ + "Server", + "Date", + "Transfer-Encoding", + "Connection", + "Vary", + "Content-Length", + "Access-Control-Allow-Origin", + "Access-Control-Allow-Methods", + "Access-Control-Allow-Headers", + "Access-Control-Expose-Headers", + "Cache-Control", + "Cross-Origin-Resource-Policy", + "X-Content-Security-Policy", + "Referrer-Policy", + "X-Robots-Tag", + "Content-Security-Policy" + ])); + + return responseMessage; + } + + public async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) { + var responseMessage = await SendUnhandledAsync(request, cancellationToken); + if (responseMessage.IsSuccessStatusCode) return responseMessage; + + //retry on gateway timeout + // if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) { + // request.ResetSendStatus(); + // return await SendAsync(request, cancellationToken); + // } + + //error handling + var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken); + if (content.Length == 0) + throw new DataException("Content was empty"); + // throw new MatrixException() { + // ErrorCode = "M_UNKNOWN", + // Error = "Unknown error, server returned no content" + // }; + + // if (!content.StartsWith('{')) throw new InvalidDataException("Encountered invalid data:\n" + content); + if (!content.TrimStart().StartsWith('{')) { + responseMessage.EnsureSuccessStatusCode(); + throw new InvalidDataException("Encountered invalid data:\n" + content); + } + //we have a matrix error + + throw new Exception("Unknown http exception"); + // MatrixException? ex; + // try { + // ex = JsonSerializer.Deserialize(content); + // } + // catch (JsonException e) { + // throw new LibMatrixException() { + // ErrorCode = "M_INVALID_JSON", + // Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken) + // }; + // } + // + // Debug.Assert(ex != null, nameof(ex) + " != null"); + // ex.RawContent = content; + // // Console.WriteLine($"Failed to send request: {ex}"); + // if (ex.RetryAfterMs is null) throw ex!; + // //we have a ratelimit error + // await Task.Delay(ex.RetryAfterMs.Value, cancellationToken); + request.ResetSendStatus(); + return await SendAsync(request, cancellationToken); + } + + // GetAsync + public Task GetAsync([StringSyntax("Uri")] string? requestUri, CancellationToken? cancellationToken = null) => + SendAsync(new HttpRequestMessage(HttpMethod.Get, requestUri), cancellationToken ?? CancellationToken.None); + + // GetFromJsonAsync + public async Task TryGetFromJsonAsync(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) { + try { + return await GetFromJsonAsync(requestUri, options, cancellationToken); + } + catch (JsonException e) { + Console.WriteLine($"Failed to deserialize response from {requestUri}: {e.Message}"); + return default; + } + catch (HttpRequestException e) { + Console.WriteLine($"Failed to get {requestUri}: {e.Message}"); + return default; + } + } + + public async Task GetFromJsonAsync(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) { + options = GetJsonSerializerOptions(options); + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + var response = await SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + + return await JsonSerializer.DeserializeAsync(responseStream, options, cancellationToken) ?? + throw new InvalidOperationException("Failed to deserialize response"); + } + + // GetStreamAsync + public async Task GetStreamAsync(string requestUri, CancellationToken cancellationToken = default) { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + var response = await SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStreamAsync(cancellationToken); + } + + public async Task PutAsJsonAsync([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) where T : notnull { + options = GetJsonSerializerOptions(options); + var request = new HttpRequestMessage(HttpMethod.Put, requestUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options), + Encoding.UTF8, "application/json"); + return await SendAsync(request, cancellationToken); + } + + public async Task PostAsJsonAsync([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) where T : notnull { + options ??= new JsonSerializerOptions(); + // options.Converters.Add(new JsonFloatStringConverter()); + // options.Converters.Add(new JsonDoubleStringConverter()); + // options.Converters.Add(new JsonDecimalStringConverter()); + options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + var request = new HttpRequestMessage(HttpMethod.Post, requestUri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options), + Encoding.UTF8, "application/json"); + return await SendAsync(request, cancellationToken); + } + + public async IAsyncEnumerable GetAsyncEnumerableFromJsonAsync([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, JsonSerializerOptions? options = null) { + options = GetJsonSerializerOptions(options); + var res = await GetAsync(requestUri); + options.PropertyNameCaseInsensitive = true; + var result = JsonSerializer.DeserializeAsyncEnumerable(await res.Content.ReadAsStreamAsync(), options); + await foreach (var resp in result) yield return resp; + } + + public static async Task CheckSuccessStatus(string url) { + //cors causes failure, try to catch + try { + var resp = await Client.GetAsync(url); + return resp.IsSuccessStatusCode; + } + catch (Exception e) { + Console.WriteLine($"Failed to check success status: {e.Message}"); + return false; + } + } + + public async Task PostAsync(string uri, HttpContent? content, CancellationToken cancellationToken = default) { + var request = new HttpRequestMessage(HttpMethod.Post, uri) { + Content = content + }; + return await SendAsync(request, cancellationToken); + } + + public async Task DeleteAsync(string url) { + var request = new HttpRequestMessage(HttpMethod.Delete, url); + await SendAsync(request); + } + + public async Task DeleteAsJsonAsync(string url, T payload) { + var request = new HttpRequestMessage(HttpMethod.Delete, url) { + Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json") + }; + return await SendAsync(request); + } +} +#endif diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Constants.cs b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Constants.cs new file mode 100644 index 00000000..02d3941f --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Constants.cs @@ -0,0 +1,7 @@ +namespace Spacebar.AdminApi.PrepareTestData; + +public class Constants { + public const string ApiBaseUrl = "http://localhost:3001/api/v9"; + public const string CdnBaseUrl = "http://localhost:3003/"; + // public const string ApiBaseUrl = "http://localhost:3001/api/v9"; +} \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Program.cs b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Program.cs index 4bfb037a..268066fe 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Program.cs +++ b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Program.cs @@ -1,8 +1,17 @@ // See https://aka.ms/new-console-template for more information using ArcaneLibs; +using Spacebar.AdminApi.PrepareTestData; using Spacebar.AdminApi.PrepareTestData.TestDataTypes; +await Utils.PostFileWithDataAsync("http://localhost:3001/api/v9/channels/1324497120836834414/messages", + "eyJhbGciOiJFUzUxMiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjExODM1Njg3NTA5MzEwOTk2NzkiLCJpYXQiOjE3NDQyMTMxNzcsImtpZCI6IjdiMWM5OTBhMWQ1ZWI3MDVjMWFjNmIxOWYwNTVmMTM5Y2FiZDhhOTZmMzg3YTU1NDM3MDRhZDY0OTMyMzViYTMifQ.AUf87OS5DsWLfBR9VVF7emOE8cG8B4JLvktr2WxF9_XQsPd0X8da2s9f9Lq5pTmYe9zOaI7DrHMuggih3uZ9NmZzAfeasEgew4gCBKIcvhxSaKWcU9DMVHgZl-ZH5HnB0yk8l5IKzIV3z6wt9Ght-F_g5SRZiNlpthva0jU2QhRro3IB", + new { + content = "Hello world", + nonce = Random.Shared.NextInt64() + }, + File.ReadAllBytes("/home/Rory/Documents/kuromi_smug.png"), "test.png", "image/png"); +return; Console.WriteLine("Hello, World!"); var tests = ClassCollector.ResolveFromAllAccessibleAssemblies(); foreach (var test in tests) { @@ -19,6 +28,7 @@ if (runMethod != null) { var task = runMethod.Invoke(testToRun, null) as Task; await task!; Console.WriteLine($"Test {testToRun.FullName} completed."); -} else { +} +else { Console.WriteLine("Test not found."); } \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/TestDataTypes/AttachmentSpamTestData.cs b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/TestDataTypes/AttachmentSpamTestData.cs new file mode 100644 index 00000000..114d45a2 --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/TestDataTypes/AttachmentSpamTestData.cs @@ -0,0 +1,87 @@ +using System.Net.Http.Json; +using System.Text.Json.Nodes; + +namespace Spacebar.AdminApi.PrepareTestData.TestDataTypes; + +public class AttachmentSpamTestData : ITestData { + public static async Task Run() { + using var hc = new HttpClient(); + var token = await Utils.CreateUser(); + hc.DefaultRequestHeaders.Authorization = new("Bearer", token); + + int guildCount = 1; + int channelCount = 100; // per guild + int messageCount = 1000; // per channel + + for (int guild = 0; guild < guildCount; guild++) { + var guildId = await Utils.CreateGuild(token); + // for (int channel = 0; channel < channelCount; channel++) { + // Console.WriteLine($"> Creating channel {channel} in guild {guild}..."); + // var channelRequest = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/guilds/{guildId}/channels", new { + // name = Guid.NewGuid().ToString()[0..30], + // type = 0 + // }); + // var channelResponse = await channelRequest.Content.ReadFromJsonAsync(); + // var channelId = channelResponse!["id"]!.ToString(); + // await SendMessages(hc, channelId, messageCount); + // } + + var ss = new SemaphoreSlim(16,16); + var tasks = Enumerable.Range(0, channelCount).Select(async channel => { + await ss.WaitAsync(); + Console.WriteLine($"> Creating channel {channel} in guild {guildId}..."); + var channelRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/guilds/{guildId}/channels", new { + name = Guid.NewGuid().ToString()[0..30], + type = 0 + }); + var channelResponse = await channelRequest.Content.ReadFromJsonAsync(); + var channelId = channelResponse!["id"]!.ToString(); + await SendMessages(hc, channelId, messageCount); + ss.Release(); + }); + await Task.WhenAll(tasks); + } + } + + private static async Task CreateChannels(HttpClient hc, string guildId, int channelCount) { + var tasks = Enumerable.Range(0, channelCount).Select(async channel => { + Console.WriteLine($"> Creating channel {channel} in guild {guildId}..."); + var channelRequest = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/guilds/{guildId}/channels", new { + name = Guid.NewGuid().ToString()[0..30], + type = 0 + }); + var channelResponse = await channelRequest.Content.ReadFromJsonAsync(); + }); + await Task.WhenAll(tasks); + } + + private static async Task SendMessages(HttpClient hc, string channelId, int maxMessageCount) { + // var ss = new SemaphoreSlim(32, 32); + // var tasks = Enumerable.Range(0, Random.Shared.Next((int)(0.75 * maxMessageCount), maxMessageCount)).Select(async message => { + // var success = false; + // while (!success) { + // await ss.WaitAsync(); + // Console.WriteLine($"> Sending message {message} in channel {channelId}..."); + // var messageReq = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/channels/{channelId}/messages", new { + // content = Guid.NewGuid().ToString() + // }); + // var messageResponse = await messageReq.Content.ReadFromJsonAsync(); + // if (messageResponse.ContainsKey("id")) { + // success = true; + // Console.WriteLine(messageResponse!["id"]!.ToString()); + // } + // } + // + // ss.Release(); + // }); + // await Task.WhenAll(tasks); + + var messageReq = await hc.PostAsJsonAsync($"http://localhost:3001/api/v9/channels/{channelId}/messages", new { + content = Guid.NewGuid().ToString() + }); + var messageResponse = await messageReq.Content.ReadFromJsonAsync(); + if (messageResponse.ContainsKey("id")) { + await hc.GetAsync($"http://localhost:5112/Users/duplicate/{messageResponse!["id"]!.ToString()}?count={maxMessageCount}"); + } + } +} \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Utils.cs b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Utils.cs index 143f50a4..398dbaf2 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Utils.cs +++ b/extra/admin-api/Utilities/Spacebar.AdminApi.PrepareTestData/Utils.cs @@ -1,5 +1,9 @@ +using System.Net; +using System.Net.Http.Headers; using System.Net.Http.Json; +using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using ArcaneLibs.Extensions; namespace Spacebar.AdminApi.PrepareTestData; @@ -8,7 +12,7 @@ public static class Utils { public static async Task CreateUser() { Console.WriteLine("> Creating user..."); using var hc = new HttpClient(); - var registerRequest = await hc.PostAsJsonAsync("http://localhost:3001/api/v9/auth/register", new { + var registerRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/auth/register", new { username = Guid.NewGuid().ToString()[0..30], password = "password", email = $"{Guid.NewGuid()}@example.com", @@ -22,14 +26,36 @@ public static class Utils { public static async Task CreateGuild(string token) { using var hc = new HttpClient(); hc.DefaultRequestHeaders.Authorization = new("Bearer", token); - + Console.WriteLine("> Creating guild..."); - var guildRequest = await hc.PostAsJsonAsync("http://localhost:3001/api/v9/guilds", new { - name = Guid.NewGuid().ToString()[0..30] + var guildRequest = await hc.PostAsJsonAsync($"{Constants.ApiBaseUrl}/guilds", new { + name = Guid.NewGuid().ToString()[..30] }); - + var guildResponse = await guildRequest.Content.ReadFromJsonAsync(); var guildId = guildResponse!["id"]!.ToString(); return guildId; } + + public static async Task PostFileWithDataAsync(string url, string token, object data, byte[] file, string filename, string contentType) { + try { + using var hc = new HttpClient(); + hc.DefaultRequestHeaders.Authorization = new("Bearer", token); + var f = new MultipartFormDataContent(); + // f.Add(new StringContent()); + f.Add(JsonContent.Create(data), "payload_json"); + f.Add(new ByteArrayContent(file) { + Headers = { ContentType = new MediaTypeHeaderValue(contentType) } + }, "files[0]", filename); + + var _resp = await hc.PostAsync(url, f); + var resp = await _resp.Content.ReadAsStringAsync(); + + Console.WriteLine(resp); + } + catch (Exception e) { + Console.WriteLine(e); + throw; + } + } } \ No newline at end of file
@column.GetValue(user)