diff --git a/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/dataSources.xml b/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/dataSources.xml new file mode 100644 index 00000000..39a278fd --- /dev/null +++ b/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/dataSources.xml @@ -0,0 +1,12 @@ + + + + + postgresql + true + org.postgresql.Driver + jdbc:postgresql://localhost:5432/spacebar + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/sqldialects.xml b/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/sqldialects.xml deleted file mode 100644 index 361bbefa..00000000 --- a/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/sqldialects.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/vcs.xml b/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/vcs.xml index b2bdec2d..b7921f6f 100644 --- a/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/vcs.xml +++ b/extra/admin-api/.idea/.idea.SpacebarAdminAPI/.idea/vcs.xml @@ -2,5 +2,6 @@ + \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminAPI/Controllers/GuildController.cs b/extra/admin-api/Spacebar.AdminAPI/Controllers/GuildController.cs new file mode 100644 index 00000000..a15dd259 --- /dev/null +++ b/extra/admin-api/Spacebar.AdminAPI/Controllers/GuildController.cs @@ -0,0 +1,316 @@ +using System.Diagnostics; +using ArcaneLibs; +using ArcaneLibs.Extensions; +using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using RabbitMQ.Client; +using Spacebar.AdminApi.Models; +using Spacebar.AdminAPI.Services; +using Spacebar.Db.Contexts; +using Spacebar.Db.Models; +using Spacebar.RabbitMqUtilities; + +namespace Spacebar.AdminAPI.Controllers; + +[ApiController] +[Route("/Guilds")] +public class GuildController(ILogger logger, Configuration config, RabbitMQConfiguration amqpConfig, SpacebarDbContext db, RabbitMQService mq, IServiceProvider sp, AuthenticationService authService) : ControllerBase { + private readonly ILogger _logger = logger; + + [HttpGet] + public IAsyncEnumerable Get() { + return db.Guilds.Select(x => new GuildModel { + Id = x.Id, + AfkChannelId = x.AfkChannelId, + AfkTimeout = x.AfkTimeout, + Banner = x.Banner, + DefaultMessageNotifications = x.DefaultMessageNotifications, + Description = x.Description, + DiscoverySplash = x.DiscoverySplash, + ExplicitContentFilter = x.ExplicitContentFilter, + Features = x.Features, + PrimaryCategoryId = x.PrimaryCategoryId, + Icon = x.Icon, + Large = x.Large, + MaxMembers = x.MaxMembers, + MaxPresences = x.MaxPresences, + MaxVideoChannelUsers = x.MaxVideoChannelUsers, + MemberCount = x.MemberCount, + PresenceCount = x.PresenceCount, + TemplateId = x.TemplateId, + MfaLevel = x.MfaLevel, + Name = x.Name, + OwnerId = x.OwnerId, + PreferredLocale = x.PreferredLocale, + PremiumSubscriptionCount = x.PremiumSubscriptionCount, + PremiumTier = x.PremiumTier, + PublicUpdatesChannelId = x.PublicUpdatesChannelId, + RulesChannelId = x.RulesChannelId, + Region = x.Region, + Splash = x.Splash, + SystemChannelId = x.SystemChannelId, + SystemChannelFlags = x.SystemChannelFlags, + Unavailable = x.Unavailable, + VerificationLevel = x.VerificationLevel, + WelcomeScreen = x.WelcomeScreen, + WidgetChannelId = x.WidgetChannelId, + WidgetEnabled = x.WidgetEnabled, + NsfwLevel = x.NsfwLevel, + Nsfw = x.Nsfw, + Parent = x.Parent, + PremiumProgressBarEnabled = x.PremiumProgressBarEnabled, + ChannelOrdering = x.ChannelOrdering, + ChannelCount = x.Channels.Count(), + RoleCount = x.Roles.Count(), + EmojiCount = x.Emojis.Count(), + StickerCount = x.Stickers.Count(), + InviteCount = x.Invites.Count(), + MessageCount = x.Messages.Count(), + BanCount = x.Bans.Count(), + VoiceStateCount = x.VoiceStates.Count(), + }).AsAsyncEnumerable(); + } + + [HttpPost("{id}/force_join")] + public async Task ForceJoinGuild([FromBody] ForceJoinRequest request, string id) { + var guild = await db.Guilds.FindAsync(id); + if (guild == null) { + return NotFound(new { entity = "Guild", id, message = "Guild not found" }); + } + + var userId = request.UserId ?? config.OverrideUid ?? (await authService.GetCurrentUser(Request)).Id; + var user = await db.Users.FindAsync(userId); + if (user == null) { + return NotFound(new { entity = "User", id = userId, message = "User not found" }); + } + + var member = await db.Members.SingleOrDefaultAsync(m => m.GuildId == id && m.Id == userId); + if (member is null) { + member = new Member { + Id = userId, + GuildId = id, + JoinedAt = DateTime.UtcNow, + PremiumSince = 0, + Roles = [await db.Roles.SingleAsync(r => r.Id == id)], + Pending = false + }; + await db.Members.AddAsync(member); + guild.MemberCount++; + db.Guilds.Update(guild); + await db.SaveChangesAsync(); + } + + if (request.MakeOwner) { + guild.OwnerId = userId; + db.Guilds.Update(guild); + await db.SaveChangesAsync(); + } else if (request.MakeAdmin) { + var roles = await db.Roles.Where(r => r.GuildId == id).OrderBy(x=>x.Position).ToListAsync(); + var adminRole = roles.FirstOrDefault(r => r.Permissions == "8" || r.Permissions == "9"); // Administrator + if (adminRole == null) { + adminRole = new Role { + Id = Guid.NewGuid().ToString(), + GuildId = id, + Name = "Instance administrator", + Color = 0, + Hoist = false, + Position = roles.Max(x=>x.Position) + 1, + Permissions = "8", // Administrator + Managed = false, + Mentionable = false + }; + await db.Roles.AddAsync(adminRole); + await db.SaveChangesAsync(); + } + + if (!member.Roles.Any(r => r.Id == adminRole.Id)) { + member.Roles.Add(adminRole); + db.Members.Update(member); + await db.SaveChangesAsync(); + } + } + + // TODO: gateway events + + return Ok(new { entity = "Guild", id, message = "Guild join forced" }); + } + + [HttpGet("{id}/delete")] + public async IAsyncEnumerable DeleteUser(string id, [FromQuery] int messageDeleteChunkSize = 100) { + var user = await db.Users.FindAsync(id); + if (user == null) { + Console.WriteLine($"User {id} not found"); + yield return new AsyncActionResult("ERROR", new { entity = "User", id, message = "User not found" }); + yield break; + } + + user.Data = "{}"; + user.Deleted = true; + user.Disabled = true; + user.Rights = 0; + db.Users.Update(user); + await db.SaveChangesAsync(); + + var factory = new ConnectionFactory { + Uri = new Uri("amqp://guest:guest@127.0.0.1/") + }; + await using var mqConnection = await factory.CreateConnectionAsync(); + await using var mqChannel = await mqConnection.CreateChannelAsync(); + + var messages = db.Messages + .AsNoTracking() + .Where(m => m.AuthorId == id); + var channels = messages + .Select(m => new { m.ChannelId, m.GuildId }) + .Distinct() + .ToList(); + yield return new("STATS", + new { + total_messages = messages.Count(), total_channels = channels.Count, + messages_per_channel = channels.ToDictionary(c => c.ChannelId, c => messages.Count(m => m.ChannelId == c.ChannelId)) + }); + var results = channels + .Select(ctx => DeleteMessagesForChannel(ctx.GuildId, ctx.ChannelId!, id, mqChannel, messageDeleteChunkSize)) + .ToList(); + var a = AggregateAsyncEnumerablesWithoutOrder(results); + await foreach (var result in a) { + yield return result; + } + + await db.Database.ExecuteSqlRawAsync("VACUUM FULL messages"); + await db.Database.ExecuteSqlRawAsync("REINDEX TABLE messages"); + } + + private async IAsyncEnumerable DeleteMessagesForChannel( + // context + string? guildId, string channelId, string authorId, + // connections + IChannel mqChannel, + // options + int messageDeleteChunkSize = 100 + ) { + { + await using var ctx = sp.CreateAsyncScope(); + await using var _db = ctx.ServiceProvider.GetRequiredService(); + await mqChannel.ExchangeDeclareAsync(exchange: channelId!, type: ExchangeType.Fanout, durable: false); + var messagesInChannel = _db.Messages.AsNoTracking().Count(m => m.AuthorId == authorId && m.ChannelId == channelId && m.GuildId == guildId); + var remaining = messagesInChannel; + while (true) { + var messageIds = _db.Database.SqlQuery($""" + DELETE FROM messages + WHERE id IN ( + SELECT id FROM messages + WHERE author_id = {authorId} + AND channel_id = {channelId} + AND guild_id = {guildId} + LIMIT {messageDeleteChunkSize} + ) RETURNING id; + """).ToList(); + if (messageIds.Count == 0) { + break; + } + + var props = new BasicProperties() { Type = "MESSAGE_BULK_DELETE" }; + var publishSuccess = false; + do { + try { + await mqChannel.BasicPublishAsync(exchange: channelId!, routingKey: "", mandatory: true, basicProperties: props, body: new { + ids = messageIds, + channel_id = channelId, + guild_id = guildId, + }.ToJson().AsBytes().ToArray()); + publishSuccess = true; + } + catch (Exception e) { + Console.WriteLine($"[RabbitMQ] Error publishing bulk delete: {e.Message}"); + await Task.Delay(10); + } + } while (!publishSuccess); + + yield return new("BULK_DELETED", new { + channel_id = channelId, + total = messagesInChannel, + deleted = messageIds.Count, + remaining = remaining -= messageIds.Count, + }); + await Task.Yield(); + } + } + } + + private async IAsyncEnumerable AggregateAsyncEnumerablesWithoutOrder(params IEnumerable> enumerables) { + var enumerators = enumerables.Select(e => e.GetAsyncEnumerator()).ToList(); + var tasks = enumerators.Select(e => e.MoveNextAsync().AsTask()).ToList(); + + try { + while (tasks.Count > 0) { + var completedTask = await Task.WhenAny(tasks); + var completedTaskIndex = tasks.IndexOf(completedTask); + + if (completedTask.IsCanceled) { + try { + await enumerators[completedTaskIndex].DisposeAsync(); + } + catch { + // ignored + } + + enumerators.RemoveAt(completedTaskIndex); + tasks.RemoveAt(completedTaskIndex); + continue; + } + + if (await completedTask) { + var enumerator = enumerators[completedTaskIndex]; + yield return enumerator.Current; + tasks[completedTaskIndex] = enumerator.MoveNextAsync().AsTask(); + } + else { + try { + await enumerators[completedTaskIndex].DisposeAsync(); + } + catch { + // ignored + } + + enumerators.RemoveAt(completedTaskIndex); + tasks.RemoveAt(completedTaskIndex); + } + } + } + finally { + foreach (var enumerator in enumerators) { + try { + await enumerator.DisposeAsync(); + } + catch { + // ignored + } + } + } + } + + // { + // "op": 0, + // "t": "GUILD_ROLE_UPDATE", + // "d": { + // "guild_id": "1006649183970562092", + // "role": { + // "id": "1006706520514028812", + // "guild_id": "1006649183970562092", + // "color": 16711680, + // "hoist": true, + // "managed": false, + // "mentionable": true, + // "name": "Adminstrator", + // "permissions": "9", + // "position": 5, + // "unicode_emoji": "💖", + // "flags": 0 + // } + // }, + // "s": 38 + // } + +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs index a8501baf..811143fe 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Controllers/UserController.cs @@ -13,7 +13,7 @@ using Spacebar.RabbitMqUtilities; namespace Spacebar.AdminAPI.Controllers; [ApiController] -[Route("/Users")] +[Route("/users")] public class UserController(ILogger logger, Configuration config, RabbitMQConfiguration amqpConfig, SpacebarDbContext db, RabbitMQService mq, IServiceProvider sp) : ControllerBase { private readonly ILogger _logger = logger; diff --git a/extra/admin-api/Spacebar.AdminAPI/Program.cs b/extra/admin-api/Spacebar.AdminAPI/Program.cs index f70aa575..93b28b15 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Program.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Program.cs @@ -45,20 +45,29 @@ builder.Services.AddRequestTimeouts(x => { } }; }); -builder.Services.AddCors(options => { - options.AddPolicy( - "Open", - policy => policy.AllowAnyOrigin().AllowAnyHeader().AllowAnyMethod()); -}); +// builder.Services.AddCors(options => { +// options.AddPolicy( +// "Open", +// policy => policy.AllowAnyOrigin().AllowAnyHeader().AllowAnyMethod()); +// }); var app = builder.Build(); +app.Use((context, next) => { + context.Response.Headers["Access-Control-Allow-Origin"] = "*"; + context.Response.Headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"; + context.Response.Headers["Access-Control-Allow-Headers"] = "*, Authorization"; + if (context.Request.Method == "OPTIONS") { + context.Response.StatusCode = 200; + return Task.CompletedTask; + } + + return next(); +}); app.UsePathBase("/_spacebar/admin"); -app.UseCors("Open"); +// app.UseCors("Open"); // Configure the HTTP request pipeline. -if (app.Environment.IsDevelopment()) { - app.MapOpenApi(); -} +app.MapOpenApi(); app.UseMiddleware(); app.UseAuthorization(); diff --git a/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs b/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs index 53d2cf21..269935bf 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs +++ b/extra/admin-api/Spacebar.AdminAPI/Services/AuthenticationService.cs @@ -13,6 +13,7 @@ public class AuthenticationService(SpacebarDbContext db, Configuration config) { public async Task GetCurrentUser(HttpRequest request) { if (!request.Headers.ContainsKey("Authorization")) { + Console.WriteLine(string.Join(", ", request.Headers.Keys)); throw new UnauthorizedAccessException(); } @@ -25,7 +26,7 @@ public class AuthenticationService(SpacebarDbContext db, Configuration config) { var res = await handler.ValidateTokenAsync(token, new TokenValidationParameters { IssuerSigningKey = new ECDsaSecurityKey(key), - ValidAlgorithms = new[] { "ES512" }, + ValidAlgorithms = ["ES512"], LogValidationExceptions = true, // These are required to be false for the token to be valid as they aren't provided by the token ValidateIssuer = false, @@ -33,7 +34,7 @@ public class AuthenticationService(SpacebarDbContext db, Configuration config) { ValidateAudience = false, }); - if (!res.IsValid) { + if (!res.IsValid && !config.DisableAuthentication) { throw new UnauthorizedAccessException(); } diff --git a/extra/admin-api/Spacebar.AdminAPI/Spacebar.AdminAPI.csproj b/extra/admin-api/Spacebar.AdminAPI/Spacebar.AdminAPI.csproj index e74d8b50..53f82843 100644 --- a/extra/admin-api/Spacebar.AdminAPI/Spacebar.AdminAPI.csproj +++ b/extra/admin-api/Spacebar.AdminAPI/Spacebar.AdminAPI.csproj @@ -7,11 +7,11 @@ - - + + - - + + diff --git a/extra/admin-api/Spacebar.AdminAPI/appsettings.Development.json b/extra/admin-api/Spacebar.AdminAPI/appsettings.Development.json index 5a367f27..03245773 100644 --- a/extra/admin-api/Spacebar.AdminAPI/appsettings.Development.json +++ b/extra/admin-api/Spacebar.AdminAPI/appsettings.Development.json @@ -2,10 +2,26 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Microsoft.AspNetCore": "Trace", //Warning + "Microsoft.AspNetCore.Mvc": "Warning", //Warning + "Microsoft.AspNetCore.HostFiltering": "Warning", //Warning + "Microsoft.AspNetCore.Cors": "Warning", //Warning + // "Microsoft.EntityFrameworkCore": "Warning" + "Microsoft.EntityFrameworkCore.Database.Command": "Debug" } }, "ConnectionStrings": { - "Spacebar": "Host=127.0.0.1; Username=postgres; Database=spacebar" + "Spacebar": "Host=127.0.0.1; Username=postgres; Database=spacebar; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;", + }, + "RabbitMQ": { + "Host": "127.0.0.1", + "Port": 5673, + "Username": "guest", + "Password": "guest" + }, + "SpacebarAdminApi": { + "Enforce2FA": true, + "OverrideUid": null, + "DisableAuthentication": false } } diff --git a/extra/admin-api/Spacebar.AdminApi.Models/ForceJoinRequest.cs b/extra/admin-api/Spacebar.AdminApi.Models/ForceJoinRequest.cs new file mode 100644 index 00000000..60bd6bce --- /dev/null +++ b/extra/admin-api/Spacebar.AdminApi.Models/ForceJoinRequest.cs @@ -0,0 +1,7 @@ +namespace Spacebar.AdminApi.Models; + +public class ForceJoinRequest { + public string? UserId { get; set; } = null!; + public bool MakeAdmin { get; set; } = false; + public bool MakeOwner { get; set; } = false; +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminApi.Models/GuildModel.cs b/extra/admin-api/Spacebar.AdminApi.Models/GuildModel.cs new file mode 100644 index 00000000..aa67ac38 --- /dev/null +++ b/extra/admin-api/Spacebar.AdminApi.Models/GuildModel.cs @@ -0,0 +1,53 @@ +namespace Spacebar.AdminApi.Models; + +public class GuildModel { + public string Id { get; set; } = null!; + public string? AfkChannelId { get; set; } + public int? AfkTimeout { get; set; } + public string? Banner { get; set; } + public int? DefaultMessageNotifications { get; set; } + public string? Description { get; set; } + public string? DiscoverySplash { get; set; } + public int? ExplicitContentFilter { get; set; } + public string Features { get; set; } = null!; + public string? PrimaryCategoryId { get; set; } + public string? Icon { get; set; } + public bool Large { get; set; } + public int? MaxMembers { get; set; } + public int? MaxPresences { get; set; } + public int? MaxVideoChannelUsers { get; set; } + public int? MemberCount { get; set; } + public int? PresenceCount { get; set; } + public string? TemplateId { get; set; } + public int? MfaLevel { get; set; } + public string Name { get; set; } = null!; + public string? OwnerId { get; set; } + public string? PreferredLocale { get; set; } + public int? PremiumSubscriptionCount { get; set; } + public int PremiumTier { get; set; } + public string? PublicUpdatesChannelId { get; set; } + public string? RulesChannelId { get; set; } + public string? Region { get; set; } + public string? Splash { get; set; } + public string? SystemChannelId { get; set; } + public int? SystemChannelFlags { get; set; } + public bool Unavailable { get; set; } + public int? VerificationLevel { get; set; } + public string WelcomeScreen { get; set; } = null!; + public string? WidgetChannelId { get; set; } + public bool WidgetEnabled { get; set; } + public int? NsfwLevel { get; set; } + public bool Nsfw { get; set; } + public string? Parent { get; set; } + public bool? PremiumProgressBarEnabled { get; set; } + public string ChannelOrdering { get; set; } = null!; + + public int ChannelCount { get; set; } + public int RoleCount { get; set; } + public int EmojiCount { get; set; } + public int StickerCount { get; set; } + public int InviteCount { get; set; } + public int MessageCount { get; set; } + public int BanCount { get; set; } + public int VoiceStateCount { get; set; } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.AdminApi.Models/UserModel.cs b/extra/admin-api/Spacebar.AdminApi.Models/UserModel.cs index 0ed66fb9..e1c0b9d2 100644 --- a/extra/admin-api/Spacebar.AdminApi.Models/UserModel.cs +++ b/extra/admin-api/Spacebar.AdminApi.Models/UserModel.cs @@ -1,4 +1,6 @@ -namespace Spacebar.AdminApi.Models; +using System.Text.Json.Serialization; + +namespace Spacebar.AdminApi.Models; public class UserModel { public string Id { get; set; } = null!; @@ -26,9 +28,16 @@ public class UserModel { public bool Disabled { get; set; } public bool Deleted { get; set; } public string? Email { get; set; } + + [JsonNumberHandling(JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString)] public ulong Flags { get; set; } + + [JsonNumberHandling(JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString)] public ulong PublicFlags { get; set; } + + [JsonNumberHandling(JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString)] public ulong Rights { get; set; } + public ApplicationModel? ApplicationBotUser { get; set; } public List ConnectedAccounts { get; set; } = new(); public int GuildCount { get; set; } diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/Program.cs b/extra/admin-api/Spacebar.CleanSettingsRows/Program.cs new file mode 100644 index 00000000..cb99a17a --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/Program.cs @@ -0,0 +1,16 @@ +using Microsoft.EntityFrameworkCore; +using Spacebar.CleanSettingsRows; +using Spacebar.Db.Contexts; + +var builder = Host.CreateApplicationBuilder(args); +builder.Services.AddHostedService(); + +builder.Services.AddDbContextPool(options => { + options + .UseNpgsql(builder.Configuration.GetConnectionString("Spacebar")) + .EnableDetailedErrors(); +}); + + +var host = builder.Build(); +host.Run(); \ No newline at end of file diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/Properties/launchSettings.json b/extra/admin-api/Spacebar.CleanSettingsRows/Properties/launchSettings.json new file mode 100644 index 00000000..0e86192b --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/Properties/launchSettings.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "Spacebar.CleanSettingsRows": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + }, + "Local": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Local" + } + } + } +} diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/Spacebar.CleanSettingsRows.csproj b/extra/admin-api/Spacebar.CleanSettingsRows/Spacebar.CleanSettingsRows.csproj new file mode 100644 index 00000000..527e8ad6 --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/Spacebar.CleanSettingsRows.csproj @@ -0,0 +1,17 @@ + + + + net9.0 + enable + enable + dotnet-Spacebar.CleanSettingsRows-18acacc0-bc7c-411f-ba03-05f1645e86ec + + + + + + + + + + diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/Worker.cs b/extra/admin-api/Spacebar.CleanSettingsRows/Worker.cs new file mode 100644 index 00000000..fd977d6e --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/Worker.cs @@ -0,0 +1,93 @@ +using System.Runtime.CompilerServices; +using Microsoft.EntityFrameworkCore; +using Spacebar.Db.Contexts; +using Spacebar.Db.Models; + +namespace Spacebar.CleanSettingsRows; + +public class Worker(ILogger logger, IServiceProvider sp) : BackgroundService { + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + logger.LogInformation("Starting settings row cleanup worker"); + + using var scope = sp.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + List tasks = []; + await foreach (var chunk in GetChunks(db, 1000, stoppingToken)) { + tasks.Add(ProcessChunk(chunk, stoppingToken)); + } + + await Task.WhenAll(tasks); + + logger.LogInformation("Finished settings row cleanup worker"); + } + + private async IAsyncEnumerable GetChunks(SpacebarDbContext db, int chunkSize, [EnumeratorCancellation] CancellationToken stoppingToken) { + var total = await db.UserSettings.CountAsync(stoppingToken); + for (var i = 0; i < total; i += chunkSize) { + var chunk = await db.UserSettings + .Include(x => x.User) + .OrderBy(x => x.Index) + .Skip(i) + .Take(chunkSize) + .ToArrayAsync(stoppingToken); + yield return chunk; + } + } + + private async Task ProcessChunk(UserSetting[] chunk, CancellationToken stoppingToken) { + if (chunk.Length == 0) return; + logger.LogInformation("Processing chunk of {Count} settings rows starting at idx={}", chunk.Length, chunk[0].Index); + var scope = sp.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + foreach (var setting in chunk) { + if (stoppingToken.IsCancellationRequested) break; + if (setting.User == null) { + logger.LogInformation("Deleting orphaned settings row {Id}", setting.Index); + db.UserSettings.Remove(setting); + } + else if (setting is { + // default settings + AfkTimeout: 3600, + AllowAccessibilityDetection: true, + AnimateEmoji: true, + AnimateStickers: 0, + ContactSyncEnabled: false, + ConvertEmoticons: false, + CustomStatus: null, + DefaultGuildsRestricted: false, + DetectPlatformAccounts: false, + DeveloperMode: true, + DisableGamesTab: true, + EnableTtsCommand: false, + ExplicitContentFilter: 0, + FriendSourceFlags: "{\"all\":true}", + GatewayConnected: false, + GifAutoPlay: false, + GuildFolders: "[]", + GuildPositions: "[]", + InlineAttachmentMedia: true, + InlineEmbedMedia: true, + MessageDisplayCompact: false, + NativePhoneIntegrationEnabled: true, + RenderEmbeds: true, + RenderReactions: true, + RestrictedGuilds: "[]", + ShowCurrentGame: true, + Status: "online", + StreamNotificationsEnabled: false, + Theme: "dark", + TimezoneOffset: 0, + FriendDiscoveryFlags: 0, + ViewNsfwGuilds: true, + // only different property: + //Locale: "en-US" + }) { + logger.LogInformation("Deleting default settings row {Id} for user {UserId}", setting.Index, setting.User.Id); + setting.User.SettingsIndex = null; + db.UserSettings.Remove(setting); + } + } + + await db.SaveChangesAsync(stoppingToken); + } +} \ No newline at end of file diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.Development.json b/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.Development.json new file mode 100644 index 00000000..e8136888 --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.Development.json @@ -0,0 +1,11 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "ConnectionStrings": { + "Spacebar": "Host=127.0.0.1; Username=postgres; Database=spacebar; Port=5432; Include Error Detail=true; Maximum Pool Size=1000; Command Timeout=6000; Timeout=600;" + } +} diff --git a/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.json b/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.json new file mode 100644 index 00000000..b2dcdb67 --- /dev/null +++ b/extra/admin-api/Spacebar.CleanSettingsRows/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/extra/admin-api/SpacebarAdminAPI.sln b/extra/admin-api/SpacebarAdminAPI.sln index 74d9de83..09e3a696 100644 --- a/extra/admin-api/SpacebarAdminAPI.sln +++ b/extra/admin-api/SpacebarAdminAPI.sln @@ -19,6 +19,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.AdminApi.Models", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.AdminApi.PrepareTestData", "Utilities\Spacebar.AdminApi.PrepareTestData\Spacebar.AdminApi.PrepareTestData.csproj", "{BCC6501C-16A7-4787-BA47-52DAE06718A8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spacebar.CleanSettingsRows", "Spacebar.CleanSettingsRows\Spacebar.CleanSettingsRows.csproj", "{9B41FAC1-4427-487C-BF3F-69554848DEBF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -56,6 +58,10 @@ Global {BCC6501C-16A7-4787-BA47-52DAE06718A8}.Debug|Any CPU.Build.0 = Debug|Any CPU {BCC6501C-16A7-4787-BA47-52DAE06718A8}.Release|Any CPU.ActiveCfg = Release|Any CPU {BCC6501C-16A7-4787-BA47-52DAE06718A8}.Release|Any CPU.Build.0 = Release|Any CPU + {9B41FAC1-4427-487C-BF3F-69554848DEBF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9B41FAC1-4427-487C-BF3F-69554848DEBF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B41FAC1-4427-487C-BF3F-69554848DEBF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9B41FAC1-4427-487C-BF3F-69554848DEBF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {F4E179D1-A3EB-4A1D-9C11-E7019F4B1EE2} = {04787943-EBB6-4DE4-96D5-4CFB4A2CEE99} diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Classes/OpenAPI/OpenAPISchema.cs b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Classes/OpenAPI/OpenAPISchema.cs new file mode 100644 index 00000000..8e399302 --- /dev/null +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Classes/OpenAPI/OpenAPISchema.cs @@ -0,0 +1,352 @@ +using System.Collections.Frozen; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Spacebar.AdminAPI.TestClient.Classes.OpenAPI; + +public class OpenApiSchema { + [JsonPropertyName("openapi")] + public string Version { get; set; } = null!; + + [JsonPropertyName("info")] + public OpenApiInfo Info { get; set; } = null!; + + [JsonPropertyName("externalDocs")] + public OpenApiExternalDocs? ExternalDocs { get; set; } + + [JsonPropertyName("paths")] + public Dictionary Paths { get; set; } = null!; + + [JsonPropertyName("servers")] + public List Servers { get; set; } = null!; + + [JsonPropertyName("components")] + public OpenApiComponents? Components { get; set; } = null!; + + public class OpenApiComponents { + [JsonPropertyName("schemas")] + public Dictionary? Schemas { get; set; } = null!; + } +} + +public class OpenApiServer { + [JsonPropertyName("url")] + public string Url { get; set; } = null!; + + [JsonPropertyName("description")] + public string? Description { get; set; } +} + +public class OpenApiPath { + public FrozenSet GetAvailableMethods() { + List methods = new(); + if (Get != null) methods.Add("GET"); + if (Post != null) methods.Add("POST"); + if (Put != null) methods.Add("PUT"); + if (Delete != null) methods.Add("DELETE"); + if (Patch != null) methods.Add("PATCH"); + if (Options != null) methods.Add("OPTIONS"); + return methods.ToFrozenSet(); + } + + public bool HasMethod(string method) { + return method.ToLower() switch { + "get" => Get != null, + "post" => Post != null, + "put" => Put != null, + "delete" => Delete != null, + "patch" => Patch != null, + "options" => Options != null, + _ => false + }; + } + + public OpenApiOperation? GetOperation(string method) { + if (!HasMethod(method)) return null; + return method.ToLower() switch { + "get" => Get, + "post" => Post, + "put" => Put, + "delete" => Delete, + "patch" => Patch, + "options" => Options, + _ => null + }; + } + + [JsonPropertyName("get")] + public OpenApiOperation? Get { get; set; } + + [JsonPropertyName("post")] + public OpenApiOperation? Post { get; set; } + + [JsonPropertyName("put")] + public OpenApiOperation? Put { get; set; } + + [JsonPropertyName("delete")] + public OpenApiOperation? Delete { get; set; } + + [JsonPropertyName("patch")] + public OpenApiOperation? Patch { get; set; } + + [JsonPropertyName("options")] + public OpenApiOperation? Options { get; set; } + + public class OpenApiOperation { + [JsonPropertyName("description")] + public string Description { get; set; } = null!; + + [JsonPropertyName("parameters")] + public List? Parameters { get; set; } + + [JsonPropertyName("requestBody")] + public OpenApiRequestBody? RequestBody { get; set; } + + public class OpenApiParameter { + [JsonPropertyName("name")] + public string Name { get; set; } = null!; + + [JsonPropertyName("in")] + public string In { get; set; } = null!; + + [JsonPropertyName("required")] + public bool Required { get; set; } + + [JsonPropertyName("schema")] + public OpenApiSchemaRef Schema { get; set; } = null!; + + [JsonPropertyName("description")] + public string? Description { get; set; } + } + } +} + +public class OpenApiExternalDocs { + [JsonPropertyName("description")] + public string Description { get; set; } = null!; + + [JsonPropertyName("url")] + public string Url { get; set; } = null!; +} + +public class OpenApiInfo { + [JsonPropertyName("title")] + public string Title { get; set; } = null!; + + [JsonPropertyName("version")] + public string Version { get; set; } = null!; + + [JsonPropertyName("description")] + public string Description { get; set; } = null!; + + [JsonPropertyName("license")] + public OpenApiLicense License { get; set; } = null!; + + public class OpenApiLicense { + [JsonPropertyName("name")] + public string Name { get; set; } = null!; + + [JsonPropertyName("url")] + public string Url { get; set; } = null!; + } +} + +public class OpenApiRequestBody { + [JsonPropertyName("content")] + public OpenApiContent Content { get; set; } = null!; + + [JsonPropertyName("required")] + public bool Required { get; set; } +} + +public class OpenApiContent { + [JsonPropertyName("application/json")] + public OpenApiSchemaContainer? ApplicationJson { get; set; } + + public class OpenApiSchemaContainer { + [JsonPropertyName("schema")] + public OpenApiSchemaRef Schema { get; set; } = null!; + } +} + +[JsonConverter(typeof(OpenApiSchemaRefConverter))] +public class OpenApiSchemaRef { + public string? Description { get; set; } + public string? Type { get; set; } = null!; + public List? Types { get; set; } = null!; + public string? Ref { get; set; } = null!; + public Dictionary? Properties { get; set; } + public List? Required { get; set; } + public int? MinLength { get; set; } + public int? MaxLength { get; set; } + public int? MinItems { get; set; } + public int? MaxItems { get; set; } + public object? Constant { get; set; } + public object? Default { get; set; } + public bool Nullable { get; set; } + public List? AnyOf { get; set; } + public List? Enum { get; set; } + public string? Format { get; set; } + + public OpenApiSchemaRef? GetReferencedSchema(OpenApiSchema schema) { + if (Ref == null) return null; + string refKey = Ref.Replace("#/components/schemas/", ""); + if (schema.Components?.Schemas != null && schema.Components.Schemas.TryGetValue(refKey, out var referencedSchema)) { + return referencedSchema; + } + + throw new KeyNotFoundException($"Referenced schema '{refKey}' not found."); + } +} + +public class OpenApiSchemaRefConverter : JsonConverter { + public override OpenApiSchemaRef? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.TokenType != JsonTokenType.StartObject) { + throw new JsonException("Expected StartObject token"); + } + + using var jsonDoc = JsonDocument.ParseValue(ref reader); + var jsonObject = jsonDoc.RootElement; + var schemaRef = new OpenApiSchemaRef(); + + foreach (var property in jsonObject.EnumerateObject()) { + switch (property.Name) { + case "type": + if (property.Value.ValueKind == JsonValueKind.String) { + schemaRef.Type = property.Value.GetString(); + } + else if (property.Value.ValueKind == JsonValueKind.Array) { + var types = new List(); + foreach (var item in property.Value.EnumerateArray()) { + if (item.ValueKind == JsonValueKind.String) { + types.Add(item.GetString()!); + } + else throw new JsonException("Expected string in type array"); + } + + schemaRef.Types = types; + } + + break; + case "$ref": + schemaRef.Ref = property.Value.GetString(); + break; + case "description": + schemaRef.Description = property.Value.GetString(); + break; + case "properties": + schemaRef.Properties = property.Value.EnumerateObject().ToDictionary(x => x.Name, x => x.Value.Deserialize(options)!); + break; + case "required": + schemaRef.Required = property.Value.EnumerateArray().Select(item => item.GetString()!).ToList(); + break; + case "minLength": + schemaRef.MinLength = property.Value.GetInt32(); + break; + case "maxLength": + schemaRef.MaxLength = property.Value.GetInt32(); + break; + case "minItems": + schemaRef.MinItems = property.Value.GetInt32(); + break; + case "maxItems": + schemaRef.MaxItems = property.Value.GetInt32(); + break; + case "const": + if (property.Value.ValueKind == JsonValueKind.String) + schemaRef.Constant = property.Value.GetString(); + else if (property.Value.ValueKind == JsonValueKind.Number) + schemaRef.Constant = property.Value.GetInt32(); + else if (property.Value.ValueKind is JsonValueKind.True or JsonValueKind.False) + schemaRef.Constant = property.Value.GetBoolean(); + else throw new JsonException($"Expected string|int|bool in const, got {property.Value.ValueKind}"); + break; + case "default": + if (property.Value.ValueKind == JsonValueKind.String) + schemaRef.Default = property.Value.GetString(); + else if (property.Value.ValueKind == JsonValueKind.Number) + schemaRef.Default = property.Value.GetInt32(); + else if (property.Value.ValueKind is JsonValueKind.True or JsonValueKind.False) + schemaRef.Default = property.Value.GetBoolean(); + else if (property.Value.ValueKind == JsonValueKind.Null) + schemaRef.Default = null; + else if (property.Value.ValueKind == JsonValueKind.Array) + if (property.Value.GetArrayLength() > 0) throw new JsonException("Expected empty array in default"); + else schemaRef.Default = Array.Empty(); + else throw new JsonException($"Expected string|int|bool|null in default, got {property.Value.ValueKind}"); + break; + case "enum": + var enumValues = new List(); + foreach (var item in property.Value.EnumerateArray()) { + if (item.ValueKind == JsonValueKind.String) + enumValues.Add(item.GetString()!); + else if (item.ValueKind == JsonValueKind.Number) + enumValues.Add(item.GetInt32()); + else if (item.ValueKind is JsonValueKind.True or JsonValueKind.False) + enumValues.Add(item.GetBoolean()); + else if (item.ValueKind == JsonValueKind.Null) + enumValues.Add(null!); + else throw new JsonException($"Expected string|int|bool|null in enum, got {item.ValueKind}"); + } + + schemaRef.Enum = enumValues; + break; + case "nullable": + schemaRef.Nullable = property.Value.GetBoolean(); + break; + case "anyOf": + schemaRef.AnyOf = property.Value.EnumerateArray().Select(item => item.Deserialize(options)!).ToList(); + break; + case "format": + schemaRef.Format = property.Value.GetString(); + break; + case "additionalProperties": //TODO + case "patternProperties": // Side effect of using JsonValue in typescript + break; + default: + Console.WriteLine($"Got unexpected prop {property.Name} in OpenApiSchemaRef!"); + break; + } + } + + return schemaRef; + } + + public override void Write(Utf8JsonWriter writer, OpenApiSchemaRef value, JsonSerializerOptions options) { + // throw new NotImplementedException("Serialization not implemented for OpenApiSchemaRef"); + writer.WriteStartObject(); + if (value.Type != null) { + writer.WriteString("type", value.Type); + } + else if (value.Types != null) { + writer.WritePropertyName("type"); + writer.WriteStartArray(); + foreach (var type in value.Types) { + writer.WriteStringValue(type); + } + + writer.WriteEndArray(); + } + + if (value.Ref != null) { + writer.WriteString("$ref", value.Ref); + } + + if (value.Description != null) { + writer.WriteString("description", value.Description); + } + + if (value.Properties != null) { + writer.WritePropertyName("properties"); + writer.WriteStartObject(); + foreach (var prop in value.Properties) { + writer.WritePropertyName(prop.Key); + JsonSerializer.Serialize(writer, prop.Value, options); + } + + writer.WriteEndObject(); + } + + writer.WriteEndObject(); + } +} \ No newline at end of file diff --git a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor index 732d46e7..c96fa30c 100644 --- a/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor +++ b/extra/admin-api/Utilities/Spacebar.AdminAPI.TestClient/Layout/NavMenu.razor @@ -14,6 +14,11 @@ Home +