From c168b8faf99e861366648b423a6198b85578e9bc Mon Sep 17 00:00:00 2001 From: Rory& Date: Thu, 23 Oct 2025 22:27:46 +0200 Subject: [PATCH] Improve openapi schema generation script --- scripts/schema.js | 176 ++++++++++------- scripts/schemaExclusions.json | 345 ++++++++++++++++++++++++++++++++++ 2 files changed, 449 insertions(+), 72 deletions(-) create mode 100644 scripts/schemaExclusions.json diff --git a/scripts/schema.js b/scripts/schema.js index 77205831..e5e52dea 100644 --- a/scripts/schema.js +++ b/scripts/schema.js @@ -20,19 +20,23 @@ Regenerates the `spacebarchat/server/assets/schemas.json` file, used for API/Gateway input validation. */ +const scriptStartTime = new Date(); + const conWarn = console.warn; console.warn = (...args) => { // silence some expected warnings if (args[0] === "initializer is expression for property id") return; if (args[0].startsWith("unknown initializer for property ") && args[0].endsWith("[object Object]")) return; conWarn(...args); -} +}; const path = require("path"); const fs = require("fs"); const TJS = require("typescript-json-schema"); const walk = require("./util/walk"); +const { redBright, yellowBright, bgRedBright, yellow } = require("picocolors"); const schemaPath = path.join(__dirname, "..", "assets", "schemas.json"); +const exclusionList = JSON.parse(fs.readFileSync(path.join(__dirname, "schemaExclusions.json"), { encoding: "utf8" })); const settings = { required: true, @@ -43,82 +47,102 @@ const settings = { defaultProps: false, }; -const ExcludeAndWarn = [ - /^Record/, - /^Partial/, -] -const Excluded = [ - "DefaultSchema", - "Schema", - "EntitySchema", - "ServerResponse", - "Http2ServerResponse", - "ExpressResponse", - "global.Express.Response", - "global.Response", - "Response", - "e.Response", - "request.Response", - "supertest.Response", - "DiagnosticsChannel.Response", - "_Response", - "ReadableStream", +const ExcludeAndWarn = [...exclusionList.manualWarn, ...exclusionList.manualWarnRe.map((r) => new RegExp(r))]; +const Excluded = [...exclusionList.manual, ...exclusionList.manualRe.map((r) => new RegExp(r)), ...exclusionList.auto.map((r) => r.value)]; +const Included = [...exclusionList.include, ...exclusionList.includeRe.map((r) => new RegExp(r))]; - // TODO: Figure out how to exclude schemas from node_modules? - "SomeJSONSchema", - "UncheckedPartialSchema", - "PartialSchema", - "UncheckedPropertiesSchema", - "PropertiesSchema", - "AsyncSchema", - "AnySchema", - "SMTPConnection.CustomAuthenticationResponse", - "TransportMakeRequestResponse", - // Emma [it/its] @ Rory& - 2025-10-14 - /.*\..*/, - /^Axios.*/, - /^APIKeyConfiguration\..*/, - /^AccountSetting\..*/, - /^BulkContactManagement\..*/, - /^Campaign.*/, - /^Contact.*/, - /^DNS\..*/, - /^Delete.*/, - /^Destroy.*/, - /^Template\..*/, - /^Webhook\..*/, - /^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric)Schema/, - /^Put/ +const excludedLambdas = [ + (n, s) => { + // attempt to import + if (JSON.stringify(s).includes(`#/definitions/import(`)) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it attempted to use import().`); + exclusionList.auto.push({ value: n, reason: "Uses import()" }); + return true; + } + }, + (n, s) => { + if (JSON.stringify(s).includes(process.cwd())) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked $PWD.`); + exclusionList.auto.push({ value: n, reason: "Leaked $PWD" }); + return true; + } + }, + (n, s) => { + if (JSON.stringify(s).includes(process.env.HOME)) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked a $HOME path.`); + exclusionList.auto.push({ value: n, reason: "Leaked $HOME" }); + return true; + } + }, + (n, s) => { + if (s["$ref"] === `#/definitions/${n}`) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it is a self-reference only schema.`); + exclusionList.auto.push({ value: n, reason: "Self-reference only schema" }); + // fs.writeFileSync(`fucked/${n}.json`, JSON.stringify(s, null, 4)); + return true; + } + }, + (n, s) => { + if (s.description?.match(/Smithy/)) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it appears to be an AWS Smithy schema.`); + exclusionList.auto.push({ value: n, reason: "AWS Smithy schema" }); + return true; + } + }, + (n, s) => { + if (s.description?.startsWith("

")) { + console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as we don't use HTML paragraphs for descriptions.`); + exclusionList.auto.push({ value: n, reason: "HTML paragraph in description" }); + return true; + } + }, + // (n, s) => { + // if (JSON.stringify(s).length <= 300) { + // console.log({n, s}); + // } + // } ]; +function includesMatch(haystack, needles, log = false) { + for (const needle of needles) { + const match = needle instanceof RegExp ? needle.test(haystack) : haystack === needle; + if (match) { + if (log) console.warn(redBright("[WARN]:"), "Excluding schema", haystack, "due to match with", needle); + return true; + } + } + return false; +} + function main() { - const program = TJS.programFromConfig( - path.join(__dirname, "..", "tsconfig.json"), - walk(path.join(__dirname, "..", "src", "schemas")), - ); + const program = TJS.programFromConfig(path.join(__dirname, "..", "tsconfig.json"), walk(path.join(__dirname, "..", "src", "schemas"))); const generator = TJS.buildGenerator(program, settings); if (!generator || !program) return; let schemas = generator.getUserSymbols().filter((x) => { return ( - ( - x.endsWith("Schema") - ||x.endsWith("Response") - || x.startsWith("API") - ) - && !ExcludeAndWarn.some(exc => { - const match = exc instanceof RegExp ? exc.test(x) : x === exc; - if (match) console.warn("Warning: Excluding schema", x); - return match; - }) - && !Excluded.some(exc => exc instanceof RegExp ? exc.test(x) : x === exc) + (x.endsWith("Schema") || x.endsWith("Response") || x.startsWith("API")) && + // !ExcludeAndWarn.some((exc) => { + // const match = exc instanceof RegExp ? exc.test(x) : x === exc; + // if (match) console.warn("Warning: Excluding schema", x); + // return match; + // }) && + // !Excluded.some((exc) => (exc instanceof RegExp ? exc.test(x) : x === exc)) + (includesMatch(x, Included) || (!includesMatch(x, ExcludeAndWarn, true) && !includesMatch(x, Excluded))) ); }); + //.sort((a,b) => a.localeCompare(b)); var definitions = {}; + if (process.env.WRITE_SCHEMA_DIR === "true") { + fs.rmSync("schemas", { recursive: true, force: true }); + fs.mkdirSync("schemas"); + } + for (const name of schemas) { - console.log("Processing schema", name); + const startTime = new Date(); + process.stdout.write(`Processing schema ${name}... `); const part = TJS.generateSchema(program, name, settings, [], generator); if (!part) continue; @@ -144,27 +168,35 @@ function main() { } } + if (definitions[name]) { + process.stdout.write(yellow(` [ERROR] Duplicate schema name detected: ${name}. Overwriting previous schema.`)); + } + + if (!includesMatch(name, Included) && excludedLambdas.some((fn) => fn(name, part))) { + continue; + } + + if (process.env.WRITE_SCHEMA_DIR === "true") fs.writeFileSync(path.join("schemas", `${name}.json`), JSON.stringify(part, null, 4)); + + process.stdout.write("Done in " + yellowBright(new Date() - startTime) + " ms, " + yellowBright(JSON.stringify(part).length) + " bytes (unformatted) "); + if (new Date() - startTime >= 20) console.log(bgRedBright("[SLOW]")); + else console.log(); + definitions = { ...definitions, [name]: { ...part } }; } deleteOneOfKindUndefinedRecursive(definitions, "$"); fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4)); - console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath); + fs.writeFileSync(__dirname + "/schemaExclusions.json", JSON.stringify(exclusionList, null, 4)); + console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath, "in", new Date() - scriptStartTime, "ms."); } function deleteOneOfKindUndefinedRecursive(obj, path) { - if ( - obj?.type === "object" && - obj?.properties?.oneofKind?.type === "undefined" - ) - return true; + if (obj?.type === "object" && obj?.properties?.oneofKind?.type === "undefined") return true; for (const key in obj) { - if ( - typeof obj[key] === "object" && - deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key) - ) { + if (typeof obj[key] === "object" && deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key)) { console.log("Deleting", path, key); delete obj[key]; } diff --git a/scripts/schemaExclusions.json b/scripts/schemaExclusions.json new file mode 100644 index 00000000..795e4547 --- /dev/null +++ b/scripts/schemaExclusions.json @@ -0,0 +1,345 @@ +{ + "include": [ + "MessageInteractionSchema" + ], + "includeRe": [], + "manual": [ + "DefaultSchema", + "Schema", + "EntitySchema", + "ReadableStream", + "SomeJSONSchema", + "UncheckedPartialSchema", + "PartialSchema", + "UncheckedPropertiesSchema", + "PropertiesSchema", + "AsyncSchema", + "AnySchema", + "SMTPConnection.CustomAuthenticationResponse", + "TransportMakeRequestResponse", + "StaticSchema", + "CSVImportResponse" + ], + "manualRe": [ + ".*\\.Response$", + "^(Http2Server|Server|Express|(Resolved|)Http|Client|_|)Response$", + ".*\\..*", + "^Axios.*", + "^ListContact(s|Lists)Response$", + "^APIKeyConfiguration\\..*", + "^AccountSetting\\..*", + "^BulkContactManagement\\..*", + "^Campaign.*", + "^Contact.*", + "^DNS\\..*", + "^Delete.*", + "^Destroy.*", + "^Template\\..*", + "^Webhook\\..*", + "^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric|StreamingBlob|TimestampDateTime|TimestampHttpDate|TimestampEpochSeconds|Simple)Schema", + "^((Create|Update)Contact(|List))Response$", + "^(T|Unt)agResourceResponse$", + "^Put", + "^Inbox", + "^Seed", + "^DomainTag", + "^IpPool", + "DomainTemplate", + "^\\$", + "^Suppression", + "^Mail(|ing)List", + "DomainTracking", + "UpdatedDomain", + "ConfigurationSet", + "ContactList", + "^IPR", + "^Job" + ], + "manualWarn": [], + "manualWarnRe": [ + "^Record", + "^Partial" + ], + "auto": [ + { + "value": "StringSchema", + "reason": "AWS Smithy schema" + }, + { + "value": "TimestampDefaultSchema", + "reason": "AWS Smithy schema" + }, + { + "value": "StaticSimpleSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StaticListSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StaticMapSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StaticStructureSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StaticErrorSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StaticOperationSchema", + "reason": "Self-reference only schema" + }, + { + "value": "UnitSchema", + "reason": "AWS Smithy schema" + }, + { + "value": "MemberSchema", + "reason": "Self-reference only schema" + }, + { + "value": "StructureSchema", + "reason": "Self-reference only schema" + }, + { + "value": "OperationSchema", + "reason": "Self-reference only schema" + }, + { + "value": "BatchGetMetricDataResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CancelExportJobResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateCustomVerificationEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateDedicatedIpPoolResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateDeliverabilityTestReportResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateEmailIdentityResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateEmailIdentityPolicyResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateExportJobResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateImportJobResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateMultiRegionEndpointResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateTenantResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "CreateTenantResourceAssociationResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetAccountResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetBlacklistReportsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetCustomVerificationEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDedicatedIpResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDedicatedIpPoolResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDedicatedIpsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDeliverabilityDashboardOptionsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDeliverabilityTestReportResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDomainDeliverabilityCampaignResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetDomainStatisticsReportResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetEmailIdentityResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetEmailIdentityPoliciesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetExportJobResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetImportJobResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetMessageInsightsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetMultiRegionEndpointResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetReputationEntityResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetSuppressedDestinationResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "GetTenantResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListCustomVerificationEmailTemplatesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListDedicatedIpPoolsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListDeliverabilityTestReportsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListDomainDeliverabilityCampaignsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListEmailIdentitiesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListEmailTemplatesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListExportJobsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListImportJobsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListMultiRegionEndpointsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListRecommendationsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListReputationEntitiesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListResourceTenantsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListSuppressedDestinationsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListTenantResourcesResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "ListTenantsResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "SendBulkEmailResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "SendCustomVerificationEmailResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "SendEmailResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "TestRenderEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdateCustomVerificationEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdateEmailIdentityPolicyResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdateEmailTemplateResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdateReputationEntityCustomerManagedStatusResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdateReputationEntityPolicyResponse", + "reason": "HTML paragraph in description" + }, + { + "value": "UpdatedDKIMAuthorityResponse", + "reason": "Uses import()" + } + ] +} \ No newline at end of file