RSCord/ServerSpacebarOld
Archived
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve openapi schema generation script

Browse Source
This commit is contained in:
Rory& 2025-10-23 22:27:46 +02:00
parent 38e681377a
commit c168b8faf9
2 changed files with 449 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
scripts/schema.js
View File

@ -20,19 +20,23 @@
Regenerates the `spacebarchat/server/assets/schemas.json` file, used for API/Gateway input validation. Regenerates the `spacebarchat/server/assets/schemas.json` file, used for API/Gateway input validation.
*/ */
const scriptStartTime = new Date();
const conWarn = console.warn; const conWarn = console.warn;
console.warn = (...args) => { console.warn = (...args) => {
// silence some expected warnings // silence some expected warnings
if (args[0] === "initializer is expression for property id") return; if (args[0] === "initializer is expression for property id") return;
if (args[0].startsWith("unknown initializer for property ") && args[0].endsWith("[object Object]")) return; if (args[0].startsWith("unknown initializer for property ") && args[0].endsWith("[object Object]")) return;
conWarn(...args); conWarn(...args);
} };
const path = require("path"); const path = require("path");
const fs = require("fs"); const fs = require("fs");
const TJS = require("typescript-json-schema"); const TJS = require("typescript-json-schema");
const walk = require("./util/walk"); const walk = require("./util/walk");
const { redBright, yellowBright, bgRedBright, yellow } = require("picocolors");
const schemaPath = path.join(__dirname, "..", "assets", "schemas.json"); const schemaPath = path.join(__dirname, "..", "assets", "schemas.json");
const exclusionList = JSON.parse(fs.readFileSync(path.join(__dirname, "schemaExclusions.json"), { encoding: "utf8" }));
const settings = { const settings = {
required: true, required: true,
@ -43,82 +47,102 @@ const settings = {
defaultProps: false, defaultProps: false,
}; };
const ExcludeAndWarn = [ const ExcludeAndWarn = [...exclusionList.manualWarn, ...exclusionList.manualWarnRe.map((r) => new RegExp(r))];
/^Record/, const Excluded = [...exclusionList.manual, ...exclusionList.manualRe.map((r) => new RegExp(r)), ...exclusionList.auto.map((r) => r.value)];
/^Partial/, const Included = [...exclusionList.include, ...exclusionList.includeRe.map((r) => new RegExp(r))];
]
const Excluded = [
"DefaultSchema",
"Schema",
"EntitySchema",
"ServerResponse",
"Http2ServerResponse",
"ExpressResponse",
"global.Express.Response",
"global.Response",
"Response",
"e.Response",
"request.Response",
"supertest.Response",
"DiagnosticsChannel.Response",
"_Response",
"ReadableStream<any>",
// TODO: Figure out how to exclude schemas from node_modules? const excludedLambdas = [
"SomeJSONSchema", (n, s) => {
"UncheckedPartialSchema", // attempt to import
"PartialSchema", if (JSON.stringify(s).includes(`#/definitions/import(`)) {
"UncheckedPropertiesSchema", console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it attempted to use import().`);
"PropertiesSchema", exclusionList.auto.push({ value: n, reason: "Uses import()" });
"AsyncSchema", return true;
"AnySchema", }
"SMTPConnection.CustomAuthenticationResponse", },
"TransportMakeRequestResponse", (n, s) => {
// Emma [it/its] @ Rory& - 2025-10-14 if (JSON.stringify(s).includes(process.cwd())) {
/.*\..*/, console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked $PWD.`);
/^Axios.*/, exclusionList.auto.push({ value: n, reason: "Leaked $PWD" });
/^APIKeyConfiguration\..*/, return true;
/^AccountSetting\..*/, }
/^BulkContactManagement\..*/, },
/^Campaign.*/, (n, s) => {
/^Contact.*/, if (JSON.stringify(s).includes(process.env.HOME)) {
/^DNS\..*/, console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked a $HOME path.`);
/^Delete.*/, exclusionList.auto.push({ value: n, reason: "Leaked $HOME" });
/^Destroy.*/, return true;
/^Template\..*/, }
/^Webhook\..*/, },
/^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric)Schema/, (n, s) => {
/^Put/ if (s["$ref"] === `#/definitions/${n}`) {
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it is a self-reference only schema.`);
exclusionList.auto.push({ value: n, reason: "Self-reference only schema" });
// fs.writeFileSync(`fucked/${n}.json`, JSON.stringify(s, null, 4));
return true;
}
},
(n, s) => {
if (s.description?.match(/Smithy/)) {
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it appears to be an AWS Smithy schema.`);
exclusionList.auto.push({ value: n, reason: "AWS Smithy schema" });
return true;
}
},
(n, s) => {
if (s.description?.startsWith("<p>")) {
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as we don't use HTML paragraphs for descriptions.`);
exclusionList.auto.push({ value: n, reason: "HTML paragraph in description" });
return true;
}
},
// (n, s) => {
// if (JSON.stringify(s).length <= 300) {
// console.log({n, s});
// }
// }
]; ];
function includesMatch(haystack, needles, log = false) {
for (const needle of needles) {
const match = needle instanceof RegExp ? needle.test(haystack) : haystack === needle;
if (match) {
if (log) console.warn(redBright("[WARN]:"), "Excluding schema", haystack, "due to match with", needle);
return true;
}
}
return false;
}
function main() { function main() {
const program = TJS.programFromConfig( const program = TJS.programFromConfig(path.join(__dirname, "..", "tsconfig.json"), walk(path.join(__dirname, "..", "src", "schemas")));
path.join(__dirname, "..", "tsconfig.json"),
walk(path.join(__dirname, "..", "src", "schemas")),
);
const generator = TJS.buildGenerator(program, settings); const generator = TJS.buildGenerator(program, settings);
if (!generator || !program) return; if (!generator || !program) return;
let schemas = generator.getUserSymbols().filter((x) => { let schemas = generator.getUserSymbols().filter((x) => {
return ( return (
( (x.endsWith("Schema") || x.endsWith("Response") || x.startsWith("API")) &&
x.endsWith("Schema") // !ExcludeAndWarn.some((exc) => {
||x.endsWith("Response") // const match = exc instanceof RegExp ? exc.test(x) : x === exc;
|| x.startsWith("API") // if (match) console.warn("Warning: Excluding schema", x);
) // return match;
&& !ExcludeAndWarn.some(exc => { // }) &&
const match = exc instanceof RegExp ? exc.test(x) : x === exc; // !Excluded.some((exc) => (exc instanceof RegExp ? exc.test(x) : x === exc))
if (match) console.warn("Warning: Excluding schema", x); (includesMatch(x, Included) || (!includesMatch(x, ExcludeAndWarn, true) && !includesMatch(x, Excluded)))
return match;
})
&& !Excluded.some(exc => exc instanceof RegExp ? exc.test(x) : x === exc)
); );
}); });
//.sort((a,b) => a.localeCompare(b));
var definitions = {}; var definitions = {};
if (process.env.WRITE_SCHEMA_DIR === "true") {
fs.rmSync("schemas", { recursive: true, force: true });
fs.mkdirSync("schemas");
}
for (const name of schemas) { for (const name of schemas) {
console.log("Processing schema", name); const startTime = new Date();
process.stdout.write(`Processing schema ${name}... `);
const part = TJS.generateSchema(program, name, settings, [], generator); const part = TJS.generateSchema(program, name, settings, [], generator);
if (!part) continue; if (!part) continue;
@ -144,27 +168,35 @@ function main() {
} }
} }
if (definitions[name]) {
process.stdout.write(yellow(` [ERROR] Duplicate schema name detected: ${name}. Overwriting previous schema.`));
}
if (!includesMatch(name, Included) && excludedLambdas.some((fn) => fn(name, part))) {
continue;
}
if (process.env.WRITE_SCHEMA_DIR === "true") fs.writeFileSync(path.join("schemas", `${name}.json`), JSON.stringify(part, null, 4));
process.stdout.write("Done in " + yellowBright(new Date() - startTime) + " ms, " + yellowBright(JSON.stringify(part).length) + " bytes (unformatted) ");
if (new Date() - startTime >= 20) console.log(bgRedBright("[SLOW]"));
else console.log();
definitions = { ...definitions, [name]: { ...part } }; definitions = { ...definitions, [name]: { ...part } };
} }
deleteOneOfKindUndefinedRecursive(definitions, "$"); deleteOneOfKindUndefinedRecursive(definitions, "$");
fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4)); fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4));
console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath); fs.writeFileSync(__dirname + "/schemaExclusions.json", JSON.stringify(exclusionList, null, 4));
console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath, "in", new Date() - scriptStartTime, "ms.");
} }
function deleteOneOfKindUndefinedRecursive(obj, path) { function deleteOneOfKindUndefinedRecursive(obj, path) {
if ( if (obj?.type === "object" && obj?.properties?.oneofKind?.type === "undefined") return true;
obj?.type === "object" &&
obj?.properties?.oneofKind?.type === "undefined"
)
return true;
for (const key in obj) { for (const key in obj) {
if ( if (typeof obj[key] === "object" && deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key)) {
typeof obj[key] === "object" &&
deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key)
) {
console.log("Deleting", path, key); console.log("Deleting", path, key);
delete obj[key]; delete obj[key];
} }

345
scripts/schemaExclusions.json Normal file
View File

@ -0,0 +1,345 @@
{
"include": [
"MessageInteractionSchema"
],
"includeRe": [],
"manual": [
"DefaultSchema",
"Schema",
"EntitySchema",
"ReadableStream<any>",
"SomeJSONSchema",
"UncheckedPartialSchema",
"PartialSchema",
"UncheckedPropertiesSchema",
"PropertiesSchema",
"AsyncSchema",
"AnySchema",
"SMTPConnection.CustomAuthenticationResponse",
"TransportMakeRequestResponse",
"StaticSchema",
"CSVImportResponse"
],
"manualRe": [
".*\\.Response$",
"^(Http2Server|Server|Express|(Resolved|)Http|Client|_|)Response$",
".*\\..*",
"^Axios.*",
"^ListContact(s|Lists)Response$",
"^APIKeyConfiguration\\..*",
"^AccountSetting\\..*",
"^BulkContactManagement\\..*",
"^Campaign.*",
"^Contact.*",
"^DNS\\..*",
"^Delete.*",
"^Destroy.*",
"^Template\\..*",
"^Webhook\\..*",
"^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric|StreamingBlob|TimestampDateTime|TimestampHttpDate|TimestampEpochSeconds|Simple)Schema",
"^((Create|Update)Contact(|List))Response$",
"^(T|Unt)agResourceResponse$",
"^Put",
"^Inbox",
"^Seed",
"^DomainTag",
"^IpPool",
"DomainTemplate",
"^\\$",
"^Suppression",
"^Mail(|ing)List",
"DomainTracking",
"UpdatedDomain",
"ConfigurationSet",
"ContactList",
"^IPR",
"^Job"
],
"manualWarn": [],
"manualWarnRe": [
"^Record",
"^Partial"
],
"auto": [
{
"value": "StringSchema",
"reason": "AWS Smithy schema"
},
{
"value": "TimestampDefaultSchema",
"reason": "AWS Smithy schema"
},
{
"value": "StaticSimpleSchema",
"reason": "Self-reference only schema"
},
{
"value": "StaticListSchema",
"reason": "Self-reference only schema"
},
{
"value": "StaticMapSchema",
"reason": "Self-reference only schema"
},
{
"value": "StaticStructureSchema",
"reason": "Self-reference only schema"
},
{
"value": "StaticErrorSchema",
"reason": "Self-reference only schema"
},
{
"value": "StaticOperationSchema",
"reason": "Self-reference only schema"
},
{
"value": "UnitSchema",
"reason": "AWS Smithy schema"
},
{
"value": "MemberSchema",
"reason": "Self-reference only schema"
},
{
"value": "StructureSchema",
"reason": "Self-reference only schema"
},
{
"value": "OperationSchema",
"reason": "Self-reference only schema"
},
{
"value": "BatchGetMetricDataResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CancelExportJobResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateCustomVerificationEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateDedicatedIpPoolResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateDeliverabilityTestReportResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateEmailIdentityResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateEmailIdentityPolicyResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateExportJobResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateImportJobResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateMultiRegionEndpointResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateTenantResponse",
"reason": "HTML paragraph in description"
},
{
"value": "CreateTenantResourceAssociationResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetAccountResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetBlacklistReportsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetCustomVerificationEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDedicatedIpResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDedicatedIpPoolResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDedicatedIpsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDeliverabilityDashboardOptionsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDeliverabilityTestReportResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDomainDeliverabilityCampaignResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetDomainStatisticsReportResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetEmailIdentityResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetEmailIdentityPoliciesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetExportJobResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetImportJobResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetMessageInsightsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetMultiRegionEndpointResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetReputationEntityResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetSuppressedDestinationResponse",
"reason": "HTML paragraph in description"
},
{
"value": "GetTenantResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListCustomVerificationEmailTemplatesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListDedicatedIpPoolsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListDeliverabilityTestReportsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListDomainDeliverabilityCampaignsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListEmailIdentitiesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListEmailTemplatesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListExportJobsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListImportJobsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListMultiRegionEndpointsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListRecommendationsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListReputationEntitiesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListResourceTenantsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListSuppressedDestinationsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListTenantResourcesResponse",
"reason": "HTML paragraph in description"
},
{
"value": "ListTenantsResponse",
"reason": "HTML paragraph in description"
},
{
"value": "SendBulkEmailResponse",
"reason": "HTML paragraph in description"
},
{
"value": "SendCustomVerificationEmailResponse",
"reason": "HTML paragraph in description"
},
{
"value": "SendEmailResponse",
"reason": "HTML paragraph in description"
},
{
"value": "TestRenderEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdateCustomVerificationEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdateEmailIdentityPolicyResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdateEmailTemplateResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdateReputationEntityCustomerManagedStatusResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdateReputationEntityPolicyResponse",
"reason": "HTML paragraph in description"
},
{
"value": "UpdatedDKIMAuthorityResponse",
"reason": "Uses import()"
}
]
}