Improve openapi schema generation script
This commit is contained in:
Rory&
parent
38e681377a
commit
c168b8faf9
@ -20,19 +20,23 @@
|
||||
Regenerates the `spacebarchat/server/assets/schemas.json` file, used for API/Gateway input validation.
|
||||
*/
|
||||
|
||||
const scriptStartTime = new Date();
|
||||
|
||||
const conWarn = console.warn;
|
||||
console.warn = (...args) => {
|
||||
// silence some expected warnings
|
||||
if (args[0] === "initializer is expression for property id") return;
|
||||
if (args[0].startsWith("unknown initializer for property ") && args[0].endsWith("[object Object]")) return;
|
||||
conWarn(...args);
|
||||
}
|
||||
};
|
||||
|
||||
const path = require("path");
|
||||
const fs = require("fs");
|
||||
const TJS = require("typescript-json-schema");
|
||||
const walk = require("./util/walk");
|
||||
const { redBright, yellowBright, bgRedBright, yellow } = require("picocolors");
|
||||
const schemaPath = path.join(__dirname, "..", "assets", "schemas.json");
|
||||
const exclusionList = JSON.parse(fs.readFileSync(path.join(__dirname, "schemaExclusions.json"), { encoding: "utf8" }));
|
||||
|
||||
const settings = {
|
||||
required: true,
|
||||
@ -43,82 +47,102 @@ const settings = {
|
||||
defaultProps: false,
|
||||
};
|
||||
|
||||
const ExcludeAndWarn = [
|
||||
/^Record/,
|
||||
/^Partial/,
|
||||
]
|
||||
const Excluded = [
|
||||
"DefaultSchema",
|
||||
"Schema",
|
||||
"EntitySchema",
|
||||
"ServerResponse",
|
||||
"Http2ServerResponse",
|
||||
"ExpressResponse",
|
||||
"global.Express.Response",
|
||||
"global.Response",
|
||||
"Response",
|
||||
"e.Response",
|
||||
"request.Response",
|
||||
"supertest.Response",
|
||||
"DiagnosticsChannel.Response",
|
||||
"_Response",
|
||||
"ReadableStream<any>",
|
||||
const ExcludeAndWarn = [...exclusionList.manualWarn, ...exclusionList.manualWarnRe.map((r) => new RegExp(r))];
|
||||
const Excluded = [...exclusionList.manual, ...exclusionList.manualRe.map((r) => new RegExp(r)), ...exclusionList.auto.map((r) => r.value)];
|
||||
const Included = [...exclusionList.include, ...exclusionList.includeRe.map((r) => new RegExp(r))];
|
||||
|
||||
// TODO: Figure out how to exclude schemas from node_modules?
|
||||
"SomeJSONSchema",
|
||||
"UncheckedPartialSchema",
|
||||
"PartialSchema",
|
||||
"UncheckedPropertiesSchema",
|
||||
"PropertiesSchema",
|
||||
"AsyncSchema",
|
||||
"AnySchema",
|
||||
"SMTPConnection.CustomAuthenticationResponse",
|
||||
"TransportMakeRequestResponse",
|
||||
// Emma [it/its] @ Rory& - 2025-10-14
|
||||
/.*\..*/,
|
||||
/^Axios.*/,
|
||||
/^APIKeyConfiguration\..*/,
|
||||
/^AccountSetting\..*/,
|
||||
/^BulkContactManagement\..*/,
|
||||
/^Campaign.*/,
|
||||
/^Contact.*/,
|
||||
/^DNS\..*/,
|
||||
/^Delete.*/,
|
||||
/^Destroy.*/,
|
||||
/^Template\..*/,
|
||||
/^Webhook\..*/,
|
||||
/^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric)Schema/,
|
||||
/^Put/
|
||||
const excludedLambdas = [
|
||||
(n, s) => {
|
||||
// attempt to import
|
||||
if (JSON.stringify(s).includes(`#/definitions/import(`)) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it attempted to use import().`);
|
||||
exclusionList.auto.push({ value: n, reason: "Uses import()" });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
(n, s) => {
|
||||
if (JSON.stringify(s).includes(process.cwd())) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked $PWD.`);
|
||||
exclusionList.auto.push({ value: n, reason: "Leaked $PWD" });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
(n, s) => {
|
||||
if (JSON.stringify(s).includes(process.env.HOME)) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it leaked a $HOME path.`);
|
||||
exclusionList.auto.push({ value: n, reason: "Leaked $HOME" });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
(n, s) => {
|
||||
if (s["$ref"] === `#/definitions/${n}`) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it is a self-reference only schema.`);
|
||||
exclusionList.auto.push({ value: n, reason: "Self-reference only schema" });
|
||||
// fs.writeFileSync(`fucked/${n}.json`, JSON.stringify(s, null, 4));
|
||||
return true;
|
||||
}
|
||||
},
|
||||
(n, s) => {
|
||||
if (s.description?.match(/Smithy/)) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as it appears to be an AWS Smithy schema.`);
|
||||
exclusionList.auto.push({ value: n, reason: "AWS Smithy schema" });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
(n, s) => {
|
||||
if (s.description?.startsWith("<p>")) {
|
||||
console.log(`\r${redBright("[WARN]")} Omitting schema ${n} as we don't use HTML paragraphs for descriptions.`);
|
||||
exclusionList.auto.push({ value: n, reason: "HTML paragraph in description" });
|
||||
return true;
|
||||
}
|
||||
},
|
||||
// (n, s) => {
|
||||
// if (JSON.stringify(s).length <= 300) {
|
||||
// console.log({n, s});
|
||||
// }
|
||||
// }
|
||||
];
|
||||
|
||||
function includesMatch(haystack, needles, log = false) {
|
||||
for (const needle of needles) {
|
||||
const match = needle instanceof RegExp ? needle.test(haystack) : haystack === needle;
|
||||
if (match) {
|
||||
if (log) console.warn(redBright("[WARN]:"), "Excluding schema", haystack, "due to match with", needle);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const program = TJS.programFromConfig(
|
||||
path.join(__dirname, "..", "tsconfig.json"),
|
||||
walk(path.join(__dirname, "..", "src", "schemas")),
|
||||
);
|
||||
const program = TJS.programFromConfig(path.join(__dirname, "..", "tsconfig.json"), walk(path.join(__dirname, "..", "src", "schemas")));
|
||||
const generator = TJS.buildGenerator(program, settings);
|
||||
if (!generator || !program) return;
|
||||
|
||||
let schemas = generator.getUserSymbols().filter((x) => {
|
||||
return (
|
||||
(
|
||||
x.endsWith("Schema")
|
||||
||x.endsWith("Response")
|
||||
|| x.startsWith("API")
|
||||
)
|
||||
&& !ExcludeAndWarn.some(exc => {
|
||||
const match = exc instanceof RegExp ? exc.test(x) : x === exc;
|
||||
if (match) console.warn("Warning: Excluding schema", x);
|
||||
return match;
|
||||
})
|
||||
&& !Excluded.some(exc => exc instanceof RegExp ? exc.test(x) : x === exc)
|
||||
(x.endsWith("Schema") || x.endsWith("Response") || x.startsWith("API")) &&
|
||||
// !ExcludeAndWarn.some((exc) => {
|
||||
// const match = exc instanceof RegExp ? exc.test(x) : x === exc;
|
||||
// if (match) console.warn("Warning: Excluding schema", x);
|
||||
// return match;
|
||||
// }) &&
|
||||
// !Excluded.some((exc) => (exc instanceof RegExp ? exc.test(x) : x === exc))
|
||||
(includesMatch(x, Included) || (!includesMatch(x, ExcludeAndWarn, true) && !includesMatch(x, Excluded)))
|
||||
);
|
||||
});
|
||||
//.sort((a,b) => a.localeCompare(b));
|
||||
|
||||
var definitions = {};
|
||||
|
||||
if (process.env.WRITE_SCHEMA_DIR === "true") {
|
||||
fs.rmSync("schemas", { recursive: true, force: true });
|
||||
fs.mkdirSync("schemas");
|
||||
}
|
||||
|
||||
for (const name of schemas) {
|
||||
console.log("Processing schema", name);
|
||||
const startTime = new Date();
|
||||
process.stdout.write(`Processing schema ${name}... `);
|
||||
const part = TJS.generateSchema(program, name, settings, [], generator);
|
||||
if (!part) continue;
|
||||
|
||||
@ -144,27 +168,35 @@ function main() {
|
||||
}
|
||||
}
|
||||
|
||||
if (definitions[name]) {
|
||||
process.stdout.write(yellow(` [ERROR] Duplicate schema name detected: ${name}. Overwriting previous schema.`));
|
||||
}
|
||||
|
||||
if (!includesMatch(name, Included) && excludedLambdas.some((fn) => fn(name, part))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (process.env.WRITE_SCHEMA_DIR === "true") fs.writeFileSync(path.join("schemas", `${name}.json`), JSON.stringify(part, null, 4));
|
||||
|
||||
process.stdout.write("Done in " + yellowBright(new Date() - startTime) + " ms, " + yellowBright(JSON.stringify(part).length) + " bytes (unformatted) ");
|
||||
if (new Date() - startTime >= 20) console.log(bgRedBright("[SLOW]"));
|
||||
else console.log();
|
||||
|
||||
definitions = { ...definitions, [name]: { ...part } };
|
||||
}
|
||||
|
||||
deleteOneOfKindUndefinedRecursive(definitions, "$");
|
||||
|
||||
fs.writeFileSync(schemaPath, JSON.stringify(definitions, null, 4));
|
||||
console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath);
|
||||
fs.writeFileSync(__dirname + "/schemaExclusions.json", JSON.stringify(exclusionList, null, 4));
|
||||
console.log("Successfully wrote", Object.keys(definitions).length, "schemas to", schemaPath, "in", new Date() - scriptStartTime, "ms.");
|
||||
}
|
||||
|
||||
function deleteOneOfKindUndefinedRecursive(obj, path) {
|
||||
if (
|
||||
obj?.type === "object" &&
|
||||
obj?.properties?.oneofKind?.type === "undefined"
|
||||
)
|
||||
return true;
|
||||
if (obj?.type === "object" && obj?.properties?.oneofKind?.type === "undefined") return true;
|
||||
|
||||
for (const key in obj) {
|
||||
if (
|
||||
typeof obj[key] === "object" &&
|
||||
deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key)
|
||||
) {
|
||||
if (typeof obj[key] === "object" && deleteOneOfKindUndefinedRecursive(obj[key], path + "." + key)) {
|
||||
console.log("Deleting", path, key);
|
||||
delete obj[key];
|
||||
}
|
||||
|
||||
345
scripts/schemaExclusions.json
Normal file
345
scripts/schemaExclusions.json
Normal file
@ -0,0 +1,345 @@
|
||||
{
|
||||
"include": [
|
||||
"MessageInteractionSchema"
|
||||
],
|
||||
"includeRe": [],
|
||||
"manual": [
|
||||
"DefaultSchema",
|
||||
"Schema",
|
||||
"EntitySchema",
|
||||
"ReadableStream<any>",
|
||||
"SomeJSONSchema",
|
||||
"UncheckedPartialSchema",
|
||||
"PartialSchema",
|
||||
"UncheckedPropertiesSchema",
|
||||
"PropertiesSchema",
|
||||
"AsyncSchema",
|
||||
"AnySchema",
|
||||
"SMTPConnection.CustomAuthenticationResponse",
|
||||
"TransportMakeRequestResponse",
|
||||
"StaticSchema",
|
||||
"CSVImportResponse"
|
||||
],
|
||||
"manualRe": [
|
||||
".*\\.Response$",
|
||||
"^(Http2Server|Server|Express|(Resolved|)Http|Client|_|)Response$",
|
||||
".*\\..*",
|
||||
"^Axios.*",
|
||||
"^ListContact(s|Lists)Response$",
|
||||
"^APIKeyConfiguration\\..*",
|
||||
"^AccountSetting\\..*",
|
||||
"^BulkContactManagement\\..*",
|
||||
"^Campaign.*",
|
||||
"^Contact.*",
|
||||
"^DNS\\..*",
|
||||
"^Delete.*",
|
||||
"^Destroy.*",
|
||||
"^Template\\..*",
|
||||
"^Webhook\\..*",
|
||||
"^(BigDecimal|BigInteger|Blob|Boolean|Document|Error|LazyRequest|List|Map|Normalized|Numeric|StreamingBlob|TimestampDateTime|TimestampHttpDate|TimestampEpochSeconds|Simple)Schema",
|
||||
"^((Create|Update)Contact(|List))Response$",
|
||||
"^(T|Unt)agResourceResponse$",
|
||||
"^Put",
|
||||
"^Inbox",
|
||||
"^Seed",
|
||||
"^DomainTag",
|
||||
"^IpPool",
|
||||
"DomainTemplate",
|
||||
"^\\$",
|
||||
"^Suppression",
|
||||
"^Mail(|ing)List",
|
||||
"DomainTracking",
|
||||
"UpdatedDomain",
|
||||
"ConfigurationSet",
|
||||
"ContactList",
|
||||
"^IPR",
|
||||
"^Job"
|
||||
],
|
||||
"manualWarn": [],
|
||||
"manualWarnRe": [
|
||||
"^Record",
|
||||
"^Partial"
|
||||
],
|
||||
"auto": [
|
||||
{
|
||||
"value": "StringSchema",
|
||||
"reason": "AWS Smithy schema"
|
||||
},
|
||||
{
|
||||
"value": "TimestampDefaultSchema",
|
||||
"reason": "AWS Smithy schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticSimpleSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticListSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticMapSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticStructureSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticErrorSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StaticOperationSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "UnitSchema",
|
||||
"reason": "AWS Smithy schema"
|
||||
},
|
||||
{
|
||||
"value": "MemberSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "StructureSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "OperationSchema",
|
||||
"reason": "Self-reference only schema"
|
||||
},
|
||||
{
|
||||
"value": "BatchGetMetricDataResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CancelExportJobResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateCustomVerificationEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateDedicatedIpPoolResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateDeliverabilityTestReportResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateEmailIdentityResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateEmailIdentityPolicyResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateExportJobResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateImportJobResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateMultiRegionEndpointResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateTenantResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "CreateTenantResourceAssociationResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetAccountResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetBlacklistReportsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetCustomVerificationEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDedicatedIpResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDedicatedIpPoolResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDedicatedIpsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDeliverabilityDashboardOptionsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDeliverabilityTestReportResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDomainDeliverabilityCampaignResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetDomainStatisticsReportResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetEmailIdentityResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetEmailIdentityPoliciesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetExportJobResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetImportJobResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetMessageInsightsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetMultiRegionEndpointResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetReputationEntityResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetSuppressedDestinationResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "GetTenantResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListCustomVerificationEmailTemplatesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListDedicatedIpPoolsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListDeliverabilityTestReportsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListDomainDeliverabilityCampaignsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListEmailIdentitiesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListEmailTemplatesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListExportJobsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListImportJobsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListMultiRegionEndpointsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListRecommendationsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListReputationEntitiesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListResourceTenantsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListSuppressedDestinationsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListTenantResourcesResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "ListTenantsResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "SendBulkEmailResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "SendCustomVerificationEmailResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "SendEmailResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "TestRenderEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdateCustomVerificationEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdateEmailIdentityPolicyResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdateEmailTemplateResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdateReputationEntityCustomerManagedStatusResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdateReputationEntityPolicyResponse",
|
||||
"reason": "HTML paragraph in description"
|
||||
},
|
||||
{
|
||||
"value": "UpdatedDKIMAuthorityResponse",
|
||||
"reason": "Uses import()"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user