diff --git a/src/.eslintrc b/.eslintrc similarity index 62% rename from src/.eslintrc rename to .eslintrc index 4993630..23202db 100644 --- a/src/.eslintrc +++ b/.eslintrc @@ -8,6 +8,12 @@ "prettier" ], "rules": { - "@typescript-eslint/no-explicit-any": "off" + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-unused-vars": [ + "warn", + { + "argsIgnorePattern": "^_" + } + ] } } diff --git a/Makefile b/Makefile index 79574dd..8865059 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ build-router: $(MAKE) HANDLER=src/router.ts build-lambda-common -build-lambda-common: +build-lambda-common: build-SpecLinterDependenciesLayer npm install rm -rf dist echo "{\"extends\": \"./tsconfig.json\", \"include\": [\"${HANDLER}\"] }" > tsconfig-only-handler.json diff --git a/__tests__/__fixtures__/definitions/restaurant-components.yaml b/__tests__/__fixtures__/definitions/restaurant-components.yaml new file mode 100644 index 0000000..6b258f0 --- /dev/null +++ b/__tests__/__fixtures__/definitions/restaurant-components.yaml @@ -0,0 +1,44 @@ +openapi: 3.0.3 +info: + version: v0.1.0 + title: restaurant-components +paths: [] +components: + schemas: + RestaurantListing: + type: array + items: + "$ref": "./restaurant-components.yaml#/components/schemas/Restaurant" + Restaurant: + type: object + properties: + id: + type: integer + format: int64 + servesCuisine: + description: The cuisine of the restaurant. + type: string + starRating: + description: + An official rating for a lodging business or food establishment, + e.g. from national associations or standards bodies. Use the author property + to indicate the rating organization, e.g. as an Organization with name + such as (e.g. HOTREC, DEHOGA, WHR, or Hotelstars). + type: object + format: starRating + menu: + description: + Either the actual menu as a structured representation, as text, + or a URL of the menu. + type: string + acceptsReservations: + description: + Indicates whether a FoodEstablishment accepts reservations. + Values can be Boolean, an URL at which reservations can be made or (for + backwards compatibility) the strings ```Yes``` or ```No```. + type: string + hasMenu: + description: + Either the actual menu as a structured representation, as text, + or a URL of the menu. + type: string diff --git a/__tests__/__fixtures__/definitions/restaurant-openapi.yaml b/__tests__/__fixtures__/definitions/restaurant-openapi.yaml new file mode 100644 index 0000000..8f2e5c0 --- /dev/null +++ b/__tests__/__fixtures__/definitions/restaurant-openapi.yaml @@ -0,0 +1,150 @@ +openapi: 3.0.0 +info: + version: v0.1.0 + title: restaurants + description: This is the API for managing detail of the restaurants. +servers: + - url: http://api.example.com/ +paths: + "/restaurants": + get: + summary: Restaurants + operationId: getRestaurants + parameters: + - name: token + in: header + required: true + schema: + type: array + items: + type: integer + format: int64 + style: simple + tags: + - Restaurant + responses: + "200": + description: Restaurant + content: + application/json: + schema: + "$ref": "#/components/schemas/RestaurantListing" + post: + summary: Restaurant + operationId: addRestaurant + tags: + - Restaurant + requestBody: + description: Restaurant + content: + application/json: + schema: + "$ref": "#/components/schemas/Restaurant" + responses: + "201": + description: Restaurant + content: + application/json: + schema: + "$ref": "#/components/schemas/Restaurant" + "/restaurants/{restaurantId}": + get: + summary: Restaurant + operationId: getRestaurant + parameters: + - name: restaurantId + in: path + description: The unique id. + required: true + schema: + type: string + - name: newProperty + in: query + description: The unique id. + required: true + schema: + type: string + tags: + - Restaurant + responses: + "200": + description: Restaurant + content: + application/json: + schema: + "$ref": "#/components/schemas/Restaurant" + put: + summary: Restaurant + operationId: updateRestaurant + parameters: + - name: restaurantId + in: path + description: The unique id. + required: true + schema: + type: string + tags: + - Restaurant + requestBody: + description: Restaurant + content: + application/json: + schema: + "$ref": "#/components/schemas/Restaurant" + responses: + "204": + description: Restaurant + delete: + summary: Restaurant + operationId: deleteRestaurant + parameters: + - name: restaurantId + in: path + description: The unique id. + required: true + schema: + type: string + tags: + - Restaurant + responses: + "204": + description: Restaurant +components: + schemas: + RestaurantListing: + type: array + items: + "$ref": "#/components/schemas/Restaurant" + Restaurant: + type: object + properties: + id: + type: integer + format: int64 + servesCuisine: + description: The cuisine of the restaurant. + type: string + starRating: + description: + An official rating for a lodging business or food establishment, + e.g. from national associations or standards bodies. Use the author property + to indicate the rating organization, e.g. as an Organization with name + such as (e.g. HOTREC, DEHOGA, WHR, or Hotelstars). + type: object + format: starRating + menu: + description: + Either the actual menu as a structured representation, as text, + or a URL of the menu. + type: string + acceptsReservations: + description: + Indicates whether a FoodEstablishment accepts reservations. + Values can be Boolean, an URL at which reservations can be made or (for + backwards compatibility) the strings ```Yes``` or ```No```. + type: string + hasMenu: + description: + Either the actual menu as a structured representation, as text, + or a URL of the menu. + type: string diff --git a/__tests__/unit/handlers/linter.test.ts b/__tests__/unit/handlers/linter.test.ts index 05a0e45..48a5a30 100644 --- a/__tests__/unit/handlers/linter.test.ts +++ b/__tests__/unit/handlers/linter.test.ts @@ -49,7 +49,7 @@ describe("post linter", () => { jest.spyOn(console, "error"); const consoleError = console.error as jest.MockedFunction; - consoleError.mockImplementation(() => {}); + consoleError.mockImplementation(); const result = await handler(ev); consoleError.mockRestore; @@ -69,7 +69,7 @@ describe("post linter", () => { jest.spyOn(console, "error"); const consoleError = console.error as jest.MockedFunction; - consoleError.mockImplementation(() => {}); + consoleError.mockImplementation(); const result = await handler(ev); consoleError.mockRestore; mockFetch.mockRestore(); @@ -92,7 +92,7 @@ describe("post linter", () => { jest.spyOn(console, "error"); const consoleError = console.error as jest.MockedFunction; - consoleError.mockImplementation(() => {}); + consoleError.mockImplementation(); const result = await handler(ev); consoleError.mockRestore; mockFetch.mockRestore(); diff --git a/package-lock.json b/package-lock.json index 626f692..cf255ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@stoplight/spectral-parsers": "^1.0.1", "@stoplight/spectral-runtime": "^1.1.0", "aws-sdk": "^2.799.0", + "busboy": "^1.4.0", "content-type-parser": "^1.0.2", "js-yaml": "^4.1.0", "source-map-support": "^0.5.21", @@ -20,6 +21,7 @@ }, "devDependencies": { "@types/aws-lambda": "^8.10.85", + "@types/busboy": "^1.3.0", "@types/jest": "^27.0.3", "@types/js-yaml": "^4.0.5", "@types/node": "^16.11.10", @@ -1583,6 +1585,15 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/busboy": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.3.0.tgz", + "integrity": "sha512-Qx7ehfGO/k2yiTVpRIVIu16oVgbJpG65WLjEhbNSoTPdQEoRCorFUKHsSznyHmIkdvzOg2W3JWeewCmfSwcgeA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", @@ -2393,6 +2404,17 @@ "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz", "integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og=" }, + "node_modules/busboy": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.4.0.tgz", + "integrity": "sha512-TytIELfX6IPn1OClqcBz0NFE6+JT9e3iW0ZpgnEl7ffsfDxvRZGHfPaSHGbrI443nSV3GutCDWuqLB6yHY92Ew==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", @@ -5834,6 +5856,14 @@ "node": ">= 0.6" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", @@ -7746,6 +7776,15 @@ "@babel/types": "^7.3.0" } }, + "@types/busboy": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.3.0.tgz", + "integrity": "sha512-Qx7ehfGO/k2yiTVpRIVIu16oVgbJpG65WLjEhbNSoTPdQEoRCorFUKHsSznyHmIkdvzOg2W3JWeewCmfSwcgeA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/estree": { "version": "0.0.39", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.39.tgz", @@ -8337,6 +8376,14 @@ "resolved": "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz", "integrity": "sha1-y5T662HIaWRR2zZTThQi+U8K7og=" }, + "busboy": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.4.0.tgz", + "integrity": "sha512-TytIELfX6IPn1OClqcBz0NFE6+JT9e3iW0ZpgnEl7ffsfDxvRZGHfPaSHGbrI443nSV3GutCDWuqLB6yHY92Ew==", + "requires": { + "streamsearch": "^1.1.0" + } + }, "bytes": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.1.tgz", @@ -10926,6 +10973,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==" + }, "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", diff --git a/package.json b/package.json index 1d74837..f6ea611 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@stoplight/spectral-parsers": "^1.0.1", "@stoplight/spectral-runtime": "^1.1.0", "aws-sdk": "^2.799.0", + "busboy": "^1.4.0", "content-type-parser": "^1.0.2", "js-yaml": "^4.1.0", "source-map-support": "^0.5.21", @@ -16,6 +17,7 @@ }, "devDependencies": { "@types/aws-lambda": "^8.10.85", + "@types/busboy": "^1.3.0", "@types/jest": "^27.0.3", "@types/js-yaml": "^4.0.5", "@types/node": "^16.11.10", diff --git a/src/handlers/linter.ts b/src/handlers/linter.ts index f7d8870..7a2742c 100644 --- a/src/handlers/linter.ts +++ b/src/handlers/linter.ts @@ -2,9 +2,15 @@ import "source-map-support/register"; import fs from "fs/promises"; import { URL, URLSearchParams } from "url"; -import { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda"; +import { + APIGatewayProxyEvent, + APIGatewayProxyEventHeaders, + APIGatewayProxyEventQueryStringParameters, + APIGatewayProxyResult, +} from "aws-lambda"; import contentTypeParser from "content-type-parser"; import yaml from "js-yaml"; +import busboy, { FileInfo } from "busboy"; import ts from "typescript"; import { OutputFormat } from "@stoplight/spectral-cli/dist/services/config"; @@ -20,6 +26,30 @@ import { ILintConfig } from "@stoplight/spectral-cli/dist/services/config"; import { fetch } from "@stoplight/spectral-runtime"; import { problems } from "../problems"; +import { Readable } from "stream"; + +class InvalidRequestError extends Error { + constructor(err: Error) { + super(err.message); + } +} + +class TypeScriptCompilationError extends Error { + constructor(err: Error) { + super(err.message); + } +} + +type Definition = { + source: string; + value: object; +}; + +type File = { + name: string; + info: FileInfo; + value: object; +}; export const handler = async ( event: APIGatewayProxyEvent @@ -47,79 +77,26 @@ export const handler = async ( const t = contentTypeParser(event.headers["content-type"]); - const isValidContent = ["application/json", "text/yaml"].includes( - `${t?.type}/${t?.subtype}` - ); + const isValidContent = [ + "application/json", + "text/yaml", + "multipart/form-data", + ].includes(`${t?.type}/${t?.subtype}`); if (!event.body || !isValidContent) { return problems.UNSUPPORTED_REQUEST_BODY; } try { - yaml.load(event.body); // works with both JSON and YAML. - } catch (err) { - console.error(`Could not parse request body: ${err.message}`); - return problems.INVALID_REQUEST_BODY_SYNTAX; - } - - let rulesUrl = - event.queryStringParameters?.rulesUrl || - "https://rules.linting.org/testing/base.yaml"; // TODO: Accept from env var. - - // Spectral requires URLs to end in .json, .yaml, or .yml. - const supportedFileExtensions = [ - ".json", - ".yaml", - ".yml", - ".js", - ".mjs", - "cjs", - ".ts", - ]; - if (!supportedFileExtensions.find((ext) => rulesUrl.endsWith(ext))) { - // Should work for both JSON and YAML. - // If it's actually JavaScript or TypeScript, ope. - const testUrl = new URL(rulesUrl); - const spectralHack = "$spectral-hack$"; - - const params = new URLSearchParams(testUrl.search); - params.append(spectralHack, ".yaml"); - - testUrl.search = params.toString(); - rulesUrl = testUrl.toString(); - } - - if (rulesUrl.endsWith(".ts")) { - // compile to js in /temp and change rulesUrl - try { - const response = await fetch(rulesUrl); - const contents = await response.text(); - const js = ts.transpileModule(contents, { - compilerOptions: { - module: ts.ModuleKind.CommonJS, - }, - }); - - if (js.diagnostics?.length) { - console.log(js.diagnostics); - } + const rulesetIdentifier = await prepareRuleset(event.queryStringParameters); + const definitions = await getDefinitions(t, event.headers, event.body); - await fs.writeFile("/tmp/.spectral.js", js.outputText); - rulesUrl = "/tmp/.spectral.js"; - } catch (err) { - const message = `TypeScript compilation error: ${err.message}`; - console.error(message); - return problems.TYPESCRIPT_COMPILATION_FAILURE; - } - } - - try { - const [ruleset, results] = await lint(event.body, { + const [ruleset, results] = await lint(definitions, { format: OutputFormat.JSON, encoding: "utf-8", ignoreUnknownFormat: false, failOnUnmatchedGlobs: true, - ruleset: rulesUrl, + ruleset: rulesetIdentifier, }); const failedCodes = results.map((r) => String(r.code)); @@ -170,6 +147,16 @@ export const handler = async ( body: JSON.stringify(allResults), }; } catch (err) { + if (err instanceof InvalidRequestError) { + console.error(`Could not parse request body: ${err.message}`); + return problems.INVALID_REQUEST_BODY_SYNTAX; + } + if (err instanceof TypeScriptCompilationError) { + const message = `TypeScript compilation error: ${err.message}`; + console.error(message); + return problems.TYPESCRIPT_COMPILATION_FAILURE; + } + const message = `Failed to retrieve lint results: ${err.message}`; console.error(message); if (err.message === "Invalid ruleset provided") { @@ -180,18 +167,137 @@ export const handler = async ( }; const lint = async function ( - source: string, + definitions: Definition[], flags: ILintConfig ): Promise<[Ruleset, IRuleResult[]]> { const spectral = new Spectral(); const ruleset = await getRuleset(flags.ruleset); spectral.setRuleset(ruleset); - const document = new Document(source, Parsers.Yaml, flags.ruleset); + const results: IRuleResult[] = []; - const results: IRuleResult[] = await spectral.run(document, { - ignoreUnknownFormat: flags.ignoreUnknownFormat, - }); + for (const definition of definitions) { + const document = new Document( + JSON.stringify(definition.value), + Parsers.Yaml, + definition.source + ); + results.push( + ...(await spectral.run(document, { + ignoreUnknownFormat: flags.ignoreUnknownFormat, + })) + ); + } return [spectral.ruleset, results]; }; + +const parseForm = async function ( + headers: APIGatewayProxyEventHeaders, + body: string +): Promise { + return new Promise((resolve, reject) => { + const bb = busboy({ headers }); + const results: File[] = []; + bb.on("file", (name, file, info) => { + const chunks = []; + file.on("data", (chunk) => { + chunks.push(chunk.toString("utf8")); + }); + + file.on("close", () => { + results.push({ name, info, value: yaml.load(chunks.join()) as object }); + }); + + file.on("error", (err) => { + reject(err); + }); + }); + bb.on("error", (err) => { + reject(err); + }); + bb.on("close", () => { + resolve(results); + }); + Readable.from(body).pipe(bb); + }); +}; + +const getDefinitions = async ( + t: { type: string; subtype: string }, + headers: APIGatewayProxyEventHeaders, + body: string +): Promise => { + try { + if (t.type === "multipart" && t.subtype === "form-data") { + const results = await parseForm(headers, body); + // TODO: Actually check content-types. + return results + .filter((file) => file.name === "definition") + .map((file) => { + return { + source: file.info.filename, + value: file.value, + }; + }); + } else { + return [{ source: "", value: yaml.load(body) as object }]; + } + } catch (err) { + throw new InvalidRequestError(err); + } +}; + +const prepareRuleset = async ( + queryStringParameters?: APIGatewayProxyEventQueryStringParameters +): Promise => { + let rulesUrl = + queryStringParameters?.rulesUrl || + "https://rules.linting.org/testing/base.yaml"; // TODO: Accept from env var. + + // Spectral requires URLs to end in .json, .yaml, or .yml. + const supportedFileExtensions = [ + ".json", + ".yaml", + ".yml", + ".js", + ".mjs", + "cjs", + ".ts", + ]; + if (!supportedFileExtensions.find((ext) => rulesUrl.endsWith(ext))) { + // Should work for both JSON and YAML. + // If it's actually JavaScript or TypeScript, ope. + const testUrl = new URL(rulesUrl); + const spectralHack = "$spectral-hack$"; + + const params = new URLSearchParams(testUrl.search); + params.append(spectralHack, ".yaml"); + + testUrl.search = params.toString(); + rulesUrl = testUrl.toString(); + } + + if (rulesUrl.endsWith(".ts")) { + // compile to js in /temp and change rulesUrl + try { + const response = await fetch(rulesUrl); + const contents = await response.text(); + const js = ts.transpileModule(contents, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + }, + }); + + if (js.diagnostics?.length) { + console.log(js.diagnostics); + } + + await fs.writeFile("/tmp/.spectral.js", js.outputText); + rulesUrl = "/tmp/.spectral.js"; + } catch (err) { + throw new TypeScriptCompilationError(err); + } + } + return rulesUrl; +}; diff --git a/template.yaml b/template.yaml index faa0809..6123f2b 100644 --- a/template.yaml +++ b/template.yaml @@ -5,16 +5,6 @@ Transform: - AWS::Serverless-2016-10-31 Resources: - SpecLinterDependenciesLayer: - Type: AWS::Serverless::LayerVersion - Properties: - ContentUri: ./ - CompatibleRuntimes: - - nodejs14.x - RetentionPolicy: Retain - Metadata: - BuildMethod: makefile - router: Type: AWS::Serverless::Function Properties: @@ -25,8 +15,6 @@ Resources: MemorySize: 128 Timeout: 100 Description: The router for the API. - Layers: - - !Ref SpecLinterDependenciesLayer Events: Root: Type: Api