diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f64348..c04f230 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ references: jobs: build_and_test: docker: - - image: circleci/node:8.11 + - image: circleci/node:12.0 steps: - *restore_repo - checkout diff --git a/.gitignore b/.gitignore index ee20bfc..58e84a0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,9 @@ cloudformation.yml ddd.js package-lock.json yarn.lock +lerna-debug.log *.lerna_backup -_book \ No newline at end of file +_book +.idea/ +.DS_STORE +.serverless/ diff --git a/.nvmrc b/.nvmrc index 22ec1e6..87e3933 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -8.10 +12.13 diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f62b5..a575431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v0.4.0] - 2019-12-19 + +### Added +- `created` and `updated` fields added to item properties +- `serverless` deployment configuration file (serverless.yml) + +### Changed +- `time` field renamed to `datetime` +- Error raised if both bbox and intersection are specified +- Search metadata changed to match the `search` extension +- Intersect parameter accepts only GeoJSON geometries + +### Removed +- `fields` filter due to issues with default behavior. To be added back in for STAC 0.9.0 which reworks how fields filter works +- Batch ingestion jobs with Fargate + + ## [v0.3.0] - 2019-10-16 ### Added diff --git a/Dockerfile b/Dockerfile index c83e243..50af9cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # - ES_HOST: Elasticsearch https endpoint -FROM node:8 +FROM node:12 ENV \ HOME=/home/sat-utils diff --git a/README.md b/README.md index d51cf05..4645d5e 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ The STAC version supported by a given version of sat-api is shown in the table b | 0.1.0 | 0.5.x | | 0.2.x | 0.6.x | | 0.3.x | 0.7.x | +| 0.4.x | 0.8.x | ## Usage diff --git a/lerna.json b/lerna.json index 4352d8e..9021a7d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "2.11.0", - "version": "0.3.0", + "version": "0.4.0-rc2", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 154308b..3b87d49 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "test": "lerna run test", "update": "lerna publish --skip-git --skip-npm", "eslint": "eslint packages/* --ext .js", + "deploy": "sls deploy", "build-api-docs": "yarn widdershins --search false --language_tabs 'nodejs:NodeJS' 'python:Python' --summary ./packages/api-lib/api-spec.yaml -o ./docs/api.md & yarn shins --inline --logo ./docs/images/logo.png -o ./docs/index.html ./docs/api.md" }, "devDependencies": { @@ -21,7 +22,11 @@ "eslint-plugin-jsx-a11y": "^6.0.3", "eslint-plugin-react": "^7.7.0", "lerna": "^2.11.0", + "serverless": "^1.53.0", + "serverless-pseudo-parameters": "^2.5.0", + "serverless-step-functions": "^2.11.0", "shins": "^2.3.2-3", "widdershins": "^3.6.6" - } + }, + "name": "sat-api" } diff --git a/packages/api-lib/api-spec.yaml b/packages/api-lib/api-spec.yaml deleted file mode 100644 index 1e05f38..0000000 --- a/packages/api-lib/api-spec.yaml +++ /dev/null @@ -1,940 +0,0 @@ -openapi: 3.0.1 -info: - title: The SAT-API - version: 0.2.0 - description: >- - Sat-api is a STAC compliant web API for searching and serving metadata for - geospatial data (including but not limited to satellite imagery). - Development Seed runs an instance of sat-api for the Landsat-8 - and Sentinel-2 imagery that is hosted on AWS. - contact: - name: Development Seed - email: info@developmentseed.org - url: 'https://developmentseed.org/contacts/' - license: - name: MIT License - url: 'https://github.com/sat-utils/sat-api/blob/master/LICENSE' -servers: - - url: 'https://sat-api.developmentseed.org/' - description: Production server - - url: 'https://sat-api-dev.developmentseed.org/' - description: Development server -paths: - /stac: - get: - summary: Return the root catalog or collection. - description: >- - Returns the root STAC Catalog or STAC Collection that is the entry point - for users to browse with STAC Browser or for search engines to crawl. - This can either return a single STAC Collection or more commonly a STAC - catalog that usually lists sub-catalogs of STAC Collections, i.e. a - simple catalog that lists all collections available through the API. - tags: - - STAC - responses: - '200': - description: A catalog json definition. Used as an entry point for a crawler. - content: - application/json: - schema: - $ref: '#/components/schemas/catalogDefinition' - - /stac/search: - get: - summary: Search STAC items by simple filtering. - description: >- - Retrieve Items matching filters. Intended as a shorthand API for simple - queries. - operationId: getSearchSTAC - tags: - - STAC - parameters: - - $ref: '#/components/parameters/bbox' - - $ref: '#/components/parameters/time' - - $ref: '#/components/parameters/limit' - - $ref: '#/components/parameters/query' - - $ref: '#/components/parameters/sort' - - $ref: '#/components/parameters/fields' - responses: - '200': - description: A feature collection. - content: - application/geo+json: - schema: - $ref: '#/components/schemas/itemCollection' - default: - description: An error occurred. - content: - application/json: - schema: - $ref: '#/components/schemas/exception' - - post: - summary: Search STAC items by full-featured filtering. - description: >- - retrieve items matching filters. Intended as the standard, full-featured - query API. This method is mandatory. - operationId: postSearchSTAC - tags: - - STAC - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/searchBody' - responses: - '200': - description: A feature collection. - content: - application/geo+json: - schema: - $ref: '#/components/schemas/itemCollection' - text/html: - schema: - type: string - default: - description: An error occurred. - content: - application/json: - schema: - $ref: '#/components/schemas/exception' - text/html: - schema: - type: string - /: - get: - summary: landing page of this API - description: >- - The landing page provides links to the API definition, the Conformance - statements and the metadata about the feature data in this dataset. - operationId: getLandingPage - tags: - - Capabilities - responses: - '200': - description: links to the API capabilities - content: - application/json: - schema: - $ref: '#/components/schemas/root' - text/html: - schema: - type: string - - /collections: - get: - summary: describe the feature collections in the dataset - operationId: describeCollections - tags: - - Capabilities - responses: - '200': - description: Metdata about the feature collections shared by this API. - content: - application/json: - schema: - $ref: '#/components/schemas/content' - - '/collections/{collectionId}': - get: - summary: 'describe the {collectionId} feature collection' - operationId: describeCollection - tags: - - Capabilities - parameters: - - $ref: '#/components/parameters/collectionId' - responses: - '200': - description: 'Metadata about the {collectionId} collection shared by this API.' - content: - application/json: - schema: - $ref: '#/components/schemas/collectionInfo' - default: - description: An error occurred. - content: - application/json: - schema: - $ref: '#/components/schemas/exception' - - '/collections/{collectionId}/items': - get: - summary: 'retrieve features of feature collection {collectionId}' - description: >- - Every feature in a dataset belongs to a collection. A dataset may - consist of multiple feature collections. A feature collection is often a - collection of features of a similar type, based on a common schema.\ - - Use content negotiation to request HTML or GeoJSON. - operationId: getFeatures - tags: - - Features - parameters: - - $ref: '#/components/parameters/collectionId' - - $ref: '#/components/parameters/limit' - - $ref: '#/components/parameters/bbox' - - $ref: '#/components/parameters/time' - - $ref: '#/components/parameters/query' - - $ref: '#/components/parameters/sort' - responses: - '200': - description: >- - Information about the feature collection plus the first features - matching the selection parameters. - content: - application/geo+json: - schema: - $ref: '#/components/schemas/itemCollection' - default: - description: An error occurred. - content: - application/json: - schema: - $ref: '#/components/schemas/exception' - - '/collections/{collectionId}/items/{featureId}': - get: - summary: retrieve a feature; use content negotiation to request HTML or GeoJSON - operationId: getFeature - tags: - - Features - parameters: - - $ref: '#/components/parameters/collectionId' - - $ref: '#/components/parameters/featureId' - responses: - '200': - description: A feature. - content: - application/geo+json: - schema: - $ref: '#/components/schemas/item' - default: - description: An error occurred. - content: - application/json: - schema: - $ref: '#/components/schemas/exception' - -components: - parameters: - limit: - name: limit - in: query - description: | - The optional limit parameter limits the number of items that are - presented in the response document. - - Only items are counted that are on the first level of the collection in - the response document. Nested objects contained within the explicitly - requested items shall not be counted. - - * Minimum = 1 - * Maximum = 10000 - * Default = 10 - required: false - schema: - type: integer - minimum: 1 - maximum: 10000 - default: 10 - style: form - explode: false - bbox: - name: bbox - in: query - description: | - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a - vertical axis (elevation or depth): - - * Lower left corner, coordinate axis 1 - * Lower left corner, coordinate axis 2 - * Lower left corner, coordinate axis 3 (optional) - * Upper right corner, coordinate axis 1 - * Upper right corner, coordinate axis 2 - * Upper right corner, coordinate axis 3 (optional) - - The coordinate reference system of the values is WGS84 - longitude/latitude (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless - a different coordinate reference system is specified in the parameter - `bbox-crs`. - - For WGS84 longitude/latitude the values are in most cases the sequence - of minimum longitude, minimum latitude, maximum longitude and maximum - latitude. However, in cases where the box spans the antimeridian the - first value (west-most box edge) is larger than the third value - (east-most box edge). - - - If a feature has multiple spatial geometry properties, it is the - decision of the server whether only a single spatial geometry property - is used to determine the extent or all relevant geometries. - required: false - schema: - type: array - minItems: 4 - maxItems: 6 - items: - type: number - style: form - explode: false - time: - name: time - in: query - description: > - Either a date-time or a period string that adheres to RFC3339. Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A period: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" or "2018-02-12T00:00:00Z/P1M6DT12H31M12S" - - Only features that have a temporal property that intersects the value - of `time` are selected. If a feature has multiple temporal properties, it - is the decision of the server whether only a single temporal property is - used to determine the extent or all relevant temporal properties. - required: false - schema: - type: string - style: form - explode: false - collectionId: - name: collectionId - in: path - required: true - description: Identifier (name) of a specific collection - schema: - type: string - featureId: - name: featureId - in: path - description: Local identifier of a specific feature - required: true - schema: - type: string - query: - name: query - in: query - description: >- - query for properties in items. Use the JSON form of the queryFilter used - in POST. - required: false - schema: - type: string - sort: - name: sort - in: query - description: Allows sorting results by the specified properties - required: false - schema: - $ref: '#/components/schemas/sort' - fields: - name: fields - in: query - description: Determines the shape of the features in the response - required: false - schema: - $ref: '#/components/schemas/fields' - style: form - explode: false - schemas: - exception: - type: object - required: - - code - properties: - code: - type: string - description: - type: string - links: - type: array - items: - $ref: '#/components/schemas/link' - link: - type: object - required: - - href - - rel - additionalProperties: true - properties: - href: - type: string - format: url - example: 'http://www.geoserver.example/stac/naip/child/catalog.json' - rel: - type: string - example: child - type: - type: string - example: application/json - title: - type: string - example: NAIP Child Catalog - searchBody: - description: The search criteria - type: object - allOf: - - $ref: '#/components/schemas/bboxFilter' - - $ref: '#/components/schemas/timeFilter' - - $ref: '#/components/schemas/intersectsFilter' - - $ref: '#/components/schemas/queryFilter' - - $ref: '#/components/schemas/sortFilter' - bbox: - description: | - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a - vertical axis (elevation or depth): - - * Lower left corner, coordinate axis 1 - * Lower left corner, coordinate axis 2 - * Lower left corner, coordinate axis 3 (optional) - * Upper right corner, coordinate axis 1 - * Upper right corner, coordinate axis 2 - * Upper right corner, coordinate axis 3 (optional) - - The coordinate reference system of the values is WGS84 - longitude/latitude (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless - a different coordinate reference system is specified in the parameter - `bbox-crs`. - - For WGS84 longitude/latitude the values are in most cases the sequence - of minimum longitude, minimum latitude, maximum longitude and maximum - latitude. However, in cases where the box spans the antimeridian the - first value (west-most box edge) is larger than the third value - (east-most box edge). - - - If a feature has multiple spatial geometry properties, it is the - decision of the server whether only a single spatial geometry property - is used to determine the extent or all relevant geometries. - type: array - minItems: 4 - maxItems: 6 - items: - type: number - example: - - -110 - - 39.5 - - -105 - - 40.5 - bboxFilter: - type: object - description: Only return items that intersect the provided bounding box. - properties: - bbox: - $ref: '#/components/schemas/bbox' - timeFilter: - description: An object representing a time based filter. - type: object - properties: - time: - $ref: '#/components/schemas/time' - intersectsFilter: - type: object - description: Only returns items that intersect with the provided polygon. - properties: - intersects: - $ref: 'http://geojson.org/schema/Geometry.json' - time: - type: string - description: > - Either a date-time or a period string that adheres to RFC 3339. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A period: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" or "2018-02-12T00:00:00Z/P1M6DT12H31M12S" - - Only features that have a temporal property that intersects the value of - `time` are selected. - - If a feature has multiple temporal properties, it is the decision of the - server whether only a single temporal property is used to determine the - extent or all relevant temporal properties. - example: '2018-02-12T00:00:00Z/2018-03-18T12:31:12Z' - catalogDefinition: - type: object - required: - - stac_version - - id - - description - - links - additionalProperties: true - properties: - stac_version: - type: string - example: 0.6.0 - id: - type: string - example: naip - title: - type: string - example: NAIP Imagery - description: - type: string - example: Catalog of NAIP Imagery. - links: - $ref: '#/components/schemas/links' - itemCollection: - type: object - required: - - features - - type - properties: - type: - type: string - enum: - - FeatureCollection - features: - type: array - items: - $ref: '#/components/schemas/item' - links: - $ref: '#/components/schemas/itemCollectionLinks' - item: - type: object - required: - - id - - type - - geometry - - bbox - - links - - properties - - assets - properties: - id: - $ref: '#/components/schemas/itemId' - bbox: - $ref: '#/components/schemas/bbox' - geometry: - $ref: 'http://geojson.org/schema/Geometry.json' - type: - $ref: '#/components/schemas/itemType' - properties: - $ref: '#/components/schemas/itemProperties' - links: - $ref: '#/components/schemas/links' - assets: - $ref: '#/components/schemas/itemAssets' - example: - type: Feature - id: CS3-20160503_132130_04 - bbox: - - -122.59750209 - - 37.48803556 - - -122.2880486 - - 37.613537207 - geometry: - type: Polygon - coordinates: - - - - -122.308150179 - - 37.488035566 - - - -122.597502109 - - 37.538869539 - - - -122.576687533 - - 37.613537207 - - - -122.2880486 - - 37.562818007 - - - -122.308150179 - - 37.488035566 - properties: - datetime: '2016-05-03T13:21:30.040Z' - links: - - rel: self - href: >- - http://https://sat-api.developmentseed.org/collections/landsat-8-l1/items/LC80100102015050LGN00.json - assets: - analytic: - title: 4-Band Analytic - href: >- - http://cool-sat.com/LC80100102015050LGN00/band4.tiff - type: image/tiff - thumbnail: - title: Thumbnail - href: >- - http://cool-sat.com/LC80100102015050LGN00/thumb.png - type: image/png - itemId: - type: string - example: path/to/example.tif - description: 'Provider identifier, a unique ID, potentially a link to a file.' - itemType: - type: string - description: The GeoJSON type - enum: - - Feature - itemAssets: - type: object - additionalProperties: - type: object - required: - - href - properties: - href: - type: string - format: url - description: Link to the asset object - example: >- - http://cool-sat.com/LC80100102015050LGN00/thumb.png - title: - type: string - description: Displayed title - example: Thumbnail - type: - type: string - description: Media type of the asset - example: image/png - itemProperties: - type: object - required: - - datetime - description: provides the core metatdata fields plus extensions - properties: - datetime: - $ref: '#/components/schemas/time' - additionalProperties: - description: Any additional properties added in via extensions. - itemCollectionLinks: - type: array - description: >- - An array of links. Can be used for pagination, e.g. by providing a link - with the `next` relation type. - items: - $ref: '#/components/schemas/link' - example: - - rel: next - href: >- - http://sat-api.developmentseed.org/collections/landsat-8-l1/items/gasd312fsaeg - root: - type: object - required: - - links - properties: - links: - type: array - items: - $ref: '#/components/schemas/link' - example: - - href: 'http://sat-api.developmentseed.org' - rel: self - type: application/json - title: this document - - href: 'http://sat-api.developmentseed.org/api' - rel: service - type: application/json - title: this document - - href: 'http://sat-api.developmentseed.org/collections' - rel: data - type: application/json - title: Metadata about the feature collections - req-classes: - type: object - required: - - conformsTo - properties: - conformsTo: - type: array - items: - type: string - example: - - 'http://www.opengis.net/spec/wfs-1/3.0/req/core' - - 'http://www.opengis.net/spec/wfs-1/3.0/req/oas30' - - 'http://www.opengis.net/spec/wfs-1/3.0/req/html' - - 'http://www.opengis.net/spec/wfs-1/3.0/req/geojson' - content: - type: object - required: - - links - - collections - properties: - links: - type: array - items: - $ref: '#/components/schemas/link' - example: - - href: 'http://data.example.org/collections.json' - rel: self - type: application/json - title: this document - - href: 'http://data.example.org/collections.html' - rel: alternate - type: text/html - title: this document as HTML - - href: 'http://schemas.example.org/1.0/foobar.xsd' - rel: describedBy - type: application/xml - title: XML schema for Acme Corporation data - collections: - type: array - items: - $ref: '#/components/schemas/collectionInfo' - collectionInfo: - type: object - required: - - name - - links - - stac_version - - id - - description - - license - - extent - properties: - name: - description: 'identifier of the collection used, for example, in URIs' - type: string - example: buildings - title: - description: human readable title of the collection - type: string - example: Buildings - description: - description: a description of the features in the collection - type: string - example: Buildings in the city of Bonn. - links: - type: array - items: - $ref: '#/components/schemas/link' - example: - - href: 'http://data.example.org/collections/buildings/items' - rel: item - type: application/geo+json - title: Buildings - - href: 'http://example.org/concepts/building.html' - rel: describedBy - type: text/html - title: Feature catalogue for buildings - extent: - $ref: '#/components/schemas/extent' - crs: - description: >- - The coordinate reference systems in which geometries may be - retrieved. Coordinate reference systems are identified by a URI. The - first coordinate reference system is the coordinate reference system - that is used by default. This is always - "http://www.opengis.net/def/crs/OGC/1.3/CRS84", i.e. WGS84 - longitude/latitude. - type: array - items: - type: string - default: - - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - stac_version: - type: string - example: 0.6.0 - id: - description: 'identifier of the collection used, for example, in URIs' - type: string - example: buildings - keywords: - title: Keywords - type: array - items: - type: string - example: - - buildings - - properties - - constructions - version: - title: Collection Version - type: string - example: 1 - license: - title: Collection License Name - type: string - example: Apache-2.0 - providers: - type: array - items: - properties: - name: - title: Organization name - type: string - example: Big Building Corp - description: - title: Provider description - type: string - example: No further processing applied. - roles: - title: Organization roles - type: array - items: - type: string - enum: - - producer - - licensor - - processor - - host - example: - - producer - - licensor - url: - title: Homepage - description: >- - Homepage on which the provider describes the dataset and - publishes contact information. - type: string - format: url - example: 'http://www.big-building.com' - queryFilter: - type: object - description: Allows users to query properties for specific values - properties: - query: - $ref: '#/components/schemas/query' - query: - type: object - description: Define which properties to query and the operatations to apply - additionalProperties: - $ref: '#/components/schemas/queryProp' - example: - 'eo:cloud_cover': - lt: 50 - queryProp: - description: Apply query operations to a specific property - anyOf: - - description: >- - if the object doesn't contain any of the operators, it is equivalent - to using the equals operator - - type: object - description: Match using an operator - properties: - eq: - description: >- - Find items with a property that is equal to the specified value. - For strings, a case-insensitive comparison must be performed. - gt: - type: number - description: >- - Find items with a property value greater than the specified - value. - lt: - type: number - description: Find items with a property value less than the specified value. - gte: - type: number - description: >- - Find items with a property value greater than or equal the - specified value. - lte: - type: number - description: >- - Find items with a property value greater than or equal the - specified value. - sortFilter: - type: object - description: Sort the results - properties: - sort: - $ref: '#/components/schemas/sort' - sort: - type: array - description: | - An array of objects containing a property name and sort direction. - minItems: 1 - items: - type: object - required: - - field - properties: - field: - type: string - direction: - type: string - default: asc - enum: - - asc - - desc - example: - - field: 'eo:cloud_cover' - direction: desc - extent: - type: object - properties: - crs: - description: >- - Coordinate reference system of the coordinates in the spatial extent - (property `spatial`). In the Core, only WGS84 longitude/latitude is - supported. Extensions may support additional coordinate reference - systems. - type: string - enum: - - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - default: 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - spatial: - description: >- - West, south, east, north edges of the spatial extent. The minimum - and maximum values apply to the coordinate reference system WGS84 - longitude/latitude that is supported in the Core. If, for example, a - projected coordinate reference system is used, the minimum and - maximum values need to be adjusted. - type: array - minItems: 4 - maxItems: 6 - items: - type: number - example: - - -180 - - -90 - - 180 - - 90 - trs: - description: >- - Temporal reference system of the coordinates in the temporal extent - (property `temporal`). In the Core, only the Gregorian calendar is - supported. Extensions may support additional temporal reference - systems. - type: string - enum: - - 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - temporal: - description: Begin and end times of the temporal extent. - type: array - minItems: 2 - maxItems: 2 - items: - type: string - format: dateTime - example: - - '2011-11-11T12:22:11Z' - - '2012-11-24T12:32:43Z' - fieldsFilter: - type: object - description: Determines the shape of the features in the response - properties: - fields: - $ref: '#/components/schemas/fields' - fields: - description: > - The geometry member determines whether the geometry is populated or is - null. The - - include and exclude members specify an array of property names that are - either - - included or excluded from the result, respectively. If both include and - exclude - - are specified, include takes precedence. - - id and links are required feature properties and cannot be excluded. - type: object - properties: - geometry: - type: boolean - include: - type: array - items: - type: string - example: - - 'eo:cloud_cover' - exclude: - type: array - items: - type: string - example: - - 'eo:sun_azimuth' -tags: - - name: STAC - description: Extension to WFS3 Core to support STAC metadata model and search API diff --git a/packages/api-lib/libs/ElasticSearchWriteableStream.js b/packages/api-lib/libs/ElasticSearchWriteableStream.js index eaf2607..e04e106 100644 --- a/packages/api-lib/libs/ElasticSearchWriteableStream.js +++ b/packages/api-lib/libs/ElasticSearchWriteableStream.js @@ -1,5 +1,5 @@ const stream = require('stream') -const logger = require('./logger') +const logger = console //require('./logger') class ElasticSearchWritableStream extends stream.Writable { constructor(config, options) { diff --git a/packages/api-lib/libs/api.js b/packages/api-lib/libs/api.js index 9ce9896..ed9b5e4 100644 --- a/packages/api-lib/libs/api.js +++ b/packages/api-lib/libs/api.js @@ -1,11 +1,18 @@ const gjv = require('geojson-validation') const extent = require('@mapbox/extent') -const { feature } = require('@turf/helpers') -const logger = require('./logger') +const yaml = require('js-yaml') +const fs = require('fs') +const logger = console //require('./logger') + +// max number of collections to retrieve +const COLLECTION_LIMIT = process.env.SATAPI_COLLECTION_LIMIT || 100 + const extractIntersects = function (params) { let intersectsGeometry - const geojsonError = new Error('Invalid GeoJSON Feature or geometry') + const geojsonError = new Error('Invalid GeoJSON geometry') + const geojsonFeatureError = + new Error('Expected GeoJSON geometry, not Feature or FeatureCollection') const { intersects } = params if (intersects) { let geojson @@ -22,9 +29,9 @@ const extractIntersects = function (params) { if (gjv.valid(geojson)) { if (geojson.type === 'FeatureCollection') { - throw geojsonError - } else if (geojson.type !== 'Feature') { - geojson = feature(geojson) + throw geojsonFeatureError + } else if (geojson.type === 'Feature') { + throw geojsonFeatureError } intersectsGeometry = geojson } else { @@ -45,8 +52,7 @@ const extractBbox = function (params) { bboxArray = bbox } const boundingBox = extent(bboxArray) - const geojson = feature(boundingBox.polygon()) - intersectsGeometry = geojson + intersectsGeometry = boundingBox.polygon() } return intersectsGeometry } @@ -105,8 +111,26 @@ const extractIds = function (params) { return idsRules } + +const extractCollectionIds = function (params) { + let idsRules + const { collections } = params + if (collections) { + if (typeof collections === 'string') { + idsRules = JSON.parse(collections) + } else { + idsRules = collections.slice() + } + } + return idsRules +} + + const parsePath = function (path) { const searchFilters = { + root: false, + api: false, + conformance: false, stac: false, collections: false, search: false, @@ -114,6 +138,8 @@ const parsePath = function (path) { items: false, itemId: false } + const api = 'api' + const conformance = 'conformance' const stac = 'stac' const collections = 'collections' const search = 'search' @@ -121,6 +147,9 @@ const parsePath = function (path) { const pathComponents = path.split('/').filter((x) => x) const { length } = pathComponents + searchFilters.root = length === 0 + searchFilters.api = pathComponents[0] === api + searchFilters.conformance = pathComponents[0] === conformance searchFilters.stac = pathComponents[0] === stac searchFilters.collections = pathComponents[0] === collections searchFilters.collectionId = @@ -192,26 +221,6 @@ const addItemLinks = function (results, endpoint) { return results } -const buildRootObject = function (endpoint) { - const stac_docs_url = process.env.STAC_DOCS_URL - const root = { - links: [ - { - href: endpoint, - rel: 'self' - }, - { - href: `${endpoint}/collections`, - rel: 'data' - }, - { - href: stac_docs_url, - rel: 'service' - } - ] - } - return root -} const collectionsToCatalogLinks = function (results, endpoint) { const stac_version = process.env.STAC_VERSION @@ -231,14 +240,6 @@ const collectionsToCatalogLinks = function (results, endpoint) { href: `${endpoint}/collections/${id}` } }) - catalog.links.push({ - rel: 'self', - href: `${endpoint}/stac` - }) - catalog.links.push({ - rel: 'search', - href: `${endpoint}/stac/search` - }) return catalog } @@ -247,7 +248,9 @@ const wrapResponseInFeatureCollection = function ( ) { return { type: 'FeatureCollection', - meta, + 'search:metadata': meta, + 'numberMatched': meta.matched, + 'numberReturned': meta.returned, features, links } @@ -261,8 +264,8 @@ const buildPageLinks = function (meta, parameters, endpoint) { (p) => `${encodeURIComponent(p)}=${encodeURIComponent(JSON.stringify(dict[p]))}` ).join('&') ) - const { found, page, limit } = meta - if ((page * limit) < found) { + const { matched, page, limit } = meta + if ((page * limit) < matched) { const newParams = Object.assign({}, parameters, { page: page + 1, limit }) const nextQueryParameters = dictToURI(newParams) pageLinks.push({ @@ -274,21 +277,151 @@ const buildPageLinks = function (meta, parameters, endpoint) { return pageLinks } -const searchItems = async function (parameters, page, limit, backend, endpoint) { - const { results: itemsResults, meta: itemsMeta } = - await backend.search(parameters, 'items', page, limit) - const pageLinks = buildPageLinks(itemsMeta, parameters, endpoint) +const searchItems = async function (collectionId, queryParameters, backend, endpoint) { + const { + limit, + next, + datetime + } = queryParameters + const bbox = extractBbox(queryParameters) + const hasIntersects = extractIntersects(queryParameters) + if (bbox && hasIntersects) { + throw new Error('Expected bbox OR intersects, not both') + } + const sort = extractSort(queryParameters) + // Prefer intersects + const intersects = hasIntersects || bbox + const query = extractStacQuery(queryParameters) + const fields = extractFields(queryParameters) + const ids = extractIds(queryParameters) + const collections = extractCollectionIds(queryParameters) + + const parameters = { + datetime, + intersects, + query, + sort, + fields, + ids, + collections + } + + // Keep only existing parameters + const searchParameters = Object.keys(parameters) + .filter((key) => parameters[key]) + .reduce((obj, key) => ({ + ...obj, + [key]: parameters[key] + }), {}) + + if (collectionId) { + searchParameters.collections = [collectionId] + } + const { results: itemsResults, 'search:metadata': itemsMeta } = + await backend.search(searchParameters, 'items', next, limit) + const pageLinks = buildPageLinks(itemsMeta, searchParameters, endpoint) const items = addItemLinks(itemsResults, endpoint) const response = wrapResponseInFeatureCollection(itemsMeta, items, pageLinks) + return response } -const search = async function ( + +const getAPI = async function () { + const spec = yaml.safeLoad(fs.readFileSync('./api.yaml', 'utf8')) + return spec +} + + +const getConformance = async function () { + const conformance = { + conformsTo: [ + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson' + ] + } + return conformance +} + + +const getCatalog = async function (backend, endpoint = '') { + const { results } = await backend.search({}, 'collections', 1, COLLECTION_LIMIT) + const catalog = collectionsToCatalogLinks(results, endpoint) + catalog.links.push({ + rel: 'service-desc', + type: 'application/vnd.oai.openapi+json;version=3.0', + href: `${endpoint}/api` + }) + catalog.links.push({ + rel: 'conformance', + type: 'application/json', + href: `${endpoint}/conformance` + }) + catalog.links.push({ + rel: 'children', + type: 'application/json', + href: `${endpoint}/collections` + }) + catalog.links.push({ + rel: 'self', + type: 'application/json', + href: `${endpoint}/` + }) + catalog.links.push({ + rel: 'search', + type: 'application/json', + href: `${endpoint}/stac/search` + }) + if (process.env.STAC_DOCS_URL) { + catalog.links.push({ + rel: 'docs', + href: process.env.STAC_DOCS_URL + }) + } + return catalog +} + + +const getCollections = async function (backend, endpoint = '') { + const { results, 'search:metadata': meta } = + await backend.search({}, 'collections', 1, COLLECTION_LIMIT) + const linkedCollections = addCollectionLinks(results, endpoint) + return { 'search:metadata': meta, collections: linkedCollections } +} + + +const getCollection = async function (collectionId, backend, endpoint = '') { + const collectionQuery = { id: collectionId } + const { results } = await backend.search( + collectionQuery, 'collections', 1, 1 + ) + const col = addCollectionLinks(results, endpoint) + if (col.length > 0) { + return col[0] + } + return { code: 404, message: 'Collection not found' } +} + + +const getItem = async function (itemId, backend, endpoint = '') { + const itemQuery = { id: itemId } + const { results } = await backend.search(itemQuery, 'items') + const [it] = addItemLinks(results, endpoint) + if (it) { + return it + } + return { code: 404, message: 'Item not found' } +} + + +const API = async function ( path = '', queryParameters = {}, backend, endpoint = '' ) { let apiResponse try { const pathElements = parsePath(path) + const hasPathElement = Object.keys(pathElements).reduce((accumulator, key) => { let containsPathElement @@ -301,6 +434,9 @@ const search = async function ( }, false) const { + root, + api, + conformance, stac, search: searchPath, collections, @@ -309,108 +445,60 @@ const search = async function ( itemId } = pathElements - const { - limit, - page, - time: datetime - } = queryParameters - const bbox = extractBbox(queryParameters) - const hasIntersects = extractIntersects(queryParameters) - const sort = extractSort(queryParameters) - // Prefer intersects - const intersects = hasIntersects || bbox - const query = extractStacQuery(queryParameters) - const fields = extractFields(queryParameters) - const ids = extractIds(queryParameters) - const parameters = { - datetime, - intersects, - query, - sort, - fields, - ids + // API Root + if (root) { + apiResponse = await getCatalog(backend, endpoint) } - const colLimit = process.env.SATAPI_COLLECTION_LIMIT || 100 - // Keep only existing parameters - const searchParameters = Object.keys(parameters) - .filter((key) => parameters[key]) - .reduce((obj, key) => ({ - ...obj, - [key]: parameters[key] - }), {}) - // Landing page url - if (!hasPathElement) { - apiResponse = buildRootObject(endpoint) + // API Definition + if (api) { + apiResponse = await getAPI() + } + // Conformance + if (conformance) { + apiResponse = await getConformance() } // Root catalog with collection links - if (stac && !searchPath) { - const { results } = - await backend.search({}, 'collections', page, colLimit) - apiResponse = collectionsToCatalogLinks(results, endpoint) + if ((stac && !searchPath) || !hasPathElement) { + apiResponse = await getCatalog(backend, endpoint) } // STAC Search if (stac && searchPath) { apiResponse = await searchItems( - searchParameters, page, limit, backend, endpoint + null, queryParameters, backend, endpoint ) } // All collections if (collections && !collectionId) { - const { results, meta } = - await backend.search({}, 'collections', page, colLimit) - const linkedCollections = addCollectionLinks(results, endpoint) - apiResponse = { meta, collections: linkedCollections } + apiResponse = await getCollections(backend, endpoint) } // Specific collection if (collections && collectionId && !items) { - const collectionQuery = { id: collectionId } - const { results } = await backend.search( - collectionQuery, 'collections', page, limit - ) - const collection = addCollectionLinks(results, endpoint) - if (collection.length > 0) { - apiResponse = collection[0] - } else { - apiResponse = new Error('Collection not found') - } + apiResponse = await getCollection(collectionId, backend, endpoint) } // Items in a collection if (collections && collectionId && items && !itemId) { - const updatedQuery = Object.assign({}, searchParameters.query, { - collections: [ - collectionId - ] - }) - const itemIdParameters = Object.assign( - {}, searchParameters, { query: updatedQuery } - ) - apiResponse = await searchItems( - itemIdParameters, page, limit, backend, endpoint - ) + apiResponse = await searchItems(collectionId, queryParameters, + backend, endpoint) } if (collections && collectionId && items && itemId) { - const itemQuery = { id: itemId } - const { results } = await backend.search(itemQuery, 'items', page, limit) - const [item] = addItemLinks(results, endpoint) - if (item) { - apiResponse = item - } else { - apiResponse = new Error('Item not found') - } + apiResponse = await getItem(itemId, backend, endpoint) } } catch (error) { logger.error(error) - apiResponse = { - code: 500, - description: error.message - } + apiResponse = { code: 500, message: error.message } } return apiResponse } module.exports = { - search, - parsePath, + getAPI, + getConformance, + getCatalog, + getCollections, + getCollection, + getItem, searchItems, + API, + parsePath, extractIntersects } diff --git a/packages/api-lib/libs/es.js b/packages/api-lib/libs/es.js index c6f8f82..9731008 100644 --- a/packages/api-lib/libs/es.js +++ b/packages/api-lib/libs/es.js @@ -5,7 +5,7 @@ const httpAwsEs = require('http-aws-es') const elasticsearch = require('elasticsearch') const through2 = require('through2') const ElasticsearchWritableStream = require('./ElasticSearchWriteableStream') -const logger = require('./logger') +const logger = console //require('./logger') let _esClient /* @@ -14,14 +14,15 @@ searching records, and managing the indexes. It looks for the ES_HOST environmen variable which is the URL to the elasticsearch host */ -// Connect to an Elasticsearch cluster +// Connect to an Elasticsearch instance async function connect() { let esConfig - let client // use local client if (!process.env.ES_HOST) { - client = new elasticsearch.Client({ host: 'localhost:9200' }) + esConfig = { + host: 'localhost:9200' + } } else { await new Promise((resolve, reject) => AWS.config.getCredentials((err) => { if (err) return reject(err) @@ -42,13 +43,15 @@ async function connect() { // Note that this doesn't abort the query. requestTimeout: 120000 // milliseconds } - client = new elasticsearch.Client(esConfig) } + logger.debug(`Elasticsearch config: ${JSON.stringify(esConfig)}`) + const client = new elasticsearch.Client(esConfig) + await new Promise((resolve, reject) => client.ping({ requestTimeout: 1000 }, (err) => { if (err) { - reject('Unable to connect to elasticsearch') + reject(`Unable to connect to elasticsearch: ${err}`) } else { resolve() } @@ -80,6 +83,8 @@ async function prepare(index) { 'type': 'object', properties: { 'datetime': { type: 'date' }, + 'created': { type: 'date' }, + 'updated': { type: 'date' }, 'eo:cloud_cover': { type: 'float' }, 'eo:gsd': { type: 'float' }, 'eo:constellation': { type: 'keyword' }, @@ -176,6 +181,8 @@ async function _stream() { if (itemCollection) { const flatProperties = Object.assign({}, itemCollection.properties, data.properties) + flatProperties.created = new Date().toISOString() + flatProperties.updated = new Date().toISOString() esDataObject = Object.assign({}, esDataObject, { properties: flatProperties }) } else { logger.error(`${data.id} has no collection`) @@ -264,11 +271,9 @@ function buildDatetimeQuery(parameters) { function buildQuery(parameters) { const eq = 'eq' const inop = 'in' - const { query, intersects } = parameters + const { query, intersects, collections } = parameters let must = [] - const should = [] if (query) { - const { collections } = query // Using reduce rather than map as we don't currently support all // stac query operators. must = Object.keys(query).reduce((accumulator, property) => { @@ -296,20 +301,20 @@ function buildQuery(parameters) { } return accumulator }, must) - - if (collections) { - collections.forEach((collection) => { - should.push({ term: { 'collection': collection } }) - }) - } } + if (collections) { + must.push({ + terms: { + 'collection': collections + } + }) + } if (intersects) { - const { geometry } = intersects must.push({ geo_shape: { - geometry: { shape: geometry } + geometry: { shape: intersects } } }) } @@ -319,7 +324,7 @@ function buildQuery(parameters) { must.push(datetimeQuery) } - const filter = { bool: { must, should } } + const filter = { bool: { must } } const queryBody = { constant_score: { filter } } @@ -372,30 +377,39 @@ function buildSort(parameters) { return sorting } +/* function buildFieldsFilter(parameters) { - const id = 'id' const { fields } = parameters - const _sourceInclude = [] - const _sourceExclude = [] + let _sourceInclude = [ + 'id', + 'type', + 'geometry', + 'bbox', + 'links', + 'assets', + 'collection', + 'properties.datetime' + ] + let _sourceExclude = [] if (fields) { const { include, exclude } = fields - if (include && include.length > 0) { - const propertiesIncludes = include.map( - (field) => (`${field}`) - ).concat( - [id] - ) - _sourceInclude.push(...propertiesIncludes) - } + // Remove exclude fields from the default include list and add them to the source exclude list if (exclude && exclude.length > 0) { - const filteredExcludes = exclude.filter((field) => - (![id].includes(field))) - const propertiesExclude = filteredExcludes.map((field) => (`${field}`)) - _sourceExclude.push(...propertiesExclude) + _sourceInclude = _sourceInclude.filter((field) => !exclude.includes(field)) + _sourceExclude = exclude + } + // Add include fields to the source include list if they're not already in it + if (include && include.length > 0) { + include.forEach((field) => { + if (_sourceInclude.indexOf(field) < 0) { + _sourceInclude.push(field) + } + }) } } - return { _sourceExclude, _sourceInclude } + return { _sourceInclude, _sourceExclude } } +*/ async function search(parameters, index = '*', page = 1, limit = 10) { let body @@ -419,22 +433,25 @@ async function search(parameters, index = '*', page = 1, limit = 10) { from: (page - 1) * limit } - const { _sourceExclude, _sourceInclude } = buildFieldsFilter(parameters) + /* disable fields filter for now + const { _sourceInclude, _sourceExclude } = buildFieldsFilter(parameters) if (_sourceExclude.length > 0) { searchParams._sourceExclude = _sourceExclude } if (_sourceInclude.length > 0) { searchParams._sourceInclude = _sourceInclude } + */ + const client = await esClient() const resultBody = await client.search(searchParams) const results = resultBody.hits.hits.map((r) => (r._source)) const response = { results, - meta: { - page, + 'search:metadata': { + next: (((page * limit) < resultBody.hits.total) ? page + 1 : null), limit, - found: resultBody.hits.total, + matched: resultBody.hits.total, returned: results.length } } diff --git a/packages/api-lib/libs/ingest.js b/packages/api-lib/libs/ingest.js index 8a05396..95cbeb6 100644 --- a/packages/api-lib/libs/ingest.js +++ b/packages/api-lib/libs/ingest.js @@ -7,8 +7,7 @@ const isUrl = require('is-url') const MemoryStream = require('memorystream') const { Readable } = require('readable-stream') const pump = require('pump') -const uuid = require('uuid/v4') -const logger = require('./logger') +const logger = console //require('./logger') const limiter = new Bottleneck({ maxConcurrent: 500 @@ -150,8 +149,7 @@ async function ingest(url, backend, recursive = true, collectionsOnly = false) { await backend.prepare('collections') await backend.prepare('items') const { toEs, esStream } = await backend.stream() - const ingestJobId = uuid() - logger.info(`${ingestJobId} Started`) + logger.info(`Job Started for ${url}`) const promise = new Promise((resolve, reject) => { pump( duplexStream, @@ -162,7 +160,7 @@ async function ingest(url, backend, recursive = true, collectionsOnly = false) { logger.error(error) reject(error) } else { - logger.info(`${ingestJobId} Completed`) + logger.info(`Job completed for ${url}`) resolve(true) } } diff --git a/packages/api-lib/libs/logger.js b/packages/api-lib/libs/logger.js index af1177b..bf34479 100644 --- a/packages/api-lib/libs/logger.js +++ b/packages/api-lib/libs/logger.js @@ -1,10 +1,20 @@ const winston = require('winston') -const logger = new (winston.Logger)({ - level: process.env.LOG_LEVEL || 'info', + +const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'debug', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.colorize(), + winston.format.printf((info) => `[${info.timestamp}] ${info.level}: ${info.message}`) + ), transports: [ - new (winston.transports.Console)({ timestamp: true }) + new winston.transports.Console({ + format: winston.format.simple(), + level: process.env.LOG_LEVEL || 'debug' + }) ] }) + module.exports = logger diff --git a/packages/api-lib/package.json b/packages/api-lib/package.json index 7cb528f..e9dec8d 100644 --- a/packages/api-lib/package.json +++ b/packages/api-lib/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/api-lib", - "version": "0.3.0", + "version": "0.4.0-rc2", "description": "A library for creating a search API of public Satellites metadata using Elasticsearch", "main": "index.js", "scripts": { @@ -32,13 +32,13 @@ "geojson-validation": "^0.1.6", "http-aws-es": "^4.0.0", "is-url": "^1.2.4", + "js-yaml": "^3.13.1", "memorystream": "^0.3.1", "pump": "^3.0.0", "request": "^2.88.0", "request-promise-native": "^1.0.5", "through2": "^3.0.1", - "uuid": "^3.3.2", - "winston": "^2.2.0" + "winston": "^3.2.0" }, "devDependencies": { "ava": "^0.16.0", diff --git a/packages/api-lib/tests/fixtures/item.json b/packages/api-lib/tests/fixtures/item.json index fd88741..0e2eb8a 100644 --- a/packages/api-lib/tests/fixtures/item.json +++ b/packages/api-lib/tests/fixtures/item.json @@ -11,7 +11,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B1.TIF", "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ @@ -19,7 +19,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B10.TIF", "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ @@ -27,7 +27,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B11.TIF", "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ @@ -35,7 +35,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B2.TIF", "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ @@ -43,7 +43,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B3.TIF", "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ @@ -51,7 +51,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B4.TIF", "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ @@ -59,7 +59,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B5.TIF", "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ @@ -67,7 +67,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B6.TIF", "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ @@ -75,7 +75,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B7.TIF", "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ @@ -83,7 +83,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B8.TIF", "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ @@ -91,12 +91,12 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B9.TIF", "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_BQA.TIF", "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_MTL.txt", @@ -162,5 +162,10 @@ "landsat:row": "10", "landsat:scene_id": "LC80100102015082LGN00" }, - "type": "Feature" + "type": "Feature", + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json b/packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json index 8242458..9ec629d 100644 --- a/packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json +++ b/packages/api-lib/tests/fixtures/stac/LC80100102015050LGN00.json @@ -11,7 +11,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B1.TIF", "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ @@ -19,7 +19,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B10.TIF", "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ @@ -27,7 +27,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B11.TIF", "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ @@ -35,7 +35,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B2.TIF", "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ @@ -43,7 +43,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B3.TIF", "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ @@ -51,7 +51,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B4.TIF", "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ @@ -59,7 +59,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B5.TIF", "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ @@ -67,7 +67,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B6.TIF", "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ @@ -75,7 +75,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B7.TIF", "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ @@ -83,7 +83,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B8.TIF", "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ @@ -91,12 +91,12 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B9.TIF", "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_BQA.TIF", "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_MTL.txt", @@ -179,5 +179,10 @@ "landsat:row": "10", "landsat:scene_id": "LC80100102015050LGN00" }, - "type": "Feature" + "type": "Feature", + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json b/packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json index 6d5e58b..3e2b305 100644 --- a/packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json +++ b/packages/api-lib/tests/fixtures/stac/LC80100102015082LGN00.json @@ -11,7 +11,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B1.TIF", "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ @@ -19,7 +19,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B10.TIF", "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ @@ -27,7 +27,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B11.TIF", "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ @@ -35,7 +35,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B2.TIF", "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ @@ -43,7 +43,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B3.TIF", "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ @@ -51,7 +51,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B4.TIF", "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ @@ -59,7 +59,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B5.TIF", "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ @@ -67,7 +67,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B6.TIF", "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ @@ -75,7 +75,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B7.TIF", "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ @@ -83,7 +83,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B8.TIF", "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ @@ -91,12 +91,12 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_B9.TIF", "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_BQA.TIF", "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015082LGN00/LC80100102015082LGN00_MTL.txt", @@ -179,5 +179,10 @@ "landsat:row": "10", "landsat:scene_id": "LC80100102015082LGN00" }, - "type": "Feature" + "type": "Feature", + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/badGeometryItem.json b/packages/api-lib/tests/fixtures/stac/badGeometryItem.json index 079f2db..b21ab88 100644 --- a/packages/api-lib/tests/fixtures/stac/badGeometryItem.json +++ b/packages/api-lib/tests/fixtures/stac/badGeometryItem.json @@ -11,7 +11,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B1.TIF", "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ @@ -19,7 +19,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B10.TIF", "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ @@ -27,7 +27,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B11.TIF", "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ @@ -35,7 +35,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B2.TIF", "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ @@ -43,7 +43,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B3.TIF", "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ @@ -51,7 +51,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B4.TIF", "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ @@ -59,7 +59,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B5.TIF", "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ @@ -67,7 +67,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B6.TIF", "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ @@ -75,7 +75,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B7.TIF", "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ @@ -83,7 +83,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B8.TIF", "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ @@ -91,12 +91,12 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_B9.TIF", "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_BQA.TIF", "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/096/008/LC80960082014185LGN00/LC80960082014185LGN00_MTL.txt", @@ -310,5 +310,10 @@ "landsat:scene_id": "LC80960082014185LGN00", "landsat:tier": "pre-collection" }, - "type": "Feature" + "type": "Feature", + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/catalog.json b/packages/api-lib/tests/fixtures/stac/catalog.json index 2b63169..40c7ae3 100644 --- a/packages/api-lib/tests/fixtures/stac/catalog.json +++ b/packages/api-lib/tests/fixtures/stac/catalog.json @@ -19,5 +19,9 @@ "rel": "child" } ], - "stac_version": "0.7.0" + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/collection.json b/packages/api-lib/tests/fixtures/stac/collection.json index 24499a0..149e15a 100644 --- a/packages/api-lib/tests/fixtures/stac/collection.json +++ b/packages/api-lib/tests/fixtures/stac/collection.json @@ -9,81 +9,81 @@ 0 ], "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B10": { "eo:bands": [ 9 ], "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B11": { "eo:bands": [ 10 ], "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B2": { "eo:bands": [ 1 ], "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B3": { "eo:bands": [ 2 ], "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B4": { "eo:bands": [ 3 ], "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B5": { "eo:bands": [ 4 ], "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B6": { "eo:bands": [ 5 ], "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B7": { "eo:bands": [ 6 ], "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B8": { "eo:bands": [ 7 ], "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B9": { "eo:bands": [ 8 ], "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "BQA": { "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "MTL": { "title": "original metadata file", @@ -260,7 +260,11 @@ "url": "https://github.com/sat-utils/sat-api" } ], - "stac_version": "0.7.0", + "stac_version": "0.8.0", + "stac_extensions": [ + "eo", + "landsat" + ], "title": "Landsat 8 L1", "version": "0.1.0" } diff --git a/packages/api-lib/tests/fixtures/stac/collection2.json b/packages/api-lib/tests/fixtures/stac/collection2.json index 40ba95e..b2f2e17 100644 --- a/packages/api-lib/tests/fixtures/stac/collection2.json +++ b/packages/api-lib/tests/fixtures/stac/collection2.json @@ -9,81 +9,81 @@ 0 ], "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B10": { "eo:bands": [ 9 ], "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B11": { "eo:bands": [ 10 ], "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B2": { "eo:bands": [ 1 ], "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B3": { "eo:bands": [ 2 ], "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B4": { "eo:bands": [ 3 ], "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B5": { "eo:bands": [ 4 ], "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B6": { "eo:bands": [ 5 ], "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B7": { "eo:bands": [ 6 ], "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B8": { "eo:bands": [ 7 ], "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "B9": { "eo:bands": [ 8 ], "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "BQA": { "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; apllication=geotiff" }, "MTL": { "title": "original metadata file", @@ -252,7 +252,11 @@ "url": "https://github.com/sat-utils/sat-api" } ], - "stac_version": "0.7.0", + "stac_version": "0.8.0", + "stac_extensions": [ + "eo", + "landsat" + ], "title": "Landsat 8 L1", "version": "0.1.0" } diff --git a/packages/api-lib/tests/fixtures/stac/collection2_item.json b/packages/api-lib/tests/fixtures/stac/collection2_item.json index e5253c8..72f9edf 100644 --- a/packages/api-lib/tests/fixtures/stac/collection2_item.json +++ b/packages/api-lib/tests/fixtures/stac/collection2_item.json @@ -11,7 +11,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B1.TIF", "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ @@ -19,7 +19,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B10.TIF", "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ @@ -27,7 +27,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B11.TIF", "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ @@ -35,7 +35,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B2.TIF", "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ @@ -43,7 +43,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B3.TIF", "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ @@ -51,7 +51,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B4.TIF", "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ @@ -59,7 +59,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B5.TIF", "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ @@ -67,7 +67,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B6.TIF", "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ @@ -75,7 +75,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B7.TIF", "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ @@ -83,7 +83,7 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B8.TIF", "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ @@ -91,12 +91,12 @@ ], "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_B9.TIF", "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_BQA.TIF", "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "href": "https://s3-us-west-2.amazonaws.com/landsat-pds/L8/010/010/LC80100102015050LGN00/LC80100102015050LGN00_MTL.txt", @@ -179,5 +179,10 @@ "landsat:row": "10", "landsat:scene_id": "collection2_item" }, - "type": "Feature" + "type": "Feature", + "stac_extensions": [ + "eo", + "landsat" + ], + "stac_version": "0.8.0" } diff --git a/packages/api-lib/tests/fixtures/stac/collectionNoChildren.json b/packages/api-lib/tests/fixtures/stac/collectionNoChildren.json index f1d0452..0a65310 100644 --- a/packages/api-lib/tests/fixtures/stac/collectionNoChildren.json +++ b/packages/api-lib/tests/fixtures/stac/collectionNoChildren.json @@ -9,81 +9,81 @@ 0 ], "title": "Band 1 (coastal)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B10": { "eo:bands": [ 9 ], "title": "Band 10 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B11": { "eo:bands": [ 10 ], "title": "Band 11 (lwir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B2": { "eo:bands": [ 1 ], "title": "Band 2 (blue)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B3": { "eo:bands": [ 2 ], "title": "Band 3 (green)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B4": { "eo:bands": [ 3 ], "title": "Band 4 (red)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B5": { "eo:bands": [ 4 ], "title": "Band 5 (nir)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B6": { "eo:bands": [ 5 ], "title": "Band 6 (swir16)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B7": { "eo:bands": [ 6 ], "title": "Band 7 (swir22)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B8": { "eo:bands": [ 7 ], "title": "Band 8 (pan)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "B9": { "eo:bands": [ 8 ], "title": "Band 9 (cirrus)", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "BQA": { "title": "Band quality data", - "type": "image/x.geotiff" + "type": "image/tiff; application=geotiff" }, "MTL": { "title": "original metadata file", @@ -248,7 +248,11 @@ "url": "https://github.com/sat-utils/sat-api" } ], - "stac_version": "0.7.0", + "stac_version": "0.8.0", + "stac_extensions": [ + "eo", + "landsat" + ], "title": "Landsat 8 L1", "version": "0.1.0" } diff --git a/packages/api-lib/tests/fixtures/stac/intersectsGeometry.json b/packages/api-lib/tests/fixtures/stac/intersectsGeometry.json new file mode 100644 index 0000000..5a0c63c --- /dev/null +++ b/packages/api-lib/tests/fixtures/stac/intersectsGeometry.json @@ -0,0 +1,27 @@ +{ + "type": "Polygon", + "coordinates": [ + [ + [ + -54.58007812499999, + 69.39578308847753 + ], + [ + -49.0869140625, + 69.39578308847753 + ], + [ + -49.0869140625, + 70.72897946208789 + ], + [ + -54.58007812499999, + 70.72897946208789 + ], + [ + -54.58007812499999, + 69.39578308847753 + ] + ] + ] +} diff --git a/packages/api-lib/tests/fixtures/stac/noIntersectsFeature.json b/packages/api-lib/tests/fixtures/stac/noIntersectsFeature.json deleted file mode 100644 index feaf691..0000000 --- a/packages/api-lib/tests/fixtures/stac/noIntersectsFeature.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "type": "Feature", - "properties": {}, - "geometry": { - "type": "Polygon", - "coordinates": [ - [ - [ - -71.4111328125, - 67.1016555307692 - ], - [ - -67.67578124999999, - 67.1016555307692 - ], - [ - -67.67578124999999, - 68.12248241161676 - ], - [ - -71.4111328125, - 68.12248241161676 - ], - [ - -71.4111328125, - 67.1016555307692 - ] - ] - ] - } -} diff --git a/packages/api-lib/tests/fixtures/stac/noIntersectsGeometry.json b/packages/api-lib/tests/fixtures/stac/noIntersectsGeometry.json new file mode 100644 index 0000000..2d5b79f --- /dev/null +++ b/packages/api-lib/tests/fixtures/stac/noIntersectsGeometry.json @@ -0,0 +1,27 @@ +{ + "type": "Polygon", + "coordinates": [ + [ + [ + -71.4111328125, + 67.1016555307692 + ], + [ + -67.67578124999999, + 67.1016555307692 + ], + [ + -67.67578124999999, + 68.12248241161676 + ], + [ + -71.4111328125, + 68.12248241161676 + ], + [ + -71.4111328125, + 67.1016555307692 + ] + ] + ] +} diff --git a/packages/api-lib/tests/integration/test_api.js b/packages/api-lib/tests/integration/test_api.js index 3bd343d..2a6edb9 100644 --- a/packages/api-lib/tests/integration/test_api.js +++ b/packages/api-lib/tests/integration/test_api.js @@ -5,26 +5,27 @@ process.env.AWS_SECRET_ACCESS_KEY = 'none' const backend = require('../../libs/es') const api = require('../../libs/api') const intersectsFeature = require('../fixtures/stac/intersectsFeature.json') -const noIntersectsFeature = require('../fixtures/stac/noIntersectsFeature.json') +const intersectsGeometry = require('../fixtures/stac/intersectsGeometry.json') +const noIntersectsGeometry = require('../fixtures/stac/noIntersectsGeometry.json') -const { search } = api +const { API } = api const endpoint = 'endpoint' test('collections', async (t) => { - const response = await search('/collections', {}, backend, endpoint) + const response = await API('/collections', {}, backend, endpoint) t.is(response.collections.length, 2) - t.is(response.meta.returned, 2) + t.is(response['search:metadata'].returned, 2) }) test('collections/{collectionId}', async (t) => { - let response = await search('/collections/landsat-8-l1', {}, backend, endpoint) + let response = await API('/collections/landsat-8-l1', {}, backend, endpoint) t.is(response.id, 'landsat-8-l1') - response = await search('/collections/collection2', {}, backend, endpoint) + response = await API('/collections/collection2', {}, backend, endpoint) t.is(response.id, 'collection2') }) test('collections/{collectionId}/items', async (t) => { - const response = await search('/collections/landsat-8-l1/items', + const response = await API('/collections/landsat-8-l1/items', {}, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features.length, 2) @@ -34,48 +35,56 @@ test('collections/{collectionId}/items', async (t) => { test('collections/{collectionId}/items/{itemId}', async (t) => { const response = - await search('/collections/landsat-8-l1/items/LC80100102015082LGN00', + await API('/collections/landsat-8-l1/items/LC80100102015082LGN00', {}, backend, endpoint) t.is(response.type, 'Feature') t.is(response.id, 'LC80100102015082LGN00') }) test('collections/{collectionId}/items with bbox', async (t) => { - let response = await search('/collections/landsat-8-l1/items', { + let response = await API('/collections/landsat-8-l1/items', { bbox: [-180, -90, 180, 90] }, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features[0].id, 'LC80100102015082LGN00') t.is(response.features[1].id, 'LC80100102015050LGN00') - response = await search('/collections/landsat-8-l1/items', { + response = await API('/collections/landsat-8-l1/items', { bbox: [-5, -5, 5, 5] }, backend, endpoint) t.is(response.features.length, 0) }) +test('collections/{collectionId}/items with bbox and intersects', async (t) => { + const response = await API('/collections/landsat-8-l1/items', { + bbox: [-180, -90, 180, 90], + intersects: intersectsGeometry + }, backend, endpoint) + t.truthy(response.code) +}) + test('collections/{collectionId}/items with time', async (t) => { - let response = await search('/collections/landsat-8-l1/items', { - time: '2015-02-19T15:06:12.565047+00:00' + let response = await API('/collections/landsat-8-l1/items', { + datetime: '2015-02-19T15:06:12.565047+00:00' }, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features[0].id, 'LC80100102015050LGN00') - response = await search('/collections/landsat-8-l1/items', { - time: '2015-02-17/2015-02-20' + response = await API('/collections/landsat-8-l1/items', { + datetime: '2015-02-17/2015-02-20' }, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features[0].id, 'LC80100102015050LGN00') - response = await search('/collections/landsat-8-l1/items', { - time: '2015-02-19/2015-02-20' + response = await API('/collections/landsat-8-l1/items', { + datetime: '2015-02-19/2015-02-20' }, backend, endpoint) t.is(response.features[0].id, 'LC80100102015050LGN00', 'Handles date range without times inclusion issue') }) test('collections/{collectionId}/items with limit', async (t) => { - const response = await search('/collections/landsat-8-l1/items', { + const response = await API('/collections/landsat-8-l1/items', { limit: 1 }, backend, endpoint) t.is(response.type, 'FeatureCollection') @@ -83,21 +92,26 @@ test('collections/{collectionId}/items with limit', async (t) => { }) test('collections/{collectionId}/items with intersects', async (t) => { - let response = await search('/collections/landsat-8-l1/items', { - intersects: intersectsFeature + let response = await API('/collections/landsat-8-l1/items', { + intersects: intersectsGeometry }, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features[0].id, 'LC80100102015082LGN00') t.is(response.features[1].id, 'LC80100102015050LGN00') - response = await search('/collections/landsat-8-l1/items', { - intersects: noIntersectsFeature + response = await API('/collections/landsat-8-l1/items', { + intersects: intersectsFeature + }, backend, endpoint) + t.truthy(response.code) + + response = await API('/collections/landsat-8-l1/items', { + intersects: noIntersectsGeometry }, backend, endpoint) t.is(response.features.length, 0) }) test('collections/{collectionId}/items with eq query', async (t) => { - const response = await search('/collections/landsat-8-l1/items', { + const response = await API('/collections/landsat-8-l1/items', { query: { 'eo:cloud_cover': { eq: 0.54 @@ -109,7 +123,7 @@ test('collections/{collectionId}/items with eq query', async (t) => { }) test('collections/{collectionId}/items with gt lt query', async (t) => { - const response = await search('/collections/landsat-8-l1/items', { + const response = await API('/collections/landsat-8-l1/items', { query: { 'eo:cloud_cover': { gt: 0.5, @@ -123,30 +137,30 @@ test('collections/{collectionId}/items with gt lt query', async (t) => { test('stac', async (t) => { - const response = await search('/stac', {}, backend, endpoint) - t.is(response.links.length, 4) + const response = await API('/stac', {}, backend, endpoint) + t.is(response.links.length, 5) }) test('stac/search bbox', async (t) => { - let response = await search('/stac/search', { + let response = await API('/stac/search', { bbox: [-180, -90, 180, 90] }, backend, endpoint) t.is(response.type, 'FeatureCollection') t.is(response.features[0].id, 'LC80100102015082LGN00') t.is(response.features[1].id, 'collection2_item') - response = await search('/stac/search', { + response = await API('/stac/search', { bbox: [-5, -5, 5, 5] }, backend, endpoint) t.is(response.features.length, 0) }) test('stac/search default sort', async (t) => { - const response = await search('/stac/search', {}, backend, endpoint) + const response = await API('/stac/search', {}, backend, endpoint) t.is(response.features[0].id, 'LC80100102015082LGN00') }) test('stac/search sort', async (t) => { - let response = await search('/stac/search', { + let response = await API('/stac/search', { sort: [{ field: 'eo:cloud_cover', direction: 'desc' @@ -154,14 +168,14 @@ test('stac/search sort', async (t) => { }, backend, endpoint) t.is(response.features[0].id, 'LC80100102015082LGN00') - response = await search('/stac/search', { + response = await API('/stac/search', { sort: '[{ "field": "eo:cloud_cover", "direction": "desc" }]' }, backend, endpoint) t.is(response.features[0].id, 'LC80100102015082LGN00') }) test('stac/search flattened collection properties', async (t) => { - let response = await search('/stac/search', { + let response = await API('/stac/search', { query: { 'eo:platform': { eq: 'platform2' @@ -170,11 +184,14 @@ test('stac/search flattened collection properties', async (t) => { }, backend, endpoint) t.is(response.features[0].id, 'collection2_item') - response = await search('/stac/search', { + response = await API('/stac/search', { query: { 'eo:platform': { eq: 'landsat-8' } + }, + fields: { + include: ['properties.eo:platform'] } }, backend, endpoint) const havePlatform = @@ -185,25 +202,45 @@ test('stac/search flattened collection properties', async (t) => { }) test('stac/search fields filter', async (t) => { - let response = await search('/stac/search', { + let response = await API('/stac/search', { + fields: { + } + }, backend, endpoint) + t.truthy(response.features[0].collection) + t.truthy(response.features[0].id) + t.truthy(response.features[0].type) + t.truthy(response.features[0].geometry) + t.truthy(response.features[0].bbox) + t.truthy(response.features[0].links) + t.truthy(response.features[0].assets) + + response = await API('/stac/search', { fields: { exclude: ['collection'] } }, backend, endpoint) t.falsy(response.features[0].collection) - response = await search('/stac/search', { + response = await API('/stac/search', { fields: { exclude: ['geometry'] } }, backend, endpoint) t.falsy(response.features[0].geometry) - response = await search('/stac/search', { + response = await API('/stac/search', { + fields: { + include: ['properties'], + exclude: ['properties.datetime'] + } + }, backend, endpoint) + t.falsy(response.features[0].properties.datetime) + + response = await API('/stac/search', { }, backend, endpoint) t.truthy(response.features[0].geometry) - response = await search('/stac/search', { + response = await API('/stac/search', { fields: { include: ['collection', 'properties.eo:epsg'] } @@ -212,7 +249,7 @@ test('stac/search fields filter', async (t) => { t.truthy(response.features[0].properties['eo:epsg']) t.falsy(response.features[0].properties['eo:cloud_cover']) - response = await search('/stac/search', { + response = await API('/stac/search', { fields: { exclude: ['id', 'links'] } @@ -220,8 +257,23 @@ test('stac/search fields filter', async (t) => { t.truthy(response.features.length, 'Does not exclude required fields') }) +test('stac/search created and updated', async (t) => { + const response = await API('/stac/search', { + query: { + 'eo:platform': { + eq: 'landsat-8' + } + }, + fields: { + include: ['properties.created', 'properties.updated'] + } + }, backend, endpoint) + t.truthy(response.features[0].properties.created) + t.truthy(response.features[0].properties.updated) +}) + test('stac/search in query', async (t) => { - const response = await search('/stac/search', { + const response = await API('/stac/search', { query: { 'landsat:path': { in: ['10'] @@ -231,8 +283,31 @@ test('stac/search in query', async (t) => { t.is(response.features.length, 3) }) +test('stac/search limit next query', async (t) => { + let response = await API('/stac/search', { + query: { + 'landsat:path': { + in: ['10'] + } + }, + limit: 2 + }, backend, endpoint) + t.is(response.features.length, 2) + + response = await API('/stac/search', { + query: { + 'landsat:path': { + in: ['10'] + } + }, + limit: 2, + next: response['search:metadata'].next + }, backend, endpoint) + t.is(response.features.length, 1) +}) + test('stac/search ids', async (t) => { - const response = await search('/stac/search', { + const response = await API('/stac/search', { ids: ['collection2_item', 'LC80100102015050LGN00'] }, backend, endpoint) t.is(response.features.length, 2) @@ -241,26 +316,32 @@ test('stac/search ids', async (t) => { }) test('stac/search collections', async (t) => { - let response = await search('/stac/search', { + let response = await API('/stac/search', { query: { - collections: ['collection2'] + collection: { + in: ['collection2'] + } } }, backend, endpoint) t.is(response.features.length, 1) t.is(response.features[0].id, 'collection2_item') - response = await search('/stac/search', { + response = await API('/stac/search', { query: { - collections: ['landsat-8-l1'] + collection: { + in: ['landsat-8-l1'] + } } }, backend, endpoint) t.is(response.features.length, 2) t.is(response.features[0].id, 'LC80100102015082LGN00') t.is(response.features[1].id, 'LC80100102015050LGN00') - response = await search('/stac/search', { + response = await API('/stac/search', { query: { - collections: ['collection2', 'landsat-8-l1'] + collection: { + in: ['collection2', 'landsat-8-l1'] + } } }, backend, endpoint) t.is(response.features.length, 3) diff --git a/packages/api-lib/tests/test_api_parsePath.js b/packages/api-lib/tests/test_api_parsePath.js index b78527f..3145d03 100644 --- a/packages/api-lib/tests/test_api_parsePath.js +++ b/packages/api-lib/tests/test_api_parsePath.js @@ -3,6 +3,51 @@ const api = require('../libs/api') test('parsePath', (t) => { let expected = { + root: true, + api: false, + conformance: false, + stac: false, + collections: false, + search: false, + collectionId: false, + items: false, + itemId: false + } + let actual = api.parsePath('/') + t.deepEqual(actual, expected) + + expected = { + root: false, + api: true, + conformance: false, + stac: false, + collections: false, + search: false, + collectionId: false, + items: false, + itemId: false + } + actual = api.parsePath('/api') + t.deepEqual(actual, expected) + + expected = { + root: false, + api: false, + conformance: true, + stac: false, + collections: false, + search: false, + collectionId: false, + items: false, + itemId: false + } + actual = api.parsePath('/conformance') + t.deepEqual(actual, expected) + + expected = { + root: false, + api: false, + conformance: false, stac: true, collections: false, search: false, @@ -10,10 +55,13 @@ test('parsePath', (t) => { items: false, itemId: false } - let actual = api.parsePath('/stac') + actual = api.parsePath('/stac') t.deepEqual(actual, expected) expected = { + root: false, + api: false, + conformance: false, stac: true, collections: false, search: true, @@ -25,6 +73,9 @@ test('parsePath', (t) => { t.deepEqual(actual, expected) expected = { + root: false, + api: false, + conformance: false, stac: false, collections: true, search: false, @@ -36,6 +87,9 @@ test('parsePath', (t) => { t.deepEqual(actual, expected) expected = { + root: false, + api: false, + conformance: false, stac: false, collections: true, search: false, @@ -47,6 +101,9 @@ test('parsePath', (t) => { t.deepEqual(actual, expected) expected = { + root: false, + api: false, + conformance: false, stac: false, collections: true, search: false, @@ -58,6 +115,9 @@ test('parsePath', (t) => { t.deepEqual(actual, expected) expected = { + root: false, + api: false, + conformance: false, stac: false, collections: true, search: false, diff --git a/packages/api-lib/tests/test_api_search.js b/packages/api-lib/tests/test_api_search.js index 63d795d..2e9b369 100644 --- a/packages/api-lib/tests/test_api_search.js +++ b/packages/api-lib/tests/test_api_search.js @@ -19,42 +19,36 @@ test('search es error', async (t) => { const errorMessage = 'errorMessage' const search = sinon.stub().throws(new Error(errorMessage)) const backend = { search } - const response = await proxyApi.search('/stac', undefined, backend, 'endpoint') + const response = await proxyApi.API('/stac', undefined, backend, 'endpoint') t.is(error.firstCall.args[0].message, errorMessage, 'Logs Elasticsearch error via Winston transport') - t.is(response.description, errorMessage) + t.is(response.message, errorMessage) t.is(response.code, 500) }) test('search /', async (t) => { - process.env.STAC_DOCS_URL = 'test' - const endpoint = 'endpoint' - const expected = { - links: [ - { - href: endpoint, - rel: 'self' - }, - { - href: `${endpoint}/collections`, - rel: 'data' - }, - { - href: 'test', - rel: 'service' - } - ] - } - const actual = await api.search('/', undefined, {}, endpoint) - t.deepEqual(actual, expected, 'Returns root API node') + const actual = await api.API('/', undefined, undefined, 'endpoint') + t.is(actual.links.length, 4) +}) + +test('search /api', async (t) => { + const actual = await api.API('/api', undefined, undefined, 'endpoint') + t.truthy(actual.openapi) +}) + +test('search /conformance', async (t) => { + const actual = await api.API('/conformance', undefined, undefined, 'endpoint') + t.truthy(actual.conformsTo) + t.is(actual.conformsTo.length, 3) }) test('search /stac', async (t) => { + process.env.STAC_DOCS_URL = 'test' const collection = 'collection' const results = { results: [{ id: collection }] } const search = sinon.stub().resolves(results) const backend = { search } - const actual = await api.search('/stac', undefined, backend, 'endpoint') + const actual = await api.API('/stac', undefined, backend, 'endpoint') const expectedLinks = [ { rel: 'child', @@ -67,6 +61,10 @@ test('search /stac', async (t) => { { rel: 'search', href: 'endpoint/stac/search' + }, + { + href: 'test', + rel: 'service' } ] t.is(search.firstCall.args[1], 'collections') @@ -76,31 +74,30 @@ test('search /stac', async (t) => { test('search /stac/search wraps results', async (t) => { const limit = 10 - const page = 1 const meta = { limit, - page, - found: 1, + next: null, + matched: 1, returned: 1 } const clonedItem = cloneMutatedItem() const results = [clonedItem] - const itemsResults = { meta, results } + const itemsResults = { 'search:metadata': meta, results } const search = sinon.stub() search.resolves(itemsResults) const backend = { search } - const actual = await api.search('/stac/search', {}, backend, 'endpoint') + const actual = await api.API('/stac/search', {}, backend, 'endpoint') t.deepEqual(actual.features[0].links, itemLinks.links, 'Adds correct relative STAC links') const expectedMeta = { limit, - page, - found: 1, + next: null, + matched: 1, returned: 1 } - t.deepEqual(actual.meta, expectedMeta, 'Adds correct response metadata fields') + t.deepEqual(actual['search:metadata'], expectedMeta, 'Adds correct response metadata fields') t.is(actual.type, 'FeatureCollection', 'Wraps response as FeatureCollection') }) @@ -113,7 +110,7 @@ test('search /stac/search query parameters', async (t) => { limit: 2, query } - api.search('/stac/search', queryParams, backend, 'endpoint') + api.API('/stac/search', queryParams, backend, 'endpoint') t.deepEqual(search.firstCall.args[0], { query }, 'Extracts query to use in search parameters') }) @@ -122,18 +119,18 @@ test('search /stac/search intersects parameter', async (t) => { const search = sinon.stub().resolves({ results: [], meta: {} }) const backend = { search } const queryParams = { - intersects: item, + intersects: item.geometry, page: 1, limit: 1 } - api.search('/stac/search', queryParams, backend, 'endpoint') - t.deepEqual(search.firstCall.args[0].intersects, item, + api.API('/stac/search', queryParams, backend, 'endpoint') + t.deepEqual(search.firstCall.args[0].intersects, item.geometry, 'Uses valid GeoJSON as intersects search parameter') search.resetHistory() - queryParams.intersects = JSON.stringify(item) - api.search('/stac/search', queryParams, backend, 'endpoint') - t.deepEqual(search.firstCall.args[0].intersects, item, + queryParams.intersects = JSON.stringify(item.geometry) + api.API('/stac/search', queryParams, backend, 'endpoint') + t.deepEqual(search.firstCall.args[0].intersects, item.geometry, 'Handles stringified GeoJSON intersects parameter') }) @@ -151,25 +148,21 @@ test('search /stac/search bbox parameter', async (t) => { limit: 1 } const expected = { - type: 'Feature', - properties: {}, - geometry: { - type: 'Polygon', - coordinates: [[ - [s, w], - [n, w], - [n, e], - [s, e], - [s, w] - ]] - } + type: 'Polygon', + coordinates: [[ + [s, w], + [n, w], + [n, e], + [s, e], + [s, w] + ]] } - await api.search('/stac/search', queryParams, backend, 'endpoint') + await api.API('/stac/search', queryParams, backend, 'endpoint') t.deepEqual(search.firstCall.args[0].intersects, expected, 'Converts a [w,s,e,n] bbox to an intersects search parameter') search.resetHistory() queryParams.bbox = `[${bbox.toString()}]` - await api.search('/stac/search', queryParams, backend, 'endpoint') + await api.API('/stac/search', queryParams, backend, 'endpoint') t.deepEqual(search.firstCall.args[0].intersects, expected, 'Converts stringified [w,s,e,n] bbox to an intersects search parameter') }) @@ -181,9 +174,9 @@ test('search /stac/search time parameter', async (t) => { const queryParams = { page: 1, limit: 2, - time: range + datetime: range } - await api.search('/stac/search', queryParams, backend, 'endpoint') + await api.API('/stac/search', queryParams, backend, 'endpoint') t.deepEqual(search.firstCall.args[0], { datetime: range }, 'Extracts time query parameter and transforms it into ' + 'datetime search parameter') @@ -204,7 +197,7 @@ test('search /collections', async (t) => { }] }) const backend = { search } - const actual = await api.search('/collections', {}, backend, 'endpoint') + const actual = await api.API('/collections', {}, backend, 'endpoint') t.is(search.firstCall.args[1], 'collections') t.is(actual.collections.length, 1) t.is(actual.collections[0].links.length, 4, 'Adds STAC links to each collection') @@ -226,7 +219,7 @@ test('search /collections/collectionId', async (t) => { }) const backend = { search } const collectionId = 'collectionId' - let actual = await api.search( + let actual = await api.API( `/collections/${collectionId}`, { test: 'test' }, backend, 'endpoint' ) t.deepEqual(search.firstCall.args[0], { id: collectionId }, @@ -239,7 +232,7 @@ test('search /collections/collectionId', async (t) => { meta, results: [] }) - actual = await api.search( + actual = await api.API( `/collections/${collectionId}`, {}, backend, 'endpoint' ) t.is(actual.message, 'Collection not found', @@ -260,16 +253,14 @@ test('search /collections/collectionId/items', async (t) => { }) const backend = { search } const collectionId = 'collectionId' - await api.search( + await api.API( `/collections/${collectionId}/items`, {}, backend, 'endpoint' ) const expectedParameters = { - query: { - collections: [collectionId] - } + collections: [collectionId] } t.deepEqual(search.firstCall.args[0], expectedParameters, - 'Calls search with the collectionId as part of the query parameter') + 'Calls search with the collectionId as a parameter') }) test('search /collections/collectionId/items/itemId', async (t) => { @@ -287,7 +278,7 @@ test('search /collections/collectionId/items/itemId', async (t) => { }) const backend = { search } const itemId = 'itemId' - const actual = await api.search( + const actual = await api.API( `/collections/collectionId/items/${itemId}`, {}, backend, 'endpoint' ) t.deepEqual(search.firstCall.args[0], { id: itemId }, diff --git a/packages/api/api.yaml b/packages/api/api.yaml new file mode 100644 index 0000000..4ef5f4d --- /dev/null +++ b/packages/api/api.yaml @@ -0,0 +1,1744 @@ +openapi: 3.0.1 +info: + title: The SpatioTemporal Asset Catalog API + Extensions + version: 0.8.1 + license: + name: Apache License 2.0 + url: 'http://www.apache.org/licenses/LICENSE-2.0' + description: >- + This is an OpenAPI definition of the core SpatioTemporal Asset Catalog API + specification. Any service that implements this endpoint to allow search of + spatiotemporal assets can be considered a STAC API. The endpoint is also + available as an OpenAPI fragment that can be integrated with other OpenAPI + definitions, and is designed to slot seamlessly into a OGC API - Features + definition. + contact: + name: STAC Specification + url: 'http://stacspec.org' +tags: + - name: Capabilities + description: essential characteristics of this API + - name: Data + description: access to data (features) + - name: STAC + description: >- + Extension to OGC API - Features to support STAC metadata model and search + API +paths: + /: + get: + tags: + - Capabilities + summary: landing page + description: |- + The landing page provides links to the API definition, the conformance + statements and to the feature collections in this dataset. + operationId: getLandingPage + responses: + '200': + $ref: '#/components/responses/LandingPage' + '500': + $ref: '#/components/responses/ServerError' + /conformance: + get: + tags: + - Capabilities + summary: information about specifications that this API conforms to + description: |- + A list of all conformance classes specified in a standard that the + server conforms to. + operationId: getConformanceDeclaration + responses: + '200': + $ref: '#/components/responses/ConformanceDeclaration' + '500': + $ref: '#/components/responses/ServerError' + /collections: + get: + tags: + - Capabilities + summary: the feature collections in the dataset + operationId: getCollections + responses: + '200': + $ref: '#/components/responses/Collections' + '500': + $ref: '#/components/responses/ServerError' + '/collections/{collectionId}': + get: + tags: + - Capabilities + summary: describe the feature collection with id `collectionId` + operationId: describeCollection + parameters: + - $ref: '#/components/parameters/collectionId' + responses: + '200': + $ref: '#/components/responses/Collection' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/ServerError' + '/collections/{collectionId}/items': + get: + tags: + - Data + summary: fetch features + description: |- + Fetch features of the feature collection with id `collectionId`. + Every feature in a dataset belongs to a collection. A dataset may + consist of multiple feature collections. A feature collection is often a + collection of features of a similar type, based on a common schema. + Use content negotiation to request HTML or GeoJSON. + operationId: getFeatures + parameters: + - $ref: '#/components/parameters/collectionId' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/bbox' + - $ref: '#/components/parameters/datetime' + responses: + '200': + $ref: '#/components/responses/Features' + '400': + $ref: '#/components/responses/InvalidParameter' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/ServerError' + '/collections/{collectionId}/items/{featureId}': + get: + tags: + - Data + summary: fetch a single feature + description: |- + Fetch the feature with id `featureId` in the feature collection + with id `collectionId`. + Use content negotiation to request HTML or GeoJSON. + operationId: getFeature + parameters: + - $ref: '#/components/parameters/collectionId' + - $ref: '#/components/parameters/featureId' + responses: + '200': + $ref: '#/components/responses/Feature' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/ServerError' + /stac: + get: + summary: Return the root catalog or collection. + description: >- + Returns the root STAC Catalog or STAC Collection that is the entry point + for users to browse with STAC Browser or for search engines to crawl. + This can either return a single STAC Collection or more commonly a STAC + catalog that usually lists sub-catalogs of STAC Collections, i.e. a + simple catalog that lists all collections available through the API. + tags: + - STAC + responses: + '200': + description: A catalog JSON definition. Used as an entry point for a crawler. + content: + application/json: + schema: + $ref: '#/components/schemas/catalogDefinition' + /stac/search: + get: + summary: Search STAC items with simple filtering. + description: >- + Retrieve Items matching filters. Intended as a shorthand API for simple + queries. This method is optional, but you MUST implement `POST + /stac/search` if you want to implement this method. + operationId: getSearchSTAC + tags: + - STAC + parameters: + - $ref: '#/components/parameters/bbox' + - $ref: '#/components/parameters/datetime' + - $ref: '#/components/parameters/limit' + - $ref: '#/components/parameters/next' + - $ref: '#/components/parameters/ids' + - $ref: '#/components/parameters/collectionsArray' + - $ref: '#/components/parameters/query' + - $ref: '#/components/parameters/sort' + - $ref: '#/components/parameters/fields' + responses: + '200': + description: A feature collection. + content: + application/geo+json: + schema: + $ref: '#/components/schemas/itemCollection' + text/html: + schema: + type: string + default: + description: An error occurred. + content: + application/json: + schema: + $ref: '#/components/schemas/exception' + text/html: + schema: + type: string + post: + summary: Search STAC items with full-featured filtering. + description: >- + retrieve items matching filters. Intended as the standard, full-featured + query API. This method is mandatory to implement if `GET /stac/search` + is implemented. If this endpoint is implemented on a server, it is + required to add a link with `rel` set to `search` to the `links` array + in `GET /stac` that refers to this endpoint. + operationId: postSearchSTAC + tags: + - STAC + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/searchBody' + responses: + '200': + description: A feature collection. + content: + application/geo+json: + schema: + $ref: '#/components/schemas/itemCollection' + text/html: + schema: + type: string + default: + description: An error occurred. + content: + application/json: + schema: + $ref: '#/components/schemas/exception' + text/html: + schema: + type: string +components: + parameters: + bbox: + name: bbox + in: query + description: >- + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, depending + on whether the coordinate reference system includes a vertical axis + (height or depth): * Lower left corner, coordinate axis 1 * Lower left + corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) + * Upper right corner, coordinate axis 1 * Upper right corner, coordinate + axis 2 * Maximum value, coordinate axis 3 (optional) The coordinate + reference system of the values is WGS 84 longitude/latitude + (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different + coordinate reference system is specified in the parameter `bbox-crs`. + For WGS 84 longitude/latitude the values are in most cases the sequence + of minimum longitude, minimum latitude, maximum longitude and maximum + latitude. However, in cases where the box spans the antimeridian the + first value (west-most box edge) is larger than the third value + (east-most box edge). If the vertical axis is included, the third and + the sixth number are the bottom and the top of the 3-dimensional + bounding box. If a feature has multiple spatial geometry properties, it + is the decision of the server whether only a single spatial geometry + property is used to determine the extent or all relevant geometries. + required: false + schema: + type: array + minItems: 4 + maxItems: 6 + items: + type: number + style: form + explode: false + collectionId: + name: collectionId + in: path + description: local identifier of a collection + required: true + schema: + type: string + datetime: + name: datetime + in: query + description: >- + Either a date-time or an interval, open or closed. Date and time + expressions adhere to RFC 3339. Open intervals are expressed using + double-dots. Examples: * A date-time: "2018-02-12T23:20:50Z" * A closed + interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Open intervals: + "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" Only features + that have a temporal property that intersects the value of `datetime` + are selected. If a feature has multiple temporal properties, it is the + decision of the server whether only a single temporal property is used + to determine the extent or all relevant temporal properties. + required: false + schema: + type: string + style: form + explode: false + featureId: + name: featureId + in: path + description: local identifier of a feature + required: true + schema: + type: string + limit: + name: limit + in: query + description: >- + The optional limit parameter limits the number of items that are + presented in the response document. Only items are counted that are on + the first level of the collection in the response document. Nested + objects contained within the explicitly requested items shall not be + counted. Minimum = 1. Maximum = 10000. Default = 10. + required: false + schema: + type: integer + minimum: 1 + maximum: 10000 + default: 10 + style: form + explode: false + next: + name: next + in: query + description: >- + The token to retrieve the next set of results, e.g., offset, page, + continuation token + required: false + schema: + $ref: '#/components/schemas/next' + style: form + ids: + name: ids + in: query + description: > + Array of Item ids to return. All other filter parameters that further + restrict the number of search results (except `next` and `limit`) are + ignored + required: false + schema: + $ref: '#/components/schemas/ids' + explode: false + collectionsArray: + name: collections + in: query + description: | + Array of Collection IDs to include in the search for items. + Only Items in one of the provided Collections will be searched + required: false + schema: + $ref: '#/components/schemas/collectionsArray' + explode: false + query: + name: query + in: query + description: >- + query for properties in items. Use the JSON form of the queryFilter used + in POST. + required: false + schema: + type: string + sort: + name: sort + in: query + description: Allows sorting results by the specified properties + required: false + schema: + $ref: '#/components/schemas/sort' + fields: + name: fields + in: query + description: Determines the shape of the features in the response + required: false + schema: + $ref: '#/components/schemas/fields' + style: form + explode: false + schemas: + collection: + type: object + required: + - id + - links + - stac_version + - description + - license + - extent + properties: + id: + description: 'identifier of the collection used, for example, in URIs' + type: string + example: address + title: + description: human readable title of the collection + type: string + example: address + description: + description: >- + Detailed multi-line description to fully explain the collection. + [CommonMark 0.29](http://commonmark.org/) syntax MAY be used for + rich text representation. + type: string + example: An address. + links: + type: array + items: + $ref: '#/components/schemas/link' + example: + - href: 'http://data.example.com/buildings' + rel: item + - href: 'http://example.com/concepts/buildings.html' + rel: describedBy + type: text/html + extent: + $ref: '#/components/schemas/extent' + itemType: + description: >- + indicator about the type of the items in the collection (the default + value is 'feature'). + type: string + default: feature + crs: + description: the list of coordinate reference systems supported by the service + type: array + items: + type: string + default: + - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + example: + - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + - 'http://www.opengis.net/def/crs/EPSG/0/4326' + stac_version: + $ref: '#/components/schemas/stac_version' + stac_extensions: + $ref: '#/components/schemas/stac_extensions' + keywords: + type: array + description: List of keywords describing the collection. + items: + type: string + version: + type: string + description: Version of the collection. + license: + $ref: '#/components/schemas/license' + providers: + $ref: '#/components/schemas/providers' + summaries: + $ref: '#/components/schemas/summaries' + example: + stac_version: 0.8.1 + stac_extensions: [] + id: Sentinel-2 + title: 'Sentinel-2 MSI: MultiSpectral Instrument, Level-1C' + description: | + Sentinel-2 is a wide-swath, high-resolution, multi-spectral + imaging mission... + license: proprietary + keywords: + - copernicus + - esa + - eu + - msi + - radiance + - sentinel + providers: + - name: ESA + roles: + - producer + - licensor + url: 'https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi' + extent: + spatial: + bbox: + - - -180 + - -56 + - 180 + - 83 + temporal: + interval: + - - '2015-06-23T00:00:00Z' + - '2019-07-10T13:44:56Z' + summaries: + datetime: + min: '2015-06-23T00:00:00Z' + max: '2019-07-10T13:44:56Z' + 'sci:citation': + - 'Copernicus Sentinel data [Year]' + 'eo:gsd': + - 10 + - 30 + - 60 + 'eo:platform': + - sentinel-2a + - sentinel-2b + 'eo:constellation': + - sentinel-2 + 'eo:instrument': + - msi + 'eo:off_nadir': + min: 0 + max: 100 + 'eo:sun_elevation': + min: 6.78 + max: 89.9 + 'eo:bands': + - - name: B1 + common_name: coastal + center_wavelength: 4.439 + gsd: 60 + - name: B2 + common_name: blue + center_wavelength: 4.966 + gsd: 10 + - name: B3 + common_name: green + center_wavelength: 5.6 + gsd: 10 + - name: B4 + common_name: red + center_wavelength: 6.645 + gsd: 10 + - name: B5 + center_wavelength: 7.039 + gsd: 20 + - name: B6 + center_wavelength: 7.402 + gsd: 20 + - name: B7 + center_wavelength: 7.825 + gsd: 20 + - name: B8 + common_name: nir + center_wavelength: 8.351 + gsd: 10 + - name: B8A + center_wavelength: 8.648 + gsd: 20 + - name: B9 + center_wavelength: 9.45 + gsd: 60 + - name: B10 + center_wavelength: 1.3735 + gsd: 60 + - name: B11 + common_name: swir16 + center_wavelength: 1.6137 + gsd: 20 + - name: B12 + common_name: swir22 + center_wavelength: 2.2024 + gsd: 20 + links: + - rel: self + href: 'http://cool-sat.com/collections/Sentinel-2' + - rel: root + href: 'http://cool-sat.com/collections' + - rel: license + href: >- + https://scihub.copernicus.eu/twiki/pub/SciHubWebPortal/TermsConditions/Sentinel_Data_Terms_and_Conditions.pdf + title: >- + Legal notice on the use of Copernicus Sentinel Data and Service + Information + collections: + type: object + required: + - links + - collections + properties: + links: + type: array + items: + $ref: '#/components/schemas/link' + collections: + type: array + items: + $ref: '#/components/schemas/collection' + confClasses: + type: object + required: + - conformsTo + properties: + conformsTo: + type: array + items: + type: string + exception: + type: object + description: >- + Information about the exception: an error code plus an optional + description. + required: + - code + properties: + code: + type: string + description: + type: string + extent: + type: object + description: >- + The extent of the features in the collection. In the Core only spatial + and temporal extents are specified. Extensions may add additional + members to represent other extents, for example, thermal or pressure + ranges. + properties: + spatial: + description: The spatial extent of the features in the collection. + type: object + properties: + bbox: + description: >- + One or more bounding boxes that describe the spatial extent of + the dataset. In the Core only a single bounding box is + supported. Extensions may support additional areas. If multiple + areas are provided, the union of the bounding boxes describes + the spatial extent. + type: array + minItems: 1 + items: + description: >- + Each bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes + a vertical axis (height or depth): * Lower left corner, + coordinate axis 1 * Lower left corner, coordinate axis 2 * + Minimum value, coordinate axis 3 (optional) * Upper right + corner, coordinate axis 1 * Upper right corner, coordinate + axis 2 * Maximum value, coordinate axis 3 (optional) The + coordinate reference system of the values is WGS 84 + longitude/latitude + (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a + different coordinate reference system is specified in `crs`. + For WGS 84 longitude/latitude the values are in most cases the + sequence of minimum longitude, minimum latitude, maximum + longitude and maximum latitude. However, in cases where the + box spans the antimeridian the first value (west-most box + edge) is larger than the third value (east-most box edge). If + the vertical axis is included, the third and the sixth number + are the bottom and the top of the 3-dimensional bounding box. + If a feature has multiple spatial geometry properties, it is + the decision of the server whether only a single spatial + geometry property is used to determine the extent or all + relevant geometries. + type: array + minItems: 4 + maxItems: 6 + items: + type: number + example: + - -180 + - -90 + - 180 + - 90 + crs: + description: >- + Coordinate reference system of the coordinates in the spatial + extent (property `bbox`). The default reference system is WGS 84 + longitude/latitude. In the Core this is the only supported + coordinate reference system. Extensions may support additional + coordinate reference systems and add additional enum values. + type: string + enum: + - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + default: 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + required: + - bbox + temporal: + description: The temporal extent of the features in the collection. + type: object + properties: + interval: + description: >- + One or more time intervals that describe the temporal extent of + the dataset. The value `null` is supported and indicates an open + time intervall. In the Core only a single time interval is + supported. Extensions may support multiple intervals. If + multiple intervals are provided, the union of the intervals + describes the temporal extent. + type: array + minItems: 1 + items: + description: >- + Begin and end times of the time interval. The timestamps are + in the coordinate reference system specified in `trs`. By + default this is the Gregorian calendar. + type: array + minItems: 2 + maxItems: 2 + items: + type: string + format: date-time + nullable: true + example: + - '2011-11-11T12:22:11Z' + - null + trs: + description: >- + Coordinate reference system of the coordinates in the temporal + extent (property `interval`). The default reference system is + the Gregorian calendar. In the Core this is the only supported + temporal reference system. Extensions may support additional + temporal reference systems and add additional enum values. + type: string + enum: + - 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + required: + - interval + required: + - spatial + - temporal + featureCollectionGeoJSON: + type: object + required: + - type + - features + properties: + type: + type: string + enum: + - FeatureCollection + features: + type: array + items: + $ref: '#/components/schemas/item' + links: + type: array + items: + $ref: '#/components/schemas/link' + timeStamp: + $ref: '#/components/schemas/timeStamp' + numberMatched: + $ref: '#/components/schemas/numberMatched' + numberReturned: + $ref: '#/components/schemas/numberReturned' + featureGeoJSON: + type: object + required: + - type + - geometry + - properties + properties: + type: + type: string + enum: + - Feature + geometry: + $ref: '#/components/schemas/geometryGeoJSON' + properties: + type: object + nullable: true + id: + oneOf: + - type: string + - type: integer + links: + type: array + items: + $ref: '#/components/schemas/link' + geometryGeoJSON: + oneOf: + - $ref: '#/components/schemas/pointGeoJSON' + - $ref: '#/components/schemas/multipointGeoJSON' + - $ref: '#/components/schemas/linestringGeoJSON' + - $ref: '#/components/schemas/multilinestringGeoJSON' + - $ref: '#/components/schemas/polygonGeoJSON' + - $ref: '#/components/schemas/multipolygonGeoJSON' + - $ref: '#/components/schemas/geometrycollectionGeoJSON' + geometrycollectionGeoJSON: + type: object + required: + - type + - geometries + properties: + type: + type: string + enum: + - GeometryCollection + geometries: + type: array + items: + $ref: '#/components/schemas/geometryGeoJSON' + landingPage: + type: object + required: + - links + properties: + title: + type: string + example: Buildings in Bonn + description: + type: string + example: >- + Access to data about buildings in the city of Bonn via a Web API + that conforms to the OGC API Features specification. + links: + type: array + items: + $ref: '#/components/schemas/link' + linestringGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - LineString + coordinates: + type: array + minItems: 2 + items: + type: array + minItems: 2 + items: + type: number + link: + type: object + required: + - href + - rel + properties: + href: + type: string + example: 'http://www.geoserver.example/stac/naip/child/catalog.json' + format: url + rel: + type: string + example: child + type: + type: string + example: application/geo+json + hreflang: + type: string + example: en + title: + type: string + example: NAIP Child Catalog + length: + type: integer + title: Link + multilinestringGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - MultiLineString + coordinates: + type: array + items: + type: array + minItems: 2 + items: + type: array + minItems: 2 + items: + type: number + multipointGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - MultiPoint + coordinates: + type: array + items: + type: array + minItems: 2 + items: + type: number + multipolygonGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - MultiPolygon + coordinates: + type: array + items: + type: array + items: + type: array + minItems: 4 + items: + type: array + minItems: 2 + items: + type: number + numberMatched: + description: |- + The number of features of the feature type that match the selection + parameters like `bbox`. + type: integer + minimum: 0 + example: 127 + numberReturned: + description: |- + The number of features in the feature collection. + A server may omit this information in a response, if the information + about the number of features is not known or difficult to compute. + If the value is provided, the value shall be identical to the number + of items in the "features" array. + type: integer + minimum: 0 + example: 10 + pointGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - Point + coordinates: + type: array + minItems: 2 + items: + type: number + polygonGeoJSON: + type: object + required: + - type + - coordinates + properties: + type: + type: string + enum: + - Polygon + coordinates: + type: array + items: + type: array + minItems: 4 + items: + type: array + minItems: 2 + items: + type: number + timeStamp: + description: >- + This property indicates the time and date when the response was + generated. + type: string + format: date-time + example: '2017-08-17T08:05:32Z' + license: + type: string + description: >- + License(s) of the data as a SPDX [License + identifier](https://spdx.org/licenses/) or + [expression](https://spdx.org/spdx-specification-21-web-version#h.jxpfx0ykyb60). + Alternatively, use `proprietary` if the license is not on the SPDX + license list or `various` if multiple licenses apply. In these two cases + links to the license texts SHOULD be added, see the `license` link + relation type. Non-SPDX licenses SHOULD add a link to the license text + with the `license` relation in the links section. The license text MUST + NOT be provided as a value of this field. If there is no public license + URL available, it is RECOMMENDED to host the license text and link to + it. + example: Apache-2.0 + providers: + type: array + description: >- + A list of providers, which may include all organizations capturing or + processing the data or the hosting provider. Providers should be listed + in chronological order with the most recent provider being the last + element of the list. + items: + type: object + title: Provider + required: + - name + properties: + name: + description: The name of the organization or the individual. + type: string + description: + description: >- + Multi-line description to add further provider information such as + processing details for processors and producers, hosting details + for hosts or basic contact information. CommonMark 0.29 syntax MAY + be used for rich text representation. + type: string + roles: + description: >- + Roles of the provider. The provider's role(s) can be one or more + of the following elements: * licensor: The organization that is + licensing the dataset under the license specified in the + collection's license field. * producer: The producer of the data + is the provider that initially captured and processed the source + data, e.g. ESA for Sentinel-2 data. * processor: A processor is + any provider who processed data to a derived product. * host: The + host is the actual provider offering the data on their storage. + There should be no more than one host, specified as last element + of the list. + type: array + items: + type: string + enum: + - producer + - licensor + - processor + - host + url: + description: >- + Homepage on which the provider describes the dataset and publishes + contact information. + type: string + format: url + searchBody: + description: The search criteria + type: object + allOf: + - $ref: '#/components/schemas/bboxFilter' + - $ref: '#/components/schemas/datetimeFilter' + - $ref: '#/components/schemas/intersectsFilter' + - $ref: '#/components/schemas/nextFilter' + - $ref: '#/components/schemas/collectionsFilter' + - $ref: '#/components/schemas/idsFilter' + - $ref: '#/components/schemas/limitFilter' + - $ref: '#/components/schemas/queryFilter' + - $ref: '#/components/schemas/sortFilter' + - $ref: '#/components/schemas/fieldsFilter' + next: + type: string + description: >- + The token to retrieve the next set of results, e.g., offset, page, + continuation token + default: null + limit: + type: integer + minimum: 1 + example: 10 + default: 10 + maximum: 10000 + description: The maximum number of results to return (page size). Defaults to 10 + bbox: + description: | + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a + vertical axis (elevation or depth): + * Lower left corner, coordinate axis 1 + * Lower left corner, coordinate axis 2 + * Lower left corner, coordinate axis 3 (optional) + * Upper right corner, coordinate axis 1 + * Upper right corner, coordinate axis 2 + * Upper right corner, coordinate axis 3 (optional) + The coordinate reference system of the values is WGS84 + longitude/latitude (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless + a different coordinate reference system is specified in the parameter + `bbox-crs`. + For WGS84 longitude/latitude the values are in most cases the sequence + of minimum longitude, minimum latitude, maximum longitude and maximum + latitude. However, in cases where the box spans the antimeridian the + first value (west-most box edge) is larger than the third value + (east-most box edge). + If a feature has multiple spatial geometry properties, it is the + decision of the server whether only a single spatial geometry property + is used to determine the extent or all relevant geometries. + type: array + minItems: 4 + maxItems: 6 + items: + type: number + example: + - -110 + - 39.5 + - -105 + - 40.5 + bboxFilter: + type: object + description: Only return items that intersect the provided bounding box. + properties: + bbox: + $ref: '#/components/schemas/bbox' + collectionsArray: + type: array + description: | + Array of Collection IDs to include in the search for items. + Only Items in one of the provided Collections will be searched + items: + type: string + ids: + type: array + description: > + Array of Item ids to return. All other filter parameters that further + restrict the number of search results (except `next` and `limit`) are + ignored + items: + type: string + datetimeFilter: + description: An object representing a date+time based filter. + type: object + properties: + datetime: + $ref: '#/components/schemas/datetime' + intersectsFilter: + type: object + description: Only returns items that intersect with the provided polygon. + properties: + intersects: + $ref: 'https://geojson.org/schema/Geometry.json' + limitFilter: + type: object + description: Only returns maximum number of results (page size) + properties: + limit: + $ref: '#/components/schemas/limit' + nextFilter: + type: object + description: Only returns the next set of results + properties: + next: + $ref: '#/components/schemas/next' + idsFilter: + type: object + description: Only returns items that match the array of given ids + properties: + ids: + $ref: '#/components/schemas/ids' + collectionsFilter: + type: object + description: Only returns the collections specified + properties: + collections: + $ref: '#/components/schemas/collectionsArray' + datetime: + type: string + description: >- + Either a date-time or an interval, open or closed. Date and time + expressions adhere to RFC 3339. Open intervals are expressed using + double-dots. Examples: * A date-time: "2018-02-12T23:20:50Z" * A closed + interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" * Open intervals: + "2018-02-12T00:00:00Z/.." or "../2018-03-18T12:31:12Z" Only features + that have a temporal property that intersects the value of `datetime` + are selected. If a feature has multiple temporal properties, it is the + decision of the server whether only a single temporal property is used + to determine the extent or all relevant temporal properties. + example: '2018-02-12T00:00:00Z/2018-03-18T12:31:12Z' + stac_version: + title: STAC version + type: string + example: 0.8.1 + stac_extensions: + title: STAC extensions + type: array + uniqueItems: true + items: + anyOf: + - title: Reference to a JSON Schema + type: string + format: uri + - title: Reference to a core extension + type: string + summaries: + description: >- + Summaries are either a unique set of all available values *or* + statistics. Statistics by default only specify the range (minimum and + maximum values), but can optionally be accompanied by additional + statistical values. The range can specify the potential range of values, + but it is recommended to be as precise as possible. The set of values + must contain at least one element and it is strongly recommended to list + all values. It is recommended to list as many properties as reasonable + so that consumers get a full overview of the Collection. Properties that + are covered by the Collection specification (e.g. `providers` and + `license`) may not be repeated in the summaries. + type: object + additionalProperties: + oneOf: + - type: array + title: Set of values + items: + description: A value of any type. + - type: object + title: Statistics + description: >- + By default, only ranges with a minimum and a maximum value can be + specified. Ranges can be specified for ordinal values only, which + means they need to have a rank order. Therefore, ranges can only + be specified for numbers and some special types of strings. + Examples: grades (A to F), dates or times. Implementors are free + to add other derived statistical values to the object, for example + `mean` or `stddev`. + required: + - min + - max + properties: + min: + anyOf: + - type: string + - type: number + max: + anyOf: + - type: string + - type: number + catalogDefinition: + type: object + required: + - stac_version + - id + - description + - links + properties: + stac_version: + $ref: '#/components/schemas/stac_version' + stac_extensions: + $ref: '#/components/schemas/stac_extensions' + id: + type: string + example: naip + title: + type: string + example: NAIP Imagery + description: + type: string + example: Catalog of NAIP Imagery. + summaries: + $ref: '#/components/schemas/summaries' + links: + type: array + items: + anyOf: + - $ref: '#/components/schemas/link' + - title: Link to search endpoint + description: >- + Link the search endpoint, which is **required** to be + specified if the API implements `/stac/search`. + type: object + required: + - href + - rel + properties: + href: + type: string + format: url + example: 'http://www.cool-sat.com/stac/search' + rel: + type: string + enum: + - search + type: + type: string + title: + type: string + itemCollection: + description: >- + A GeoJSON FeatureCollection augmented with foreign members that contain + values relevant to a STAC entity + type: object + required: + - features + - type + properties: + type: + type: string + enum: + - FeatureCollection + features: + type: array + items: + $ref: '#/components/schemas/item' + links: + $ref: '#/components/schemas/itemCollectionLinks' + 'search:metadata': + type: object + required: + - next + - returned + properties: + next: + type: string + nullable: true + limit: + type: integer + nullable: true + minimum: 0 + example: 5 + matched: + type: integer + minimum: 0 + example: 1 + returned: + type: integer + minimum: 0 + example: 1 + item: + description: >- + A GeoJSON Feature augmented with foreign members that contain values + relevant to a STAC entity + type: object + required: + - stac_version + - id + - type + - geometry + - bbox + - links + - properties + - assets + properties: + stac_version: + $ref: '#/components/schemas/stac_version' + stac_extensions: + $ref: '#/components/schemas/stac_extensions' + id: + $ref: '#/components/schemas/itemId' + bbox: + $ref: '#/components/schemas/bbox' + geometry: + $ref: 'https://geojson.org/schema/Geometry.json' + type: + $ref: '#/components/schemas/itemType' + properties: + $ref: '#/components/schemas/itemProperties' + links: + type: array + items: + $ref: '#/components/schemas/link' + assets: + $ref: '#/components/schemas/itemAssets' + example: + stac_version: 0.8.1 + stac_extensions: + - eo + - 'https://example.com/cs-extension/1.0/schema.json' + type: Feature + id: CS3-20160503_132131_05 + bbox: + - -122.59750209 + - 37.48803556 + - -122.2880486 + - 37.613537207 + geometry: + type: Polygon + coordinates: + - - - -122.308150179 + - 37.488035566 + - - -122.597502109 + - 37.538869539 + - - -122.576687533 + - 37.613537207 + - - -122.2880486 + - 37.562818007 + - - -122.308150179 + - 37.488035566 + properties: + datetime: '2016-05-03T13:22:30.040Z' + title: A CS3 item + license: PDDL-1.0 + providers: + - name: CoolSat + roles: + - producer + - licensor + url: 'https://cool-sat.com/' + 'eo:sun_azimuth': 168.7 + 'eo:cloud_cover': 0.12 + 'eo:off_nadir': 1.4 + 'eo:platform': coolsat2 + 'eo:instrument': cool_sensor_v1 + 'eo:bands': [] + 'eo:sun_elevation': 33.4 + 'eo:gsd': 0.512 + collection: CS3 + links: + - rel: self + href: 'http://cool-sat.com/collections/CS3/items/20160503_132130_04' + - rel: root + href: 'http://cool-sat.com/collections' + - rel: parent + href: 'http://cool-sat.com/collections/CS3' + - rel: collection + href: 'http://cool-sat.com/collections/CS3' + assets: + analytic: + href: >- + http://cool-sat.com/static-catalog/CS3/20160503_132130_04/analytic.tif + title: 4-Band Analytic + thumbnail: + href: >- + http://cool-sat.com/static-catalog/CS3/20160503_132130_04/thumbnail.png + title: Thumbnail + itemId: + type: string + example: path/to/example.tif + description: 'Provider identifier, a unique ID, potentially a link to a file.' + itemType: + type: string + description: The GeoJSON type + enum: + - Feature + itemAssets: + type: object + additionalProperties: + type: object + required: + - href + properties: + href: + type: string + format: url + description: Link to the asset object + example: >- + http://cool-sat.com/catalog/collections/cs/items/CS3-20160503_132130_04/thumb.png + title: + type: string + description: Displayed title + example: Thumbnail + type: + type: string + description: Media type of the asset + example: image/png + itemProperties: + type: object + required: + - datetime + description: provides the core metatdata fields plus extensions + properties: + datetime: + $ref: '#/components/schemas/datetime' + additionalProperties: + description: >- + Any additional properties added in via Item specification or + extensions. + itemCollectionLinks: + type: array + description: >- + An array of links. Can be used for pagination, e.g. by providing a link + with the `next` relation type. + items: + $ref: '#/components/schemas/link' + example: + - rel: next + href: >- + http://api.cool-sat.com/stac/search?next=ANsXtp9mrqN0yrKWhf-y2PUpHRLQb1GT-mtxNcXou8TwkXhi1Jbk + queryFilter: + type: object + description: Allows users to query properties for specific values + properties: + query: + $ref: '#/components/schemas/query' + query: + type: object + description: Define which properties to query and the operatations to apply + additionalProperties: + $ref: '#/components/schemas/queryProp' + example: + 'eo:cloud_cover': + lt: 50 + providers: + eq: Planet + published: + gt: '2018-02-12T00:00:00Z' + lte: '2018-03-18T12:31:12Z' + 'pl:item_type': + startsWith: PSScene + product: + in: + - foo + - bar + - baz + queryProp: + description: Apply query operations to a specific property + anyOf: + - description: >- + if the object doesn't contain any of the operators, it is equivalent + to using the equals operator + - type: object + description: Match using an operator + properties: + eq: + description: >- + Find items with a property that is equal to the specified value. + For strings, a case-insensitive comparison must be performed. + neq: + description: >- + Find items that *don't* contain the specified value. For + strings, a case-insensitive comparison must be performed. + gt: + type: number + description: >- + Find items with a property value greater than the specified + value. + lt: + type: number + description: Find items with a property value less than the specified value. + gte: + type: number + description: >- + Find items with a property value greater than or equal the + specified value. + lte: + type: number + description: >- + Find items with a property value greater than or equal the + specified value. + startsWith: + type: string + description: >- + Find items with a property that begins with the specified + string. A case-insensitive comparison must be performed. + endsWith: + type: string + description: >- + Find items with a property that ends with the specified string. + A case-insensitive comparison must be performed. + contains: + type: string + description: >- + Find items with a property that contains with the specified + string. A case-insensitive comparison must be performed. + in: + type: array + items: + type: string + description: >- + Find items with a property that matches one of the specified + strings. A case-insensitive comparison must be performed. + sortFilter: + type: object + description: Sort the results + properties: + sort: + $ref: '#/components/schemas/sort' + sort: + type: array + description: | + An array of objects containing a property name and sort direction. + minItems: 1 + items: + type: object + required: + - field + properties: + field: + type: string + direction: + type: string + default: asc + enum: + - asc + - desc + example: + - field: 'eo:cloud_cover' + - field: providers + direction: desc + fieldsFilter: + type: object + description: Determines the shape of the features in the response + properties: + fields: + $ref: '#/components/schemas/fields' + fields: + description: | + The include and exclude members specify an array of + property names that are either included or excluded + from the result, respectively. If both include and + exclude are specified, include takes precedence. + Values should include the full JSON path of the property. + type: object + properties: + include: + type: array + items: + type: string + exclude: + type: array + items: + type: string + example: + include: + - id + - 'properties.eo:cloud_cover' + exclude: + - geometry + - properties.datetime + responses: + LandingPage: + description: |- + The landing page provides links to the API definition + (link relations `service-desc` and `service-doc`), + the Conformance declaration (path `/conformance`, + link relation `conformance`), and the Feature + Collections (path `/collections`, link relation + `data`). + content: + application/json: + schema: + $ref: '#/components/schemas/landingPage' + example: + title: Buildings in Bonn + description: >- + Access to data about buildings in the city of Bonn via a Web API + that conforms to the OGC API Features specification. + links: + - href: 'http://data.example.org/' + rel: self + type: application/json + title: this document + - href: 'http://data.example.org/api' + rel: service-desc + type: application/vnd.oai.openapi+json;version=3.0 + title: the API definition + - href: 'http://data.example.org/api.html' + rel: service-doc + type: text/html + title: the API documentation + - href: 'http://data.example.org/conformance' + rel: conformance + type: application/json + title: OGC API conformance classes implemented by this server + - href: 'http://data.example.org/collections' + rel: data + type: application/json + title: Information about the feature collections + text/html: + schema: + type: string + ConformanceDeclaration: + description: |- + The URIs of all conformance classes supported by the server. + To support "generic" clients that want to access multiple + OGC API Features implementations - and not "just" a specific + API / server, the server declares the conformance + classes it implements and conforms to. + content: + application/json: + schema: + $ref: '#/components/schemas/confClasses' + example: + conformsTo: + - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core' + - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30' + - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html' + - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson' + text/html: + schema: + type: string + Collections: + description: >- + The feature collections shared by this API. The dataset is organized as + one or more feature collections. This resource provides information + about and access to the collections. The response contains the list of + collections. For each collection, a link to the items in the collection + (path `/collections/{collectionId}/items`, link relation `items`) as + well as key information about the collection. This information includes: + * A local identifier for the collection that is unique for the dataset; + * A list of coordinate reference systems (CRS) in which geometries may + be returned by the server. The first CRS is the default coordinate + reference system (the default is always WGS 84 with axis order + longitude/latitude); * An optional title and description for the + collection; * An optional extent that can be used to provide an + indication of the spatial and temporal extent of the collection - + typically derived from the data; * An optional indicator about the type + of the items in the collection (the default value, if the indicator is + not provided, is 'feature'). + content: + application/json: + schema: + $ref: '#/components/schemas/collections' + text/html: + schema: + type: string + Collection: + description: >- + Information about the feature collection with id `collectionId`. The + response contains a link to the items in the collection (path + `/collections/{collectionId}/items`, link relation `items`) as well as + key information about the collection. This information includes: * A + local identifier for the collection that is unique for the dataset; * A + list of coordinate reference systems (CRS) in which geometries may be + returned by the server. The first CRS is the default coordinate + reference system (the default is always WGS 84 with axis order + longitude/latitude); * An optional title and description for the + collection; * An optional extent that can be used to provide an + indication of the spatial and temporal extent of the collection - + typically derived from the data; * An optional indicator about the type + of the items in the collection (the default value, if the indicator is + not provided, is 'feature'). + content: + application/json: + schema: + $ref: '#/components/schemas/collection' + text/html: + schema: + type: string + Features: + description: >- + The response is a document consisting of features in the collection. The + features included in the response are determined by the server based on + the query parameters of the request. To support access to larger + collections without overloading the client, the API supports paged + access with links to the next page, if more features are selected that + the page size. The `bbox` and `datetime` parameter can be used to select + only a subset of the features in the collection (the features that are + in the bounding box or time interval). The `bbox` parameter matches all + features in the collection that are not associated with a location, too. + The `datetime` parameter matches all features in the collection that are + not associated with a time stamp or interval, too. The `limit` parameter + may be used to control the subset of the selected features that should + be returned in the response, the page size. Each page may include + information about the number of selected and returned features + (`numberMatched` and `numberReturned`) as well as links to support + paging (link relation `next`). + content: + application/geo+json: + schema: + $ref: '#/components/schemas/featureCollectionGeoJSON' + text/html: + schema: + type: string + Feature: + description: |- + fetch the feature with id `featureId` in the feature collection + with id `collectionId` + content: + application/geo+json: + schema: + $ref: '#/components/schemas/item' + text/html: + schema: + type: string + InvalidParameter: + description: A query parameter has an invalid value. + content: + application/json: + schema: + $ref: '#/components/schemas/exception' + text/html: + schema: + type: string + NotFound: + description: The requested URI was not found. + ServerError: + description: A server error occurred. + content: + application/json: + schema: + $ref: '#/components/schemas/exception' + text/html: + schema: + type: string +servers: + - url: 'http://dev.cool-sat.com' + description: Development server + - url: 'http://www.cool-sat.com' + description: Production server diff --git a/packages/api/index.js b/packages/api/index.js index 59dc7c0..1afb63c 100644 --- a/packages/api/index.js +++ b/packages/api/index.js @@ -1,11 +1,6 @@ -/* eslint-disable new-cap, no-lonely-if */ - -'use strict' - const satlib = require('@sat-utils/api-lib') -module.exports.handler = async (event) => { - // determine endpoint +function determineEndpoint(event) { let endpoint = process.env.SATAPI_URL if (typeof endpoint === 'undefined') { if ('X-Forwarded-Host' in event.headers) { @@ -17,21 +12,10 @@ module.exports.handler = async (event) => { } } } + return endpoint +} - const buildResponse = async (statusCode, result) => { - const response = { - statusCode, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', // Required for CORS support to work - 'Access-Control-Allow-Credentials': true - }, - body: result - } - return response - } - - // get payload +function buildRequest(event) { const method = event.httpMethod let query = {} if (method === 'POST' && event.body) { @@ -39,13 +23,26 @@ module.exports.handler = async (event) => { } else if (method === 'GET' && event.queryStringParameters) { query = event.queryStringParameters } + return query +} - const result = await satlib.api.search(event.path, query, satlib.es, endpoint) - let returnResponse - if (result instanceof Error) { - returnResponse = buildResponse(404, result.message) - } else { - returnResponse = buildResponse(200, JSON.stringify(result)) +function buildResponse(statusCode, result) { + return { + statusCode, + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', // Required for CORS support to work + 'Access-Control-Allow-Credentials': true + }, + body: result } - return returnResponse +} + +module.exports.handler = async (event) => { + const endpoint = determineEndpoint(event) + const query = buildRequest(event) + const result = await satlib.api.API(event.path, query, satlib.es, endpoint) + return result instanceof Error ? + buildResponse(404, result.message) : + buildResponse(200, JSON.stringify(result)) } diff --git a/packages/api/package.json b/packages/api/package.json index 95e4886..64d7792 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/api", - "version": "0.3.0", + "version": "0.4.0-rc2", "description": "The api lambda function for sat-api", "main": "index.js", "repository": { @@ -26,14 +26,16 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.3.0" + "@sat-utils/api-lib": "^0.4.0-rc2" }, "devDependencies": { "ava": "^0.25.0", "aws-event-mocks": "0.0.0", "proxyquire": "^2.1.0", "sinon": "^7.1.1", - "webpack": "~4.5.0", - "webpack-cli": "~2.0.14" + "webpack": "~4.41", + "webpack-cli": "~3.3", + "zip-webpack-plugin": "^3.0.0", + "copy-webpack-plugin": "^5.1.1" } } diff --git a/packages/api/webpack.config.js b/packages/api/webpack.config.js index 0ca6a60..0c4e39b 100644 --- a/packages/api/webpack.config.js +++ b/packages/api/webpack.config.js @@ -1,12 +1,13 @@ +const path = require('path') +const ZipPlugin = require('zip-webpack-plugin') +const CopyPlugin = require('copy-webpack-plugin') -const path = require('path'); +let mode = 'development' +let devtool = 'inline-source-map' -let mode = 'development'; -let devtool = 'inline-source-map'; - -if(process.env.PRODUCTION) { - mode = 'production', - devtool = false +if (process.env.PRODUCTION) { + mode = 'production' + devtool = false } module.exports = { @@ -18,10 +19,22 @@ module.exports = { path: path.resolve(__dirname, 'dist') }, externals: [ - 'aws-sdk', - 'electron', - {'formidable': 'url'} + 'aws-sdk' ], devtool, - target: 'node' -}; \ No newline at end of file + optimization: { + usedExports: true + }, + target: 'node', + plugins: [ + new CopyPlugin([ + { + from: 'api.yaml', + to: 'api.yaml' + } + ]), + new ZipPlugin({ + filename: 'api.zip' + }) + ] +} diff --git a/packages/ingest/index.js b/packages/ingest/index.js index c836f4d..721c588 100644 --- a/packages/ingest/index.js +++ b/packages/ingest/index.js @@ -1,43 +1,11 @@ 'use strict' -const AWS = require('aws-sdk') const satlib = require('@sat-utils/api-lib') +const logger = console -// Runs on Fargate -const runIngestTask = async function (input, envvars) { - const ecs = new AWS.ECS() - const params = { - cluster: process.env.CLUSTER_ARN, - taskDefinition: process.env.TASK_ARN, - launchType: 'FARGATE', - networkConfiguration: { - awsvpcConfiguration: { - subnets: process.env.SUBNETS.split(' '), - assignPublicIp: 'ENABLED', - securityGroups: process.env.SECURITY_GROUPS.split(' ') - } - }, - overrides: { - containerOverrides: [ - { - command: [ - 'node', - 'packages/ingest/bin/ingest.js', - JSON.stringify(input) - ], - environment: envvars, - name: 'SatApi' - } - ], - executionRoleArn: process.env.ECS_ROLE_ARN, - taskRoleArn: process.env.ECS_ROLE_ARN - } - } - return ecs.runTask(params).promise() -} module.exports.handler = async function handler(event) { - console.log(`Ingest Event: ${JSON.stringify(event)}`) + logger.info(`Ingest Event: ${JSON.stringify(event)}`) try { if (event.Records && (event.Records[0].EventSource === 'aws:sns')) { // event is SNS message of updated file on s3 @@ -56,7 +24,7 @@ module.exports.handler = async function handler(event) { } } = s3Record const url = `https://${bucketName}.s3.amazonaws.com/${key}` - console.log(`Ingesting catalog file ${url}`) + logger.log(`Ingesting catalog file ${url}`) const recursive = false return satlib.ingest.ingest(url, satlib.es, recursive) }) @@ -71,15 +39,6 @@ module.exports.handler = async function handler(event) { const recurse = recursive === undefined ? true : recursive const collections = collectionsOnly === undefined ? false : collectionsOnly await satlib.ingest.ingest(url, satlib.es, recurse, collections) - } else if (event.fargate) { - // event is URL to a catalog node - start a Fargate instance to process - console.log(`Starting Fargate ingesttask ${JSON.stringify(event.fargate)}`) - const envvars = [ - { 'name': 'ES_HOST', 'value': process.env.ES_HOST }, - { 'name': 'ES_BATCH_SIZE', 'value': process.env.ES_BATCH_SIZE }, - { 'name': 'LOG_LEVEL', 'value': process.env.LOG_LEVEL || 'info' } - ] - await runIngestTask(event.fargate, envvars) } } catch (error) { console.log(error) diff --git a/packages/ingest/package.json b/packages/ingest/package.json index 8838bf9..5b2a0e8 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/ingest", - "version": "0.3.0", + "version": "0.4.0-rc2", "description": "ingest lambda function of sat-api", "main": "index.js", "bin": { @@ -29,7 +29,7 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.3.0" + "@sat-utils/api-lib": "^0.4.0-rc2" }, "devDependencies": { "ava": "^0.25.0", @@ -38,6 +38,7 @@ "proxyquire": "^2.1.0", "sinon": "^7.1.1", "webpack": "~4.5.0", - "webpack-cli": "~2.0.14" + "webpack-cli": "~2.0.14", + "zip-webpack-plugin": "^3.0.0" } } diff --git a/packages/ingest/webpack.config.js b/packages/ingest/webpack.config.js index 0ca6a60..b06fde7 100644 --- a/packages/ingest/webpack.config.js +++ b/packages/ingest/webpack.config.js @@ -1,12 +1,12 @@ +const path = require('path') +const ZipPlugin = require('zip-webpack-plugin') -const path = require('path'); +let mode = 'development' +let devtool = 'inline-source-map' -let mode = 'development'; -let devtool = 'inline-source-map'; - -if(process.env.PRODUCTION) { - mode = 'production', - devtool = false +if (process.env.PRODUCTION) { + mode = 'production' + devtool = false } module.exports = { @@ -18,10 +18,13 @@ module.exports = { path: path.resolve(__dirname, 'dist') }, externals: [ - 'aws-sdk', - 'electron', - {'formidable': 'url'} + 'aws-sdk' ], devtool, - target: 'node' -}; \ No newline at end of file + target: 'node', + plugins: [ + new ZipPlugin({ + filename: 'ingest.zip' + }) + ] +} diff --git a/serverless.yml b/serverless.yml new file mode 100644 index 0000000..7036cad --- /dev/null +++ b/serverless.yml @@ -0,0 +1,72 @@ +service: stac-api + +provider: + name: aws + runtime: nodejs12.x + stage: alpha + region: us-west-2 + environment: + LOG_LEVEL: DEBUG + ES_HOST: + Fn::GetAtt: [ElasticSearchInstance, DomainEndpoint] + iamRoleStatements: + - Effect: "Allow" + Resource: "arn:aws:es:#{AWS::Region}:#{AWS::AccountId}:domain/*" + Action: "es:*" + +package: + individually: true + +functions: + api: + handler: index.handler + package: + artifact: packages/api/dist/api.zip + environment: + STAC_ID: "stac-api" + STAC_TITLE: "STAC API" + STAC_DESCRIPTION: "STAC API" + STAC_VERSION: 0.8.1 + events: + - http: ANY / + - http: ANY {proxy+} + ingest: + handler: index.handler + timeout: 900 + package: + artifact: packages/ingest/dist/ingest.zip + events: + - sns: + topicName: ${self:service}-${self:provider.stage}-queue + displayName: Ingest STAC Item + # - sqs: + # arn: + # Fn::GetAtt: [ingestQueue, Arn] + #events: + # - sns: + # topicName: ${self:service}-${self:provider.stage}-ingest-queue + # displayName: STAC Item ingest topic + +resources: + Resources: +# ingestQueue: +# Type: AWS::SQS::Queue +# Properties: +# DelaySeconds: 1 +# QueueName: ${self:service}-${self:provider.stage}-ingest-queue + ElasticSearchInstance: + Type: AWS::Elasticsearch::Domain + Properties: + EBSOptions: + EBSEnabled: true + VolumeType: gp2 + VolumeSize: 35 + ElasticsearchClusterConfig: + InstanceType: t2.small.elasticsearch + InstanceCount: 3 + DedicatedMasterEnabled: false + ZoneAwarenessEnabled: false + ElasticsearchVersion: 6.8 + +plugins: + - serverless-pseudo-parameters \ No newline at end of file