From 8af64a591a61943e2fac30be50ee518e60cf2e1a Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 16:47:15 +0200 Subject: [PATCH 1/4] new endpoint observations/places_counts --- lib/controllers/v1/observations_controller.js | 65 +++++++++++++++++++ lib/inaturalist_api.js | 3 + lib/views/swagger_v1.yml.ejs | 38 +++++++++++ schema/fixtures.js | 1 + test/integration/v1/observations.js | 39 +++++++++++ 5 files changed, 146 insertions(+) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index dc15db72..e3ecca97 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,6 +363,11 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } + if (data.aggregations?.by_place) { + response.aggregations = { + by_place: data.aggregations.by_place + }; + } response.page = Number( req.elastic_query.page ); response.per_page = Number( req.elastic_query.per_page ); response.results = obs; @@ -406,11 +411,71 @@ ObservationsController.speciesCountsCacheWrapper = async req => ( "ObservationsController.speciesCounts" ) ); +ObservationsController.placesCountsCacheWrapper = async req => ( + ObservationsController.methodCacheWrapper( req, + ObservationsController.placesCounts, + "ObservationsController.placesCounts" ) +); + ObservationsController.speciesCounts = async req => { const leafCounts = await ObservationsController.leafCounts( req ); return TaxaController.speciesCountsResponse( req, leafCounts ); }; +ObservationsController.placesCounts = async (req, options = {}) => { + + const page = parseInt(req.query.page || 1, 10); + const perPage = parseInt(req.query.per_page || 30, 10); + const offset = (page - 1) * perPage; + const order = req.query.order === "asc" ? "asc" : "desc"; + + req.query.aggs = { + by_place: { + terms: { + field: "place_ids", + size: 1000, + order: { "_count": order } + } + } + }; + req.query.per_page = 0; + + const data = await ObservationsController.resultsForRequest(req, options); + const buckets = data.aggregations?.by_place?.buckets || []; + + const places = await Promise.all( + buckets.map(bucket => + Place.findByID(bucket.key, { + fields: ["id", "name", "display_name"] + }) + ) + ); + + const results = places + .map((place, i) => { + if (!place) return null; + return { + count: buckets[i].doc_count, + place: { + id: place.id, + name: place.name, + display_name: place.display_name + } + }; + }) + .filter(Boolean); + + const paginated = results.slice(offset, offset + perPage); + + return { + total_results: results.length, + page, + per_page: perPage, + results: paginated + }; +}; + + ObservationsController.taxa = async req => { if ( !req.query.user_id ) { throw new Error( 422 ); diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index b51fd2ad..24dfa6f8 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,6 +298,9 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); + dfault( "get", "/v1/observations/places_counts", ObservationsController.placesCountsCacheWrapper, { + setTTL: true + } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { setTTL: true } ); diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index a877ded9..e09cb312 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1043,6 +1043,29 @@ paths: description: Unexpected error schema: $ref: "#/definitions/Error" + /observations/places_counts: + get: + summary: Observation Place Counts + description: | + Given zero to many of the following parameters, returns the number of observations matching + the search criteria, grouped by place. Each result includes the place and the count of + associated observations. This endpoint works similarly to `/observations`, but instead of + returning individual observations, it returns aggregated results per place. A + maximum of 1000 results will be returned + parameters: + <%- include( "_observation_search_params_v1.yml.ejs", { type: "index" } ) %> + tags: + - Observations + responses: + 200: + description: | + Returns an object with metadata and an array of taxa + schema: + $ref: "#/definitions/PlacesCountsResponse" + default: + description: Unexpected error + schema: + $ref: "#/definitions/Error" /observations/popular_field_values: get: summary: Observation Popular Field Values @@ -4416,6 +4439,21 @@ definitions: type: integer taxon: $ref: "#/definitions/ShowTaxon" + PlacesCountsResponse: + allOf: + - $ref: "#/definitions/BaseResponse" + - required: + - results + properties: + results: + type: array + items: + type: object + properties: + count: + type: integer + place: + $ref: "#/definitions/CorePlace" PlacesResponse: allOf: - $ref: "#/definitions/BaseResponse" diff --git a/schema/fixtures.js b/schema/fixtures.js index 84c1b1eb..3232da07 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -99,6 +99,7 @@ "name": "United States", "slug": "united-states", "display_name_autocomplete": "United States", + "display_name": "United States", "location": "48.8907012939,-116.9820022583", "admin_level": 0, "bbox_area": 5500, diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 7e03ad8a..715a63cc 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1409,6 +1409,45 @@ describe( "Observations", ( ) => { } ); } ); + describe.only( "place_counts", ( ) => { + it( "returns JSON", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) + .expect( "Content-Type", /json/ ) + .expect( 200, done ); + } ); + + it( "sorts by count desc by default", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); + } ).expect( 200, done ); + } ); + + it( "can sort by count asc", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { + expect( res.body.results.length ).to.be.greaterThan( 1 ); + expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); + } ).expect( 200, done ); + } ); + + it( "supports pagination", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { + expect( res.body.page ).to.eq( 2 ); + expect( res.body.per_page ).to.eq( 1 ); + } ).expect( 200, done ); + } ); + + it( "returns results places counts with expected fields", function ( done ) { + request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + const result = res.body.results[0]; + expect( result ).to.have.property( "count" ); + expect( result.place ).to.have.property( "id" ); + expect( result.place ).to.have.property( "name" ); + expect( result.place ).to.have.property( "display_name" ); + } ).expect( 200, done ); + } ); + } ); + describe( "iconic_taxa_counts", ( ) => { it( "returns json", function ( done ) { request( this.app ).get( "/v1/observations/iconic_taxa_counts" ) From f292c04bdd3b1bd01b0b90cefe684785227e5bc1 Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 17:10:13 +0200 Subject: [PATCH 2/4] fix eslint --- lib/controllers/v1/observations_controller.js | 46 +++++++++---------- test/integration/v1/observations.js | 8 ++-- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index e3ecca97..97519c52 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -363,7 +363,7 @@ ObservationsController.prepareElasticDataForResponse = ( data, req ) => { } else if ( req.inat && req.inat.impliedBounds ) { response.total_bounds = req.inat.impliedBounds; } - if (data.aggregations?.by_place) { + if ( data.aggregations?.by_place ) { response.aggregations = { by_place: data.aggregations.by_place }; @@ -422,11 +422,10 @@ ObservationsController.speciesCounts = async req => { return TaxaController.speciesCountsResponse( req, leafCounts ); }; -ObservationsController.placesCounts = async (req, options = {}) => { - - const page = parseInt(req.query.page || 1, 10); - const perPage = parseInt(req.query.per_page || 30, 10); - const offset = (page - 1) * perPage; +ObservationsController.placesCounts = async ( req, options = {} ) => { + const page = parseInt( req.query.page || 1, 10 ); + const perPage = parseInt( req.query.per_page || 30, 10 ); + const offset = ( page - 1 ) * perPage; const order = req.query.order === "asc" ? "asc" : "desc"; req.query.aggs = { @@ -434,26 +433,24 @@ ObservationsController.placesCounts = async (req, options = {}) => { terms: { field: "place_ids", size: 1000, - order: { "_count": order } + order: { _count: order } } } }; req.query.per_page = 0; - const data = await ObservationsController.resultsForRequest(req, options); + const data = await ObservationsController.resultsForRequest( req, options ); const buckets = data.aggregations?.by_place?.buckets || []; const places = await Promise.all( - buckets.map(bucket => - Place.findByID(bucket.key, { - fields: ["id", "name", "display_name"] - }) - ) + buckets.map( bucket => Place.findByID( bucket.key, { + fields: ["id", "name", "display_name"] + } ) ) ); const results = places - .map((place, i) => { - if (!place) return null; + .map( ( place, i ) => { + if ( !place ) return null; return { count: buckets[i].doc_count, place: { @@ -462,20 +459,19 @@ ObservationsController.placesCounts = async (req, options = {}) => { display_name: place.display_name } }; - }) - .filter(Boolean); + } ) + .filter( Boolean ); - const paginated = results.slice(offset, offset + perPage); + const paginated = results.slice( offset, offset + perPage ); - return { - total_results: results.length, - page, - per_page: perPage, - results: paginated - }; + return { + total_results: results.length, + page, + per_page: perPage, + results: paginated + }; }; - ObservationsController.taxa = async req => { if ( !req.query.user_id ) { throw new Error( 422 ); diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 715a63cc..367bca83 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1415,28 +1415,28 @@ describe( "Observations", ( ) => { .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); - + it( "sorts by count desc by default", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); } ).expect( 200, done ); } ); - + it( "can sort by count asc", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); } ).expect( 200, done ); } ); - + it( "supports pagination", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { expect( res.body.page ).to.eq( 2 ); expect( res.body.per_page ).to.eq( 1 ); } ).expect( 200, done ); } ); - + it( "returns results places counts with expected fields", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { const result = res.body.results[0]; From d9b6fd2bef9f4e8501fc0c19483637c802d4c9ab Mon Sep 17 00:00:00 2001 From: Javi Date: Thu, 17 Apr 2025 17:10:33 +0200 Subject: [PATCH 3/4] fix test --- test/integration/v1/observations.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 367bca83..b5a31ae2 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1409,7 +1409,7 @@ describe( "Observations", ( ) => { } ); } ); - describe.only( "place_counts", ( ) => { + describe( "place_counts", ( ) => { it( "returns JSON", function ( done ) { request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) .expect( "Content-Type", /json/ ) From 086bc3f53a5aedb0684a337e64841260e5ab1007 Mon Sep 17 00:00:00 2001 From: Javi Date: Sun, 11 May 2025 21:51:51 +0200 Subject: [PATCH 4/4] change places to place --- lib/inaturalist_api.js | 2 +- lib/views/swagger_v1.yml.ejs | 2 +- test/integration/v1/observations.js | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js index 24dfa6f8..127880f0 100644 --- a/lib/inaturalist_api.js +++ b/lib/inaturalist_api.js @@ -298,7 +298,7 @@ InaturalistAPI.server = async ( ) => { dfault( "get", "/v1/observations/species_counts", ObservationsController.speciesCountsCacheWrapper, { setTTL: true } ); - dfault( "get", "/v1/observations/places_counts", ObservationsController.placesCountsCacheWrapper, { + dfault( "get", "/v1/observations/place_counts", ObservationsController.placesCountsCacheWrapper, { setTTL: true } ); dfault( "get", "/v1/observations/taxa_counts_by_month", ObservationsController.taxaCountsByMonth, { diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index e09cb312..d34a175f 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -1043,7 +1043,7 @@ paths: description: Unexpected error schema: $ref: "#/definitions/Error" - /observations/places_counts: + /observations/place_counts: get: summary: Observation Place Counts description: | diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index b5a31ae2..9bf83e86 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -1411,34 +1411,34 @@ describe( "Observations", ( ) => { describe( "place_counts", ( ) => { it( "returns JSON", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ) + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ) .expect( "Content-Type", /json/ ) .expect( 200, done ); } ); it( "sorts by count desc by default", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[0].count ).to.be.at.least( res.body.results[1].count ); } ).expect( 200, done ); } ); it( "can sort by count asc", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=asc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=asc&order_by=created_at" ).expect( res => { expect( res.body.results.length ).to.be.greaterThan( 1 ); expect( res.body.results[1].count ).to.be.at.least( res.body.results[0].count ); } ).expect( 200, done ); } ); it( "supports pagination", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at&per_page=1&page=2" ).expect( res => { expect( res.body.page ).to.eq( 2 ); expect( res.body.per_page ).to.eq( 1 ); } ).expect( 200, done ); } ); it( "returns results places counts with expected fields", function ( done ) { - request( this.app ).get( "/v1/observations/places_counts?order=desc&order_by=created_at" ).expect( res => { + request( this.app ).get( "/v1/observations/place_counts?order=desc&order_by=created_at" ).expect( res => { const result = res.body.results[0]; expect( result ).to.have.property( "count" ); expect( result.place ).to.have.property( "id" );