From 9f6ce516cf3a05a09be2da83e9b8b6876048bf32 Mon Sep 17 00:00:00 2001 From: Kevin Chen Date: Tue, 21 Oct 2025 23:36:08 -0400 Subject: [PATCH 1/2] feat: impl obsv search for multiple term_id and term_value_id combos --- lib/controllers/v1/observations_controller.js | 10 +++ lib/models/observation_query_builder.js | 63 +++++++++++++----- .../_observation_search_params_v1.yml.ejs | 1 + lib/views/swagger_v1.yml.ejs | 12 ++++ openapi/schema/request/observations_search.js | 10 +++ schema/fixtures.js | 66 +++++++++++++++++++ test/integration/v1/observations.js | 31 +++++++++ 7 files changed, 176 insertions(+), 17 deletions(-) diff --git a/lib/controllers/v1/observations_controller.js b/lib/controllers/v1/observations_controller.js index 77069926..751a8417 100644 --- a/lib/controllers/v1/observations_controller.js +++ b/lib/controllers/v1/observations_controller.js @@ -111,6 +111,16 @@ ObservationsController.searchCacheWrapper = async req => ( ); ObservationsController.search = async ( req, options = { } ) => { + if ( req.query.read_term_id_and_value_id_as_pairs === "true" + && req.query.term_id && req.query.term_value_id + && util.paramArray( req.query.term_id ).length + !== util.paramArray( req.query.term_value_id ).length + ) { + const e = new Error( ); + e.status = 400; + e.message = "term_id and term_value_id must be the same length when read_term_id_and_value_id_as_pairs = true."; + throw e; + } if ( req.query.return_bounds === "true" ) { // If we've been asked to return the bounds but also to return obs in a // place, assume that the bounds are the bounding box of the place and not diff --git a/lib/models/observation_query_builder.js b/lib/models/observation_query_builder.js index eda9a8b5..c3b0df9a 100644 --- a/lib/models/observation_query_builder.js +++ b/lib/models/observation_query_builder.js @@ -943,27 +943,56 @@ ObservationQueryBuilder.reqToElasticQueryComponents = async req => { } if ( params.term_id ) { - const initialFilters = []; - initialFilters.push( - esClient.termFilter( "annotations.controlled_attribute_id.keyword", params.term_id ) - ); - initialFilters.push( { range: { "annotations.vote_score_short": { gte: 0 } } } ); - const nestedQuery = { - nested: { - path: "annotations", - query: { - bool: { - filter: initialFilters + if ( params.read_term_id_and_value_id_as_pairs === "true" && params.term_value_id + && util.paramArray( params.term_id ).length + === util.paramArray( params.term_value_id ).length ) { + const termValueIdArray = util.paramArray( params.term_value_id ); + const termIdTermValuePairs = _.map( + util.paramArray( params.term_id ), ( termId, index ) => ( { + termId, + termValueId: termValueIdArray[index] + } ) + ); + _.each( termIdTermValuePairs, pair => { + const termIdFilter = esClient.termFilter( "annotations.controlled_attribute_id.keyword", pair.termId ); + const voteScoreZeroOrPosFilter = { range: { "annotations.vote_score_short": { gte: 0 } } }; + const termValueIdFilter = esClient.termFilter( "annotations.controlled_value_id.keyword", pair.termValueId ); + const nestedQuery = { + nested: { + path: "annotations", + query: { + bool: { + filter: [termIdFilter, voteScoreZeroOrPosFilter, termValueIdFilter] + } + } + } + }; + // each pair is its own searchFilter to ensure obsv with ALL pairs would pass. + searchFilters.push( nestedQuery ); + } ); + } else { + const initialFilters = []; + initialFilters.push( + esClient.termFilter( "annotations.controlled_attribute_id.keyword", params.term_id ) + ); + initialFilters.push( { range: { "annotations.vote_score_short": { gte: 0 } } } ); + const nestedQuery = { + nested: { + path: "annotations", + query: { + bool: { + filter: initialFilters + } } } + }; + if ( params.term_value_id ) { + nestedQuery.nested.query.bool.filter.push( + esClient.termFilter( "annotations.controlled_value_id.keyword", params.term_value_id ) + ); } - }; - if ( params.term_value_id ) { - nestedQuery.nested.query.bool.filter.push( - esClient.termFilter( "annotations.controlled_value_id.keyword", params.term_value_id ) - ); + searchFilters.push( nestedQuery ); } - searchFilters.push( nestedQuery ); if ( params.without_term_value_id ) { inverseFilters.push( ObservationQueryBuilder.annotationTermValueNestedFilter( params.term_id, params.without_term_value_id diff --git a/lib/views/_observation_search_params_v1.yml.ejs b/lib/views/_observation_search_params_v1.yml.ejs index 9bbcf445..9531e16b 100644 --- a/lib/views/_observation_search_params_v1.yml.ejs +++ b/lib/views/_observation_search_params_v1.yml.ejs @@ -57,6 +57,7 @@ - $ref: "#/parameters/without_term_id" - $ref: "#/parameters/without_term_value_id" - $ref: "#/parameters/term_id_or_unknown" + - $ref: "#/parameters/read_term_id_and_value_id_as_pairs" - $ref: "#/parameters/annotation_user_id" # other - $ref: "#/parameters/acc_above" diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index e233caac..2c941dbc 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -2233,6 +2233,18 @@ parameters: in: query description: | Observations that have been favorited by at least one user + read_term_id_and_value_id_as_pairs: + name: read_term_id_and_value_id_as_pairs + type: boolean + in: query + description: | + Determines how to interpret the term_id and term_value_id parameters when both are provided and are of same length. + It does not affect how without_term_value_id will be interpreted. + If false, observations will satisfy having at least one of term_value_id's for any of the provided term_id's. + If true, term_id and term_value_id are grouped and filtered for in pairs. + For example, term_id=1,1,3 and term_value_id=2,5,6 is interpreted as the following {term_id, term_value_id} pairs: + {1, 2}, {1, 5}, and {1, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score. + If this parameter is true, but term_id and term_value_id are not the same length, error code 400 is returned. sounds: name: sounds type: boolean diff --git a/openapi/schema/request/observations_search.js b/openapi/schema/request/observations_search.js index 6a4f0492..e1bee4fb 100644 --- a/openapi/schema/request/observations_search.js +++ b/openapi/schema/request/observations_search.js @@ -131,6 +131,16 @@ module.exports = Joi.object( ).keys( { term_value_id: Joi.array( ).items( Joi.number( ).integer( ) ), without_term_value_id: Joi.array( ).items( Joi.number( ).integer( ) ), term_id_or_unknown: Joi.array( ).items( Joi.number( ).integer( ) ), + read_term_id_and_value_id_as_pairs: Joi.boolean( ) + .description( + "Determines how to interpret the term_id and term_value_id parameters when both are provided and are of same length. " + + "It does not affect how without_term_value_id will be interpreted.\n\n" + + "If false, observations will satisfy having at least one of term_value_id's for any of the provided term_id's.\n" + + "If true, term_id and term_value_id are grouped and filtered for in pairs. " + + "For example, term_id=1,1,3 and term_value_id=2,5,6 is interpreted as the following {term_id, term_value_id} pairs: " + + "{1, 2}, {1, 5}, and {1, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score.\n" + + "If this parameter is true, but term_id and term_value_id are not the same length, error code 400 is returned. " + ), annotation_user_id: Joi.array( ).items( Joi.string( ) ), acc_above: Joi.number( ).integer( ), acc_below: Joi.number( ).integer( ), diff --git a/schema/fixtures.js b/schema/fixtures.js index d75370e9..f7e38660 100644 --- a/schema/fixtures.js +++ b/schema/fixtures.js @@ -1222,6 +1222,72 @@ } ], "sounds_count": 1 + }, + { + "id": 2025102201, + "user": { "id": 2025102299 }, + "annotations": [ + { + "controlled_attribute_id": 10, + "controlled_value_id": 20, + "concatenated_attr_val": "10|20", + "vote_score_short": 0 + }, + { + "controlled_attribute_id": 10, + "controlled_value_id": 21, + "concatenated_attr_val": "10|21", + "vote_score_short": 0 + }, + { + "controlled_attribute_id": 12, + "controlled_value_id": 22, + "concatenated_attr_val": "12|22", + "vote_score_short": 0 + } + ] + }, + { + "id": 2025102202, + "user": { "id": 2025102299 }, + "annotations": [ + { + "controlled_attribute_id": 10, + "controlled_value_id": 20, + "concatenated_attr_val": "10|20", + "vote_score_short": 0 + }, + { + "controlled_attribute_id": 12, + "controlled_value_id": 22, + "concatenated_attr_val": "12|22", + "vote_score_short": 0 + }, + { + "controlled_attribute_id": 13, + "controlled_value_id": 23, + "concatenated_attr_val": "13|23", + "vote_score_short": 0 + } + ] + }, + { + "id": 2025102203, + "user": { "id": 2025102299 }, + "annotations": [ + { + "controlled_attribute_id": 10, + "controlled_value_id": 20, + "concatenated_attr_val": "10|20", + "vote_score_short": 0 + }, + { + "controlled_attribute_id": 10, + "controlled_value_id": 21, + "concatenated_attr_val": "10|21", + "vote_score_short": 0 + } + ] } ] }, diff --git a/test/integration/v1/observations.js b/test/integration/v1/observations.js index 7e03ad8a..85413cfa 100644 --- a/test/integration/v1/observations.js +++ b/test/integration/v1/observations.js @@ -699,6 +699,28 @@ describe( "Observations", ( ) => { } ).expect( 200, done ); } ); + it( "filters by term_id and term_value_id in pairs when requested", function ( done ) { + request( this.app ).get( "/v1/observations?term_id=10,10,12&term_value_id=20,21,22&read_term_id_and_value_id_as_pairs=true" ) + .expect( res => { + expect( res.body.results.length ).to.eq( 1 ); + expect( res.body.results.map( r => r.id ) ).to.contain( 2025102201 ); + } ).expect( 200, done ); + } ); + + it( "filters by term_id and term_value_id in pairs when requested -- multiple results", function ( done ) { + request( this.app ).get( "/v1/observations?term_id=10,12&term_value_id=20,22&read_term_id_and_value_id_as_pairs=true" ) + .expect( res => { + expect( res.body.results.length ).to.eq( 2 ); + expect( res.body.results.map( r => r.id ) ).to.contain( 2025102201 ); + expect( res.body.results.map( r => r.id ) ).to.contain( 2025102202 ); + } ).expect( 200, done ); + } ); + + it( "throws 400 when filtering by term_id and term_value_id are unequal length when requesting to read them as pairs", function ( done ) { + request( this.app ).get( "/v1/observations?term_id=10,10,12&term_value_id=20,21&read_term_id_and_value_id_as_pairs=true" ) + .expect( 400, done ); + } ); + it( "filters by without_term_id", function ( done ) { request( this.app ).get( "/v1/observations?without_term_id=1&id=6,7,9" ) .expect( res => { @@ -733,6 +755,15 @@ describe( "Observations", ( ) => { } ).expect( 200, done ); } ); + it( "filters by without_term_value_id unaffected by read_term_id_and_value_id_as_pairs", function ( done ) { + request( this.app ).get( "/v1/observations?term_id=1&without_term_value_id=1&read_term_id_and_value_id_as_pairs=true" ) + .expect( res => { + expect( res.body.results.map( r => r.id ) ).to.contain( 8 ); + expect( res.body.results.map( r => r.id ) ).not.to.contain( 7 ); + expect( res.body.results.map( r => r.id ) ).not.to.contain( 6 ); + } ).expect( 200, done ); + } ); + it( "can return only ids", function ( done ) { request( this.app ).get( "/v1/observations?id=2&only_id=true&per_page=1" ) .expect( res => { From d18730764336486faf7a57b89a0684cd62827d09 Mon Sep 17 00:00:00 2001 From: Kevin Chen Date: Sat, 25 Oct 2025 12:20:08 -0400 Subject: [PATCH 2/2] fix: typo in documentation for read_term_id_and_value_id_as_pairs --- lib/views/swagger_v1.yml.ejs | 2 +- openapi/schema/request/observations_search.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/views/swagger_v1.yml.ejs b/lib/views/swagger_v1.yml.ejs index 2c941dbc..67316970 100644 --- a/lib/views/swagger_v1.yml.ejs +++ b/lib/views/swagger_v1.yml.ejs @@ -2243,7 +2243,7 @@ parameters: If false, observations will satisfy having at least one of term_value_id's for any of the provided term_id's. If true, term_id and term_value_id are grouped and filtered for in pairs. For example, term_id=1,1,3 and term_value_id=2,5,6 is interpreted as the following {term_id, term_value_id} pairs: - {1, 2}, {1, 5}, and {1, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score. + {1, 2}, {1, 5}, and {3, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score. If this parameter is true, but term_id and term_value_id are not the same length, error code 400 is returned. sounds: name: sounds diff --git a/openapi/schema/request/observations_search.js b/openapi/schema/request/observations_search.js index e1bee4fb..261e8ea9 100644 --- a/openapi/schema/request/observations_search.js +++ b/openapi/schema/request/observations_search.js @@ -138,7 +138,7 @@ module.exports = Joi.object( ).keys( { + "If false, observations will satisfy having at least one of term_value_id's for any of the provided term_id's.\n" + "If true, term_id and term_value_id are grouped and filtered for in pairs. " + "For example, term_id=1,1,3 and term_value_id=2,5,6 is interpreted as the following {term_id, term_value_id} pairs: " - + "{1, 2}, {1, 5}, and {1, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score.\n" + + "{1, 2}, {1, 5}, and {3, 6}. Observations will satisfy having an entry for every pair with a 0 or positive vote score.\n" + "If this parameter is true, but term_id and term_value_id are not the same length, error code 400 is returned. " ), annotation_user_id: Joi.array( ).items( Joi.string( ) ),