From 873b923d1695b13bfc4815fc416eedf826c26d86 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 18:22:58 -0700 Subject: [PATCH 1/7] rm unused es6->es7 migration script It was introduced as part of "initial commit" without any particular explanation: ee26c2f2ad88b4c377a392978c16a221244e9ff2 I'm **guessing** this existed to facilitate updating es6 to es7. In es6 _type was recommended to be "_doc", and in es7, _doc was the only valid value. For details, see: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html --- package.json | 1 - scripts/reset_type.js | 26 -------------------------- 2 files changed, 27 deletions(-) delete mode 100644 scripts/reset_type.js diff --git a/package.json b/package.json index 341e1693..e1a46127 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,6 @@ "integration": "./bin/integration", "create_index": "./bin/create_index", "drop_index": "node scripts/drop_index", - "reset_type": "node scripts/reset_type", "update_settings": "node scripts/update_settings" }, "repository": { diff --git a/scripts/reset_type.js b/scripts/reset_type.js deleted file mode 100644 index 91485237..00000000 --- a/scripts/reset_type.js +++ /dev/null @@ -1,26 +0,0 @@ -var config = require('pelias-config').generate().esclient; -var es = require('elasticsearch'); -var client = new es.Client(config); -var schema = require('../schema'); - -var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName; -var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args - -if( !_type ){ - console.error( 'you must provide the target es \'type\' as the first cli argument' ); - process.exit(1); -} - -var mapping = schema.mappings[_type]; -if( !mapping ){ - console.error( 'could not find a mapping in the schema file for', _index+'/'+_type ); - process.exit(1); -} - -client.indices.deleteMapping( { index: _index, type: _type }, function( err, res ){ - console.log( '[delete mapping]', '\t', _index+'/'+_type, err || '\t', res ); - client.indices.putMapping( { index: _index, type: _type, body:mapping }, function( err, res ){ - console.log( '[put mapping]', '\t\t', _index+'/'+_type, err || '\t', res ); - process.exit( !!err ); - }); -}); From 49795550de0e1a3a1a7cbf6a14e6b27e4035c7a5 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Mon, 12 Jun 2023 13:01:34 -0700 Subject: [PATCH 2/7] Mapping types are deprecated in es7, error in es8 Since we don't support es6, there's no need to specify it in any case, and removing it paves the way to es8. See: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html --- configValidation.js | 1 - integration/address_matching.js | 16 -------- integration/admin_abbreviations.js | 15 -------- integration/admin_matching.js | 4 -- integration/analyzer_peliasPhrase.js | 5 --- .../autocomplete_abbreviated_street_names.js | 5 --- ...ocomplete_directional_synonym_expansion.js | 10 ----- .../autocomplete_street_synonym_expansion.js | 10 ----- integration/bounding_box.js | 2 - integration/dynamic_templates.js | 10 ----- integration/multi_token_synonyms.js | 1 - integration/run.js | 4 +- .../source_layer_sourceid_filtering.js | 15 ++------ scripts/create_index.js | 1 - test/configValidation.js | 38 +------------------ 15 files changed, 6 insertions(+), 131 deletions(-) diff --git a/configValidation.js b/configValidation.js index f2f65452..41e824e2 100644 --- a/configValidation.js +++ b/configValidation.js @@ -6,7 +6,6 @@ const Joi = require('@hapi/joi'); const schema = Joi.object().required().keys({ schema: Joi.object().required().keys({ indexName: Joi.string().required(), - typeName: Joi.string().required() }), esclient: Joi.object().required() }).unknown(true); diff --git a/integration/address_matching.js b/integration/address_matching.js index 1ca81baa..18062f03 100644 --- a/integration/address_matching.js +++ b/integration/address_matching.js @@ -16,7 +16,6 @@ module.exports.tests.functional = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { address_parts: { name: 'Mapzen HQ', number: 30, @@ -29,7 +28,6 @@ module.exports.tests.functional = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '2', body: { address_parts: { name: 'Fake Venue', number: 300, @@ -42,7 +40,6 @@ module.exports.tests.functional = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '3', body: { address_parts: { name: 'Mock British Address', number: 3000, @@ -55,7 +52,6 @@ module.exports.tests.functional = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '4', body: { address_parts: { name: 'Mystery Location', number: 300, @@ -69,7 +65,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.number': 30 } } ]}}} @@ -84,7 +79,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match_phrase: { 'address_parts.street': 'west 26th street' } } ]}}} @@ -99,7 +93,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match_phrase: { 'address_parts.street': 'W 26th ST' } } ]}}} @@ -114,7 +107,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': '10010' } } ]}}} @@ -129,7 +121,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'e24dn' } } ]}}} @@ -144,7 +135,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': '100-10' } } ]}}} @@ -159,7 +149,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': '10 0 10' } } ]}}} @@ -174,7 +163,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'E2-4DN' } } ]}}} @@ -189,7 +177,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { match: { 'address_parts.zip': 'E2 4DN' } } ]}}} @@ -224,7 +211,6 @@ module.exports.tests.venue_vs_address = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'Union Square' }, phrase: { default: 'Union Square' } @@ -239,7 +225,6 @@ module.exports.tests.venue_vs_address = function(test, common){ let id = i + 100; // id offset suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: String(id), body: { name: { default: `${id} Union Square` }, @@ -263,7 +248,6 @@ module.exports.tests.venue_vs_address = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', size: TOTAL_ADDRESS_DOCS+1, body: { diff --git a/integration/admin_abbreviations.js b/integration/admin_abbreviations.js index a2b61061..be9b6da7 100644 --- a/integration/admin_abbreviations.js +++ b/integration/admin_abbreviations.js @@ -19,7 +19,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { parent: { @@ -33,7 +32,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '2', body: { parent: { @@ -47,7 +45,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -70,7 +67,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -93,7 +89,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -116,7 +111,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -147,7 +141,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { parent: { @@ -161,7 +154,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '2', body: { parent: { @@ -175,7 +167,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -198,7 +189,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -221,7 +211,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -244,7 +233,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { @@ -275,7 +263,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { parent: { @@ -289,7 +276,6 @@ module.exports.tests.synonyms = function (test, common) { suite.action(done => { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '2', body: { parent: { @@ -303,7 +289,6 @@ module.exports.tests.synonyms = function (test, common) { suite.assert(done => { suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { diff --git a/integration/admin_matching.js b/integration/admin_matching.js index 707030d4..3036adc2 100644 --- a/integration/admin_matching.js +++ b/integration/admin_matching.js @@ -16,7 +16,6 @@ module.exports.tests.functional = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { parent: { country: 'Test Country', @@ -46,7 +45,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'parent.country': 'Test Country' } } } }, function( err, res ){ t.equal( err, undefined ); @@ -59,7 +57,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'parent.country_a': 'TestCountry' } } } }, function( err, res ){ t.equal( err, undefined ); @@ -72,7 +69,6 @@ module.exports.tests.functional = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'parent.country_id': '100' } } } }, function( err, res ){ t.equal( err, undefined ); diff --git a/integration/analyzer_peliasPhrase.js b/integration/analyzer_peliasPhrase.js index 3232db51..4c984e50 100644 --- a/integration/analyzer_peliasPhrase.js +++ b/integration/analyzer_peliasPhrase.js @@ -161,7 +161,6 @@ module.exports.tests.slop_query = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'Lake Cayuga' }, phrase: { default: 'Lake Cayuga' } } }, done ); @@ -171,7 +170,6 @@ module.exports.tests.slop_query = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '2', body: { name: { default: 'Cayuga Lake' }, phrase: { default: 'Cayuga Lake' } } }, done ); @@ -181,7 +179,6 @@ module.exports.tests.slop_query = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '3', body: { name: { default: '7991 Lake Cayuga Dr' }, phrase: { default: '7991 Lake Cayuga Dr' } } }, done ); @@ -256,7 +253,6 @@ module.exports.tests.slop = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: '52 Görlitzer Straße' } } }, done); @@ -269,7 +265,6 @@ module.exports.tests.slop = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, searchType: 'dfs_query_then_fetch', body: { query: { match_phrase: { 'name.default': { diff --git a/integration/autocomplete_abbreviated_street_names.js b/integration/autocomplete_abbreviated_street_names.js index a2f5f8fb..2f84c623 100644 --- a/integration/autocomplete_abbreviated_street_names.js +++ b/integration/autocomplete_abbreviated_street_names.js @@ -21,7 +21,6 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'Grolmanstraße' } } }, done); @@ -31,7 +30,6 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -62,7 +60,6 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // suite.action( function( done ){ // suite.client.index({ // index: suite.props.index, -// type: config.schema.typeName, // id: '1', // body: { name: { default: 'Grolmanstr.' } } // }, done); @@ -75,7 +72,6 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // suite.assert( function( done ){ // suite.client.search({ // index: suite.props.index, -// type: config.schema.typeName, // body: { query: { match: { // 'name.default': { // 'analyzer': 'peliasQueryPartialToken', @@ -96,7 +92,6 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // suite.assert( function( done ){ // suite.client.search({ // index: suite.props.index, -// type: config.schema.typeName, // body: { query: { match: { // 'name.default': { // 'analyzer': 'peliasQuery', diff --git a/integration/autocomplete_directional_synonym_expansion.js b/integration/autocomplete_directional_synonym_expansion.js index 30f0a10d..37d63be1 100644 --- a/integration/autocomplete_directional_synonym_expansion.js +++ b/integration/autocomplete_directional_synonym_expansion.js @@ -21,7 +21,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'north' } } }, done); @@ -31,7 +30,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -49,7 +47,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -78,7 +75,6 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'n' } } }, done); @@ -88,7 +84,6 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -117,7 +112,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'n' } } }, done); @@ -127,7 +121,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -145,7 +138,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -174,7 +166,6 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'north' } } }, done); @@ -184,7 +175,6 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', diff --git a/integration/autocomplete_street_synonym_expansion.js b/integration/autocomplete_street_synonym_expansion.js index 55e1719c..f9245d42 100644 --- a/integration/autocomplete_street_synonym_expansion.js +++ b/integration/autocomplete_street_synonym_expansion.js @@ -22,7 +22,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'center' } } }, done); @@ -32,7 +31,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -50,7 +48,6 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -79,7 +76,6 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'ctr' } } }, done); @@ -89,7 +85,6 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -118,7 +113,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'ctr' } } }, done); @@ -128,7 +122,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -146,7 +139,6 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', @@ -175,7 +167,6 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'center' } } }, done); @@ -185,7 +176,6 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { match: { 'name.default': { 'analyzer': 'peliasQuery', diff --git a/integration/bounding_box.js b/integration/bounding_box.js index ecd23d0a..41abff05 100644 --- a/integration/bounding_box.js +++ b/integration/bounding_box.js @@ -15,7 +15,6 @@ module.exports.tests.index_and_retrieve = function(test, common){ suite.action( function( done ){ suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { bounding_box: '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}' @@ -28,7 +27,6 @@ module.exports.tests.index_and_retrieve = function(test, common){ suite.client.get( { index: suite.props.index, - type: config.schema.typeName, id: '1' }, function (err, res) { diff --git a/integration/dynamic_templates.js b/integration/dynamic_templates.js index 4ece07b7..41ac8875 100644 --- a/integration/dynamic_templates.js +++ b/integration/dynamic_templates.js @@ -32,13 +32,11 @@ function nameAssertion( analyzer, common ){ return function(t){ var suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; // index a document from a normal document layer suite.action( done => { suite.client.index({ index: suite.props.index, - type: _type, id: '1', body: { name: { default: 'foo', alt: 'bar' } } }, done ); @@ -50,7 +48,6 @@ function nameAssertion( analyzer, common ){ suite.client.indices.getMapping({ index: suite.props.index, - include_type_name: false }, (err, res) => { const properties = res[suite.props.index].mappings.properties; @@ -71,13 +68,11 @@ function phraseAssertion( analyzer, common ){ return function(t){ const suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; // index a document from a normal document layer suite.action( done => { suite.client.index({ index: suite.props.index, - type: _type, id: '1', body: { phrase: { default: 'foo', alt: 'bar' } } }, done ); @@ -89,7 +84,6 @@ function phraseAssertion( analyzer, common ){ suite.client.indices.getMapping({ index: suite.props.index, - include_type_name: false }, ( err, res ) => { const properties = res[suite.props.index].mappings.properties; @@ -110,13 +104,11 @@ function addendumAssertion( namespace, value, common ){ return function(t){ const suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; // index a document including the addendum suite.action( done => { suite.client.index({ index: suite.props.index, - type: _type, id: '1', body: { addendum: { [namespace]: value } }, }, done ); @@ -127,7 +119,6 @@ function addendumAssertion( namespace, value, common ){ suite.assert( done => { suite.client.indices.getMapping({ index: suite.props.index, - include_type_name: false, }, ( err, res ) => { const properties = res[suite.props.index].mappings.properties; @@ -156,7 +147,6 @@ function addendumAssertion( namespace, value, common ){ suite.assert( done => { suite.client.get({ index: suite.props.index, - type: _type, id: 1 }, ( err, res ) => { t.false( err ); diff --git a/integration/multi_token_synonyms.js b/integration/multi_token_synonyms.js index 721b01f4..41a7e4e9 100644 --- a/integration/multi_token_synonyms.js +++ b/integration/multi_token_synonyms.js @@ -19,7 +19,6 @@ module.exports.tests.functional = function (test, common) { suite.action(function (done) { suite.client.index({ index: suite.props.index, - type: config.schema.typeName, id: '1', body: { name: { default: 'set' }, phrase: { default: 'set' }, diff --git a/integration/run.js b/integration/run.js index 42f835ae..4c906792 100644 --- a/integration/run.js +++ b/integration/run.js @@ -12,9 +12,7 @@ const common = { }, create: { schema: schema, - create: { - include_type_name: false - } + create: { } }, summaryMap: (res) => { return res.hits.hits.map(h => { diff --git a/integration/source_layer_sourceid_filtering.js b/integration/source_layer_sourceid_filtering.js index 8fb17238..15377eca 100644 --- a/integration/source_layer_sourceid_filtering.js +++ b/integration/source_layer_sourceid_filtering.js @@ -15,28 +15,28 @@ module.exports.tests.source_filter = function(test, common){ // index some docs suite.action( function( done ){ suite.client.index({ - index: suite.props.index, type: config.schema.typeName, + index: suite.props.index, id: '1', body: { source: 'osm', layer: 'node', source_id: 'dataset/1' } }, done ); }); suite.action( function( done ){ suite.client.index({ - index: suite.props.index, type: config.schema.typeName, + index: suite.props.index, id: '2', body: { source: 'osm', layer: 'address', source_id: 'dataset/2' } }, done ); }); suite.action( function( done ){ suite.client.index({ - index: suite.props.index, type: config.schema.typeName, + index: suite.props.index, id: '3', body: { source: 'geonames', layer: 'address', source_id: 'dataset/1' } }, done ); }); suite.action( function( done ){ suite.client.index({ - index: suite.props.index, type: config.schema.typeName, + index: suite.props.index, id: '4', body: { source: 'foo bar baz' } }, done ); }); @@ -45,7 +45,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { source: 'osm' @@ -61,7 +60,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { layer: 'address' @@ -77,7 +75,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { source_id: 'dataset/1' @@ -93,7 +90,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { bool: { must: [ { term: { source: 'osm' } }, { term: { source_id: 'dataset/1' } } @@ -108,7 +104,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { source: 'OSM' @@ -124,7 +119,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { source: 'foo' @@ -140,7 +134,6 @@ module.exports.tests.source_filter = function(test, common){ suite.assert( function( done ){ suite.client.search({ index: suite.props.index, - type: config.schema.typeName, body: { query: { term: { source: 'foo bar baz' diff --git a/scripts/create_index.js b/scripts/create_index.js index 61e76de3..9072cd1d 100644 --- a/scripts/create_index.js +++ b/scripts/create_index.js @@ -30,7 +30,6 @@ const indexName = config.schema.indexName; const req = { index: indexName, body: schema, - include_type_name: false }; client.indices.create(req, (err, res) => { diff --git a/test/configValidation.js b/test/configValidation.js index d4a2a676..f5469b6a 100644 --- a/test/configValidation.js +++ b/test/configValidation.js @@ -19,7 +19,7 @@ module.exports.tests.interface = function(test, common) { test('config without schema.indexName should throw error', function(t) { var config = { - schema: { typeName: 'example_type' }, + schema: {}, esclient: {} }; @@ -30,25 +30,11 @@ module.exports.tests.interface = function(test, common) { }); - test('config without schema.typeName should throw error', function (t) { - var config = { - schema: { indexName: 'example_index' }, - esclient: {} - }; - - t.throws(function () { - configValidation.validate(config); - }, /"schema.typeName" is required/, 'schema.typeName should exist'); - t.end(); - - }); - test('config with non-string schema.indexName should throw error', function(t) { [null, 17, {}, [], false].forEach((value) => { var config = { schema: { indexName: value, - typeName: 'example_type' }, esclient: {} }; @@ -63,32 +49,11 @@ module.exports.tests.interface = function(test, common) { }); - test('config with non-string schema.typeName should throw error', function (t) { - [null, 17, {}, [], false].forEach((value) => { - var config = { - schema: { - indexName: 'example_index', - typeName: value - }, - esclient: {} - }; - - t.throws(function () { - configValidation.validate(config); - }, /"schema.typeName" must be a string/, 'schema.typeName should be a string'); - - }); - - t.end(); - - }); - test('config with non-object esclient should throw error', function(t) { [null, 17, [], 'string', true].forEach((value) => { var config = { schema: { indexName: 'example_index', - typeName: 'example_type' }, esclient: value }; @@ -107,7 +72,6 @@ module.exports.tests.interface = function(test, common) { var config = { schema: { indexName: 'example_index', - typeName: 'example_type' }, esclient: {} }; From 0ca7916bc763e2ddd309b09ba15eb470a7cc1d0e Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Mon, 12 Jun 2023 13:01:50 -0700 Subject: [PATCH 3/7] Old 'edgeNGram' spelling dropped in elastic8 The rename to edge_ngram originally occurred in 6.4. Since the minimum supported elasticsearch is 7.4.2, we can safely use the modern spelling unconditionally. --- settings.js | 2 +- test/fixtures/expected.json | 2 +- test/settings.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/settings.js b/settings.js index 40d44c43..f2dd633a 100644 --- a/settings.js +++ b/settings.js @@ -226,7 +226,7 @@ function generate(){ "only_on_same_position": "true" }, "peliasOneEdgeGramFilter": { - "type" : "edgeNGram", + "type" : "edge_ngram", "min_gram" : 1, "max_gram" : 24 }, diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json index be715c99..abc586b9 100644 --- a/test/fixtures/expected.json +++ b/test/fixtures/expected.json @@ -248,7 +248,7 @@ "only_on_same_position": "true" }, "peliasOneEdgeGramFilter": { - "type": "edgeNGram", + "type": "edge_ngram", "min_gram": 1, "max_gram": 24 }, diff --git a/test/settings.js b/test/settings.js index b939441f..d23b1c63 100644 --- a/test/settings.js +++ b/test/settings.js @@ -490,7 +490,7 @@ module.exports.tests.peliasOneEdgeGramFilter = function(test, common) { var s = settings(); t.equal(typeof s.analysis.filter.peliasOneEdgeGramFilter, 'object', 'there is a peliasIndexOneEdgeGram filter'); var filter = s.analysis.filter.peliasOneEdgeGramFilter; - t.equal(filter.type, 'edgeNGram'); + t.equal(filter.type, 'edge_ngram'); t.equal(filter.min_gram, 1); t.equal(filter.max_gram, 24); t.end(); From c22d1619dd8a391a4d8e91fdeefd7de0bbc7ad1f Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Mon, 12 Jun 2023 13:12:20 -0700 Subject: [PATCH 4/7] FIX: `text` does not support doc_values From: https://www.elastic.co/guide/en/elasticsearch/reference/current/doc-values.html > Doc values are supported on almost all field types, with the notable exception of text and annotated_text fields. The error produced on elastic8.8 was: > [mapper_parsing_exception] unknown parameter [doc_values] on mapper [ngram] of type [text] AFAIK this was never supported, but maybe setting it to "false" was just silently ignored in older versions of elasticsearch? --- mappings/partial/admin.json | 3 +- mappings/partial/countryAbbreviation.json | 3 +- test/fixtures/expected.json | 84 ++++++++--------------- 3 files changed, 30 insertions(+), 60 deletions(-) diff --git a/mappings/partial/admin.json b/mappings/partial/admin.json index 5aef0094..5107ef7b 100644 --- a/mappings/partial/admin.json +++ b/mappings/partial/admin.json @@ -8,8 +8,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } } diff --git a/mappings/partial/countryAbbreviation.json b/mappings/partial/countryAbbreviation.json index 67b605f1..d86cf7ba 100644 --- a/mappings/partial/countryAbbreviation.json +++ b/mappings/partial/countryAbbreviation.json @@ -8,8 +8,7 @@ "type": "text", "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram", "search_analyzer": "peliasQuery", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } } diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json index abc586b9..40d2cdc1 100644 --- a/test/fixtures/expected.json +++ b/test/fixtures/expected.json @@ -2415,8 +2415,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2430,8 +2429,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2453,8 +2451,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2468,8 +2465,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2491,8 +2487,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2506,8 +2501,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2529,8 +2523,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2544,8 +2537,7 @@ "type": "text", "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram", "search_analyzer": "peliasQuery", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2567,8 +2559,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2582,8 +2573,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2605,8 +2595,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2620,8 +2609,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2643,8 +2631,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2658,8 +2645,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2681,8 +2667,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2696,8 +2681,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2719,8 +2703,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2734,8 +2717,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2757,8 +2739,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2772,8 +2753,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2795,8 +2775,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2810,8 +2789,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2833,8 +2811,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2848,8 +2825,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2871,8 +2847,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2886,8 +2861,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2909,8 +2883,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, @@ -2924,8 +2897,7 @@ "type": "text", "analyzer": "peliasIndexOneEdgeGram", "search_analyzer": "peliasAdmin", - "similarity": "peliasDefaultSimilarity", - "doc_values": false + "similarity": "peliasDefaultSimilarity" } } }, From b1ff96055d7e54a18d382031d1e8bb5db04bbc4a Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 20:43:22 -0700 Subject: [PATCH 5/7] inline `plugin` call since we only support es7+ --- scripts/check_plugins.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/check_plugins.js b/scripts/check_plugins.js index 6ab738cf..175594a6 100644 --- a/scripts/check_plugins.js +++ b/scripts/check_plugins.js @@ -10,15 +10,6 @@ const required = ['analysis-icu']; // list of failures let failures = []; -// returns the appropriate plugin name for the configured Elasticsearch version -function elasticsearchPluginUtility() { - if (config.esclient.apiVersion === '2.4') { - return 'plugin'; - } else { - return 'elasticsearch-plugin'; - } -} - cli.header("checking elasticsearch plugins"); client.nodes.info(null, (err, res) => { @@ -76,7 +67,7 @@ client.nodes.info(null, (err, res) => { failures.forEach(failure => { console.error( `\nyou can install the missing packages on '${failure.node.name}' [${failure.node.ip}] with the following command(s):\n` ); failure.plugins.forEach(plugin => { - console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/${elasticsearchPluginUtility()} install ${plugin}`) ); + console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/elasticsearch-plugin install ${plugin}`) ); }); }); console.error( colors.white("\nnote:") + "some plugins may require you to restart elasticsearch.\n"); From 431f3e6beda1ba34bbff29acfa4de3d721c6fd13 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 14 Jun 2023 19:49:43 -0700 Subject: [PATCH 6/7] Fix "list_analyzers" and "output_mapping" I think they've been broken since b6e92d4bd1c0d7bea1a7f8d2817dc4b282ce1007, which changed schema to have a single mapping. Presumably this was done as part of deprecating `doc._type` --- README.md | 2 +- scripts/list_analyzers.js | 3 +-- scripts/output_mapping.js | 19 +------------------ 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 1e9ae3c4..3a5c65fb 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ node scripts/update_settings.js # update index settings #### output schema file -Use this script to pretty-print the whole schema file or a single mapping to stdout. +Use this script to pretty-print the schema's mappings to stdout. ```bash node scripts/output_mapping.js diff --git a/scripts/list_analyzers.js b/scripts/list_analyzers.js index ee386f56..f7e367c6 100644 --- a/scripts/list_analyzers.js +++ b/scripts/list_analyzers.js @@ -1,7 +1,6 @@ const _ = require('lodash') const colors = require('colors') const cli = require('./cli'); -const config = require('pelias-config').generate(); const schema = require('../schema'); const DEFAULT_ANALYZER = 'standard'; const NOT_APPLICABLE_ANALYZER = 'n/a'; @@ -54,7 +53,7 @@ const error = function(vals) { } // parse mapping -const mapping = schema.mappings[config.schema.typeName]; +const mapping = schema.mappings; const dynamic = mapping.dynamic_templates.map(t => _.first(_.map(t, v => v))); // process and single mapping property (recursively) diff --git a/scripts/output_mapping.js b/scripts/output_mapping.js index c05da280..3cc6b050 100644 --- a/scripts/output_mapping.js +++ b/scripts/output_mapping.js @@ -1,21 +1,4 @@ -var config = require('pelias-config').generate(); -var es = require('elasticsearch'); -var client = new es.Client(config.esclient); var schema = require('../schema'); -var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName; -var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args - -// print out mapping for just one type -if ( _type ) { - var mapping = schema.mappings[_type]; - if( !mapping ){ - console.error( 'could not find a mapping in the schema file for', _index+'/'+_type ); - process.exit(1); - } - console.log( JSON.stringify( mapping, null, 2 ) ); -//print out the entire schema mapping -} else { - console.log( JSON.stringify( schema, null, 2 ) ); -} +console.log( JSON.stringify( schema.mappings, null, 2 ) ); From 01d908cb9eb698e41b7829308dfe303b0089de92 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Mon, 10 Jul 2023 15:06:22 -0700 Subject: [PATCH 7/7] update to es8 compatible pelias-config --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e1a46127..643e6834 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "elasticsearch": "^16.0.0", "glob": "^7.1.6", "lodash": "^4.17.15", - "pelias-config": "^4.5.0", + "pelias-config": "^6.0.0", "pelias-logger": "^1.3.0", "semver": "^7.1.1" },