diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 24d2c33e..00000000 --- a/.jshintrc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "node": true, - "curly": true, - "eqeqeq": true, - "esversion": 6, - "freeze": true, - "immed": true, - "indent": 2, - "latedef": false, - "newcap": true, - "noarg": true, - "noempty": true, - "nonbsp": true, - "nonew": true, - "plusplus": false, - "undef": true, - "unused": false, - "maxparams": 4, - "maxdepth": 4 -} diff --git a/configValidation.js b/configValidation.js index f2f65452..b765c261 100644 --- a/configValidation.js +++ b/configValidation.js @@ -1,4 +1,4 @@ -const Joi = require('@hapi/joi'); +const Joi = require('@hapi/joi') // Schema Configuration // schema.indexName: populated by defaults if not overridden @@ -9,13 +9,13 @@ const schema = Joi.object().required().keys({ typeName: Joi.string().required() }), esclient: Joi.object().required() -}).unknown(true); +}).unknown(true) module.exports = { - validate: function validate(config) { - const validated = schema.validate(config); + validate: function validate (config) { + const validated = schema.validate(config) if (validated.error) { - throw new Error(validated.error.details[0].message); + throw new Error(validated.error.details[0].message) } } -}; +} diff --git a/integration/_hits_total_helper.js b/integration/_hits_total_helper.js index 5347eac7..a032216c 100644 --- a/integration/_hits_total_helper.js +++ b/integration/_hits_total_helper.js @@ -1,4 +1,4 @@ -const _ = require('lodash'); +const _ = require('lodash') /** * The way hits are defined in ES7 changed @@ -9,12 +9,12 @@ const _ = require('lodash'); * { value: 1, relation: 'eq' } */ - function helper(hits){ - let totalHits = 0; - if (_.has(hits, 'total')) { - totalHits = _.isPlainObject(hits.total) ? hits.total.value : hits.total; - } - return totalHits; - } +function helper (hits) { + let totalHits = 0 + if (_.has(hits, 'total')) { + totalHits = _.isPlainObject(hits.total) ? hits.total.value : hits.total + } + return totalHits +} - module.exports = helper; +module.exports = helper diff --git a/integration/address_matching.js b/integration/address_matching.js index 1ca81baa..4c3770c7 100644 --- a/integration/address_matching.js +++ b/integration/address_matching.js @@ -1,212 +1,300 @@ // validate analyzer is behaving as expected -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); -const getTotalHits = require('./_hits_total_helper'); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() +const getTotalHits = require('./_hits_total_helper') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.functional = function(test, common){ - test( 'functional', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.functional = function (test, common) { + test('functional', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index some docs - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '1', body: { address_parts: { - name: 'Mapzen HQ', - number: 30, - street: 'West 26th Street', - zip: 10010 - }} - }, done ); - }); + id: '1', + body: { + address_parts: { + name: 'Mapzen HQ', + number: 30, + street: 'West 26th Street', + zip: 10010 + } + } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '2', body: { address_parts: { - name: 'Fake Venue', - number: 300, - street: 'west 26th street', - zip: '100 10' - }} - }, done ); - }); + id: '2', + body: { + address_parts: { + name: 'Fake Venue', + number: 300, + street: 'west 26th street', + zip: '100 10' + } + } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '3', body: { address_parts: { - name: 'Mock British Address', - number: 3000, - street: 'Buckinghamshire Street', - zip: 'E2 4DN' - }} - }, done ); - }); + id: '3', + body: { + address_parts: { + name: 'Mock British Address', + number: 3000, + street: 'Buckinghamshire Street', + zip: 'E2 4DN' + } + } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '4', body: { address_parts: { - name: 'Mystery Location', - number: 300, - street: 'east 26th street', - zip: '100 10' - }} - }, done ); - }); + id: '4', + body: { + address_parts: { + name: 'Mystery Location', + number: 300, + street: 'east 26th street', + zip: '100 10' + } + } + }, done) + }) // search by street number - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.number': 30 } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match street number' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match: { + 'address_parts.number': 30 + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'match street number') + done() + }) + }) // search by street name - case insensitive - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match_phrase: { 'address_parts.street': 'west 26th street' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 2, 'match street name' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { match_phrase: { 'address_parts.street': 'west 26th street' } } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 2, 'match street name') + done() + }) + }) // search by street name - using abbreviations - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match_phrase: { 'address_parts.street': 'W 26th ST' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 2, 'match street name - abbr' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match_phrase: { + 'address_parts.street': 'W 26th ST' + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 2, 'match street name - abbr') + done() + }) + }) // search by zip - numeric zip - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': '10010' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match: { + 'address_parts.zip': '10010' + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 3, 'match zip - numeric') + done() + }) + }) // search by zip - string zip - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': 'e24dn' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match: { + 'address_parts.zip': 'e24dn' + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'match zip - string') + done() + }) + }) // search by zip - numeric zip - with punctuation - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': '100-10' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric - punct' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match: { + 'address_parts.zip': '100-10' + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 3, 'match zip - numeric - punct') + done() + }) + }) // search by zip - numeric zip - with whitespace - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': '10 0 10' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 3, 'match zip - numeric - whitespace' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { + match: { + 'address_parts.zip': '10 0 10' + } + } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 3, 'match zip - numeric - whitespace') + done() + }) + }) // search by zip - string zip - with punctuation - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': 'E2-4DN' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string - punct' ); - done(); - }); - }); + body: { + query: { + bool: { + must: [ + { match: { 'address_parts.zip': 'E2-4DN' } } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'match zip - string - punct') + done() + }) + }) // search by zip - string zip - with whitespace - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, - body: { query: { bool: { must: [ - { match: { 'address_parts.zip': 'E2 4DN' } } - ]}}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'match zip - string - whitespace' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + body: { + query: { + bool: { + must: [ + { match: { 'address_parts.zip': 'E2 4DN' } } + ] + } + } + } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'match zip - string - whitespace') + done() + }) + }) + suite.run(t.end) + }) +} -module.exports.tests.venue_vs_address = function(test, common){ - test( 'venue_vs_address', function(t){ +module.exports.tests.venue_vs_address = function (test, common) { + test('venue_vs_address', function (t) { // This test shows that partial matching addresses score higher than exact matching // venues with the same name. @@ -217,26 +305,27 @@ module.exports.tests.venue_vs_address = function(test, common){ // Unfortunately there seems to be no easy way of fixing this, it's an artifact of us // storing the street names in the name.default field. - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a venue - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '1', body: { + id: '1', + body: { name: { default: 'Union Square' }, phrase: { default: 'Union Square' } } - }, done ); - }); + }, done) + }) // index multiple streets, having many in the index is important // because it influences the 'norms' and TF/IDF scoring - const testFactory = function(i){ - return function( done ){ - let id = i + 100; // id offset + const testFactory = function (i) { + return function (done) { + let id = i + 100 // id offset suite.client.index({ index: suite.props.index, type: config.schema.typeName, @@ -249,23 +338,23 @@ module.exports.tests.venue_vs_address = function(test, common){ street: 'Union Square' } } - }, done); - }; - }; + }, done) + } + } - const TOTAL_ADDRESS_DOCS=9; - for( var i=0; i= hits[1]._score ); - t.true( hits[1]._score >= hits[2]._score ); - done(); - }); - }); + t.true(hits[0]._score >= hits[1]._score) + t.true(hits[1]._score >= hits[2]._score) + done() + }) + }) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} // test the minimum amount of slop required to retrieve address documents -module.exports.tests.slop = function(test, common){ - test( 'slop', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.slop = function (test, common) { + test('slop', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: '52 Görlitzer Straße' } } - }, done); - }); + }, done) + }) // search using 'peliasPhrase' // in this case we require a slop of 3 to return the same // record with the street number and street name reversed. // (as is common in European countries, such as Germany). - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -268,59 +261,57 @@ module.exports.tests.slop = function(test, common){ 'name.default': { 'analyzer': 'peliasPhrase', 'query': 'Görlitzer Straße 52', - 'slop': 3, + 'slop': 3 } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // @see: https://github.com/pelias/api/issues/600 -module.exports.tests.unicode = function(test, common){ - test( 'normalization', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasPhrase' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - var latin_large_letter_e_with_acute = String.fromCodePoint(0x00C9); - var latin_small_letter_e_with_acute = String.fromCodePoint(0x00E9); - var combining_acute_accent = String.fromCodePoint(0x0301); - var latin_large_letter_e = String.fromCodePoint(0x0045); - var latin_small_letter_e = String.fromCodePoint(0x0065); +module.exports.tests.unicode = function (test, common) { + test('normalization', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasPhrase') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + const latinLargeLetterEWithAcute = String.fromCodePoint(0x00C9) + const latinSmallLetterEWithAcute = String.fromCodePoint(0x00E9) + const combiningAcuteAccent = String.fromCodePoint(0x0301) + const latinLargeLetterE = String.fromCodePoint(0x0045) + const latinSmallLetterE = String.fromCodePoint(0x0065) // Chambéry (both forms appear the same) - var composed = "Chamb" + latin_small_letter_e_with_acute + "ry"; - var decomposed = "Chamb" + combining_acute_accent + latin_small_letter_e + "ry" + let composed = 'Chamb' + latinSmallLetterEWithAcute + 'ry' + let decomposed = 'Chamb' + combiningAcuteAccent + latinSmallLetterE + 'ry' - assertAnalysis( 'composed', composed, ['chambery'] ); - assertAnalysis( 'decomposed', decomposed, ['chambery'] ); + assertAnalysis('composed', composed, ['chambery']) + assertAnalysis('decomposed', decomposed, ['chambery']) // Één (both forms appear the same) - var composed = latin_large_letter_e_with_acute + latin_small_letter_e_with_acute + "n"; - var decomposed = combining_acute_accent + latin_large_letter_e + combining_acute_accent + latin_small_letter_e + "n" + composed = latinLargeLetterEWithAcute + latinSmallLetterEWithAcute + 'n' + decomposed = combiningAcuteAccent + latinLargeLetterE + combiningAcuteAccent + latinSmallLetterE + 'n' - assertAnalysis( 'composed', composed, ['een'] ); - assertAnalysis( 'decomposed', decomposed, ['een'] ); + assertAnalysis('composed', composed, ['een']) + assertAnalysis('decomposed', decomposed, ['een']) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('peliasPhrase: ' + name, testFunction); + function test (name, testFunction) { + return tape('peliasPhrase: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/analyzer_peliasQuery.js b/integration/analyzer_peliasQuery.js index 7fe0af38..f9e28b28 100644 --- a/integration/analyzer_peliasQuery.js +++ b/integration/analyzer_peliasQuery.js @@ -1,120 +1,113 @@ // validate analyzer is behaving as expected - -var tape = require('tape'), - elastictest = require('elastictest'), - punctuation = require('../punctuation'); - -module.exports.tests = {}; - -module.exports.tests.analyze = function(test, common){ - test( 'analyze', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasQuery' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis('asciifolding', 'é', ['e']); - assertAnalysis('asciifolding', 'ß', ['ss']); - assertAnalysis('asciifolding', 'æ', ['ae']); - assertAnalysis('asciifolding', 'ł', ['l']); - assertAnalysis('asciifolding', 'ɰ', ['m']); - assertAnalysis('lowercase', 'F', ['f']); - assertAnalysis('trim', ' f ', ['f']); - assertAnalysis('remove_ordinals', '26t', ['26']); - assertAnalysis('remove_ordinals', '26th', ['26']); - assertAnalysis('removeAllZeroNumericPrefix', '00001', ['1']); - assertAnalysis('unique', '1 1 1', ['1','1','1']); - assertAnalysis('notnull', ' / / ', []); +const elastictest = require('elastictest') +const punctuation = require('../punctuation') + +module.exports.tests = {} + +module.exports.tests.analyze = function (test, common) { + test('analyze', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasQuery') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('asciifolding', 'é', ['e']) + assertAnalysis('asciifolding', 'ß', ['ss']) + assertAnalysis('asciifolding', 'æ', ['ae']) + assertAnalysis('asciifolding', 'ł', ['l']) + assertAnalysis('asciifolding', 'ɰ', ['m']) + assertAnalysis('lowercase', 'F', ['f']) + assertAnalysis('trim', ' f ', ['f']) + assertAnalysis('remove_ordinals', '26t', ['26']) + assertAnalysis('remove_ordinals', '26th', ['26']) + assertAnalysis('removeAllZeroNumericPrefix', '00001', ['1']) + assertAnalysis('unique', '1 1 1', ['1', '1', '1']) + assertAnalysis('notnull', ' / / ', []) // no stemming is applied - assertAnalysis('no kstem', 'mcdonalds', ['mcdonalds']); - assertAnalysis('no kstem', 'McDonald\'s', ['mcdonalds']); - assertAnalysis('no kstem', 'peoples', ['peoples']); + assertAnalysis('no kstem', 'mcdonalds', ['mcdonalds']) + assertAnalysis('no kstem', 'McDonald\'s', ['mcdonalds']) + assertAnalysis('no kstem', 'peoples', ['peoples']) // remove punctuation (handled by the char_filter) - assertAnalysis( 'punctuation', punctuation.all.join(''), ['&'] ); + assertAnalysis('punctuation', punctuation.all.join(''), ['&']) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} -module.exports.tests.functional = function(test, common){ - test( 'functional', function(t){ +module.exports.tests.functional = function (test, common) { + test('functional', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasQuery') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasQuery' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up + assertAnalysis('country', 'Trinidad and Tobago', [ 'trinidad', 'and', 'tobago' ]) + assertAnalysis('place', 'Toys "R" Us!', [ 'toys', 'r', 'us' ]) + assertAnalysis('address', '101 mapzen place', [ '101', 'mapzen', 'place' ]) - assertAnalysis( 'country', 'Trinidad and Tobago', [ 'trinidad', 'and', 'tobago' ]); - assertAnalysis( 'place', 'Toys "R" Us!', [ 'toys', 'r', 'us' ]); - assertAnalysis( 'address', '101 mapzen place', [ '101', 'mapzen', 'place' ]); + suite.run(t.end) + }) +} - suite.run( t.end ); - }); -}; +module.exports.tests.address = function (test, common) { + test('address', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasQuery') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up -module.exports.tests.address = function(test, common){ - test( 'address', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasQuery' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'address', '101 mapzen place', [ + assertAnalysis('address', '101 mapzen place', [ '101', 'mapzen', 'place' - ]); + ]) - assertAnalysis( 'address', '30 w 26 st', [ + assertAnalysis('address', '30 w 26 st', [ '30', 'w', '26', 'st' - ]); + ]) - assertAnalysis( 'address', '4B 921 83 st', [ + assertAnalysis('address', '4B 921 83 st', [ '4b', '921', '83', 'st' - ]); + ]) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} // @see: https://github.com/pelias/api/issues/600 -module.exports.tests.unicode = function(test, common){ - test( 'normalization', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasQuery' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - var latin_large_letter_e_with_acute = String.fromCodePoint(0x00C9); - var latin_small_letter_e_with_acute = String.fromCodePoint(0x00E9); - var combining_acute_accent = String.fromCodePoint(0x0301); - var latin_large_letter_e = String.fromCodePoint(0x0045); - var latin_small_letter_e = String.fromCodePoint(0x0065); +module.exports.tests.unicode = function (test, common) { + test('normalization', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasQuery') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + const latinLargeLetterEWithAcute = String.fromCodePoint(0x00C9) + const latinSmallLetterEWithAcute = String.fromCodePoint(0x00E9) + const combiningAcuteAccent = String.fromCodePoint(0x0301) + const latinLargeLetterE = String.fromCodePoint(0x0045) + const latinSmallLetterE = String.fromCodePoint(0x0065) // Chambéry (both forms appear the same) - var composed = "Chamb" + latin_small_letter_e_with_acute + "ry"; - var decomposed = "Chamb" + combining_acute_accent + latin_small_letter_e + "ry" + let composed = 'Chamb' + latinSmallLetterEWithAcute + 'ry' + let decomposed = 'Chamb' + combiningAcuteAccent + latinSmallLetterE + 'ry' - assertAnalysis( 'composed', composed, ['chambery'] ); - assertAnalysis( 'decomposed', decomposed, ['chambery'] ); + assertAnalysis('composed', composed, ['chambery']) + assertAnalysis('decomposed', decomposed, ['chambery']) // Één (both forms appear the same) - var composed = latin_large_letter_e_with_acute + latin_small_letter_e_with_acute + "n"; - var decomposed = combining_acute_accent + latin_large_letter_e + combining_acute_accent + latin_small_letter_e + "n" + composed = latinLargeLetterEWithAcute + latinSmallLetterEWithAcute + 'n' + decomposed = combiningAcuteAccent + latinLargeLetterE + combiningAcuteAccent + latinSmallLetterE + 'n' - assertAnalysis( 'composed', composed, ['een'] ); - assertAnalysis( 'decomposed', decomposed, ['een'] ); + assertAnalysis('composed', composed, ['een']) + assertAnalysis('decomposed', decomposed, ['een']) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('peliasQuery: ' + name, testFunction); + function test (name, testFunction) { + return tape('peliasQuery: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/analyzer_peliasStreet.js b/integration/analyzer_peliasStreet.js index 6ad9355e..68c42bbc 100644 --- a/integration/analyzer_peliasStreet.js +++ b/integration/analyzer_peliasStreet.js @@ -1,225 +1,217 @@ // validate analyzer is behaving as expected const elastictest = require('elastictest') -module.exports.tests = {}; - -module.exports.tests.analyze = function(test, common){ - test( 'analyze', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'lowercase', 'F', ['f']); - assertAnalysis( 'asciifolding', 'Max-Beer-Straße', ['0:max', '1:beer', '2:strasse', '2:str']); - assertAnalysis( 'trim', ' f ', ['f'] ); - assertAnalysis( 'keyword_street_suffix', 'foo Street', ['0:foo', '1:street', '1:st'] ); - assertAnalysis( 'keyword_street_suffix', 'foo Road', ['0:foo', '1:road', '1:rd'] ); - assertAnalysis( 'keyword_street_suffix', 'foo Crescent', ['0:foo', '1:crescent', '1:cres'] ); - assertAnalysis( 'keyword_compass', 'north foo', ['0:north', '0:n', '1:foo'] ); - assertAnalysis( 'keyword_compass', 'SouthWest foo', ['0:southwest', '0:sw', '1:foo'] ); - assertAnalysis( 'keyword_compass', 'foo SouthWest', ['0:foo', '1:southwest', '1:sw'] ); - assertAnalysis( 'remove_ordinals', '1st 2nd 3rd 4th 5th', ['1','2','3','4','5'] ); - assertAnalysis( 'remove_ordinals', 'Ast th 101st', ['ast','th','101'] ); - - suite.run( t.end ); - }); -}; - -module.exports.tests.functional = function(test, common){ - test( 'functional', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'USA address', 'west 26th street', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]); - assertAnalysis( 'USA address', 'West 26th Street', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]); - assertAnalysis( 'USA address', 'w 26th st', [ '0:w', '0:west', '1:26', '2:st', '2:street' ]); - assertAnalysis( 'USA address', 'WEST 26th STREET', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]); - assertAnalysis( 'USA address', 'WEST 26th ST', [ '0:west', '0:w', '1:26', '2:st', '2:street' ]); - - suite.run( t.end ); - }); -}; - -module.exports.tests.normalize_punctuation = function(test, common){ - test( 'normalize punctuation', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - var expected = [ '0:chapala', '1:street', '1:st' ]; - - assertAnalysis( 'single space', 'Chapala Street', expected ); - assertAnalysis( 'double space', 'Chapala Street', expected ); - assertAnalysis( 'triple space', 'Chapala Street', expected ); - assertAnalysis( 'quad space', 'Chapala Street', expected ); - - suite.run( t.end ); - }); -}; - -module.exports.tests.remove_ordinals = function(test, common){ - test( 'remove ordinals', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'ordindals', "1st", ["1"] ); - assertAnalysis( 'ordindals', "22nd", ["22"] ); - assertAnalysis( 'ordindals', "333rd", ["333"] ); - assertAnalysis( 'ordindals', "4444th", ["4444"] ); - assertAnalysis( 'ordindals', "2500th", ["2500"] ); +module.exports.tests = {} + +module.exports.tests.analyze = function (test, common) { + test('analyze', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('lowercase', 'F', ['f']) + assertAnalysis('asciifolding', 'Max-Beer-Straße', ['0:max', '1:beer', '2:strasse', '2:str']) + assertAnalysis('trim', ' f ', ['f']) + assertAnalysis('keyword_street_suffix', 'foo Street', ['0:foo', '1:street', '1:st']) + assertAnalysis('keyword_street_suffix', 'foo Road', ['0:foo', '1:road', '1:rd']) + assertAnalysis('keyword_street_suffix', 'foo Crescent', ['0:foo', '1:crescent', '1:cres']) + assertAnalysis('keyword_compass', 'north foo', ['0:north', '0:n', '1:foo']) + assertAnalysis('keyword_compass', 'SouthWest foo', ['0:southwest', '0:sw', '1:foo']) + assertAnalysis('keyword_compass', 'foo SouthWest', ['0:foo', '1:southwest', '1:sw']) + assertAnalysis('remove_ordinals', '1st 2nd 3rd 4th 5th', ['1', '2', '3', '4', '5']) + assertAnalysis('remove_ordinals', 'Ast th 101st', ['ast', 'th', '101']) + + suite.run(t.end) + }) +} + +module.exports.tests.functional = function (test, common) { + test('functional', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('USA address', 'west 26th street', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]) + assertAnalysis('USA address', 'West 26th Street', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]) + assertAnalysis('USA address', 'w 26th st', [ '0:w', '0:west', '1:26', '2:st', '2:street' ]) + assertAnalysis('USA address', 'WEST 26th STREET', [ '0:west', '0:w', '1:26', '2:street', '2:st' ]) + assertAnalysis('USA address', 'WEST 26th ST', [ '0:west', '0:w', '1:26', '2:st', '2:street' ]) + + suite.run(t.end) + }) +} + +module.exports.tests.normalize_punctuation = function (test, common) { + test('normalize punctuation', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + var expected = [ '0:chapala', '1:street', '1:st' ] + + assertAnalysis('single space', 'Chapala Street', expected) + assertAnalysis('double space', 'Chapala Street', expected) + assertAnalysis('triple space', 'Chapala Street', expected) + assertAnalysis('quad space', 'Chapala Street', expected) + + suite.run(t.end) + }) +} + +module.exports.tests.remove_ordinals = function (test, common) { + test('remove ordinals', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('ordindals', '1st', ['1']) + assertAnalysis('ordindals', '22nd', ['22']) + assertAnalysis('ordindals', '333rd', ['333']) + assertAnalysis('ordindals', '4444th', ['4444']) + assertAnalysis('ordindals', '2500th', ['2500']) // teens - assertAnalysis( 'teens', "11th", ["11"] ); - assertAnalysis( 'teens', "12th", ["12"] ); - assertAnalysis( 'teens', "13th", ["13"] ); - assertAnalysis( 'teens', "14th", ["14"] ); - assertAnalysis( 'teens', "15th", ["15"] ); - assertAnalysis( 'teens', "16th", ["16"] ); - assertAnalysis( 'teens', "17th", ["17"] ); - assertAnalysis( 'teens', "18th", ["18"] ); - assertAnalysis( 'teens', "19th", ["19"] ); - assertAnalysis( 'teens', "20th", ["20"] ); + assertAnalysis('teens', '11th', ['11']) + assertAnalysis('teens', '12th', ['12']) + assertAnalysis('teens', '13th', ['13']) + assertAnalysis('teens', '14th', ['14']) + assertAnalysis('teens', '15th', ['15']) + assertAnalysis('teens', '16th', ['16']) + assertAnalysis('teens', '17th', ['17']) + assertAnalysis('teens', '18th', ['18']) + assertAnalysis('teens', '19th', ['19']) + assertAnalysis('teens', '20th', ['20']) // teens (hundreds) - assertAnalysis( 'teens - hundreds', "111th", ["111"] ); - assertAnalysis( 'teens - hundreds', "112th", ["112"] ); - assertAnalysis( 'teens - hundreds', "113th", ["113"] ); - assertAnalysis( 'teens - hundreds', "114th", ["114"] ); - assertAnalysis( 'teens - hundreds', "115th", ["115"] ); - assertAnalysis( 'teens - hundreds', "116th", ["116"] ); - assertAnalysis( 'teens - hundreds', "117th", ["117"] ); - assertAnalysis( 'teens - hundreds', "118th", ["118"] ); - assertAnalysis( 'teens - hundreds', "119th", ["119"] ); - assertAnalysis( 'teens - hundreds', "120th", ["120"] ); + assertAnalysis('teens - hundreds', '111th', ['111']) + assertAnalysis('teens - hundreds', '112th', ['112']) + assertAnalysis('teens - hundreds', '113th', ['113']) + assertAnalysis('teens - hundreds', '114th', ['114']) + assertAnalysis('teens - hundreds', '115th', ['115']) + assertAnalysis('teens - hundreds', '116th', ['116']) + assertAnalysis('teens - hundreds', '117th', ['117']) + assertAnalysis('teens - hundreds', '118th', ['118']) + assertAnalysis('teens - hundreds', '119th', ['119']) + assertAnalysis('teens - hundreds', '120th', ['120']) // teens (wrong suffix) - assertAnalysis( 'teens - wrong suffix', "11st", ["11st"] ); - assertAnalysis( 'teens - wrong suffix', "12nd", ["12nd"] ); - assertAnalysis( 'teens - wrong suffix', "13rd", ["13rd"] ); - assertAnalysis( 'teens - wrong suffix', "111st", ["111st"] ); - assertAnalysis( 'teens - wrong suffix', "112nd", ["112nd"] ); - assertAnalysis( 'teens - wrong suffix', "113rd", ["113rd"] ); + assertAnalysis('teens - wrong suffix', '11st', ['11st']) + assertAnalysis('teens - wrong suffix', '12nd', ['12nd']) + assertAnalysis('teens - wrong suffix', '13rd', ['13rd']) + assertAnalysis('teens - wrong suffix', '111st', ['111st']) + assertAnalysis('teens - wrong suffix', '112nd', ['112nd']) + assertAnalysis('teens - wrong suffix', '113rd', ['113rd']) // uppercase - assertAnalysis( 'uppercase', "1ST", ["1"] ); - assertAnalysis( 'uppercase', "22ND", ["22"] ); - assertAnalysis( 'uppercase', "333RD", ["333"] ); - assertAnalysis( 'uppercase', "4444TH", ["4444"] ); + assertAnalysis('uppercase', '1ST', ['1']) + assertAnalysis('uppercase', '22ND', ['22']) + assertAnalysis('uppercase', '333RD', ['333']) + assertAnalysis('uppercase', '4444TH', ['4444']) // autocomplete - assertAnalysis( 'autocomplete', "26", ["26"] ); - assertAnalysis( 'autocomplete', "26t", ["26"] ); - assertAnalysis( 'autocomplete', "26th", ["26"] ); - assertAnalysis( 'autocomplete', "3", ["3"] ); - assertAnalysis( 'autocomplete', "3r", ["3"] ); - assertAnalysis( 'autocomplete', "3rd", ["3"] ); + assertAnalysis('autocomplete', '26', ['26']) + assertAnalysis('autocomplete', '26t', ['26']) + assertAnalysis('autocomplete', '26th', ['26']) + assertAnalysis('autocomplete', '3', ['3']) + assertAnalysis('autocomplete', '3r', ['3']) + assertAnalysis('autocomplete', '3rd', ['3']) // wrong suffix - assertAnalysis( 'wrong suffix (do nothing)', "0th", ["0th"] ); - assertAnalysis( 'wrong suffix (do nothing)', "26s", ["26s"] ); - assertAnalysis( 'wrong suffix (do nothing)', "26st", ["26st"] ); - assertAnalysis( 'wrong suffix (do nothing)', "31t", ["31t"] ); - assertAnalysis( 'wrong suffix (do nothing)', "31th", ["31th"] ); - assertAnalysis( 'wrong suffix (do nothing)', "21r", ["21r"] ); - assertAnalysis( 'wrong suffix (do nothing)', "21rd", ["21rd"] ); - assertAnalysis( 'wrong suffix (do nothing)', "29n", ["29n"] ); - assertAnalysis( 'wrong suffix (do nothing)', "29nd", ["29nd"] ); - - suite.run( t.end ); - }); -}; - -module.exports.tests.tokenizer = function(test, common){ - test( 'tokenizer', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - var expected = [ '0:bedell', '1:street', '1:st', '2:133', '3:avenue', '3:ave', '3:av' ]; + assertAnalysis('wrong suffix (do nothing)', '0th', ['0th']) + assertAnalysis('wrong suffix (do nothing)', '26s', ['26s']) + assertAnalysis('wrong suffix (do nothing)', '26st', ['26st']) + assertAnalysis('wrong suffix (do nothing)', '31t', ['31t']) + assertAnalysis('wrong suffix (do nothing)', '31th', ['31th']) + assertAnalysis('wrong suffix (do nothing)', '21r', ['21r']) + assertAnalysis('wrong suffix (do nothing)', '21rd', ['21rd']) + assertAnalysis('wrong suffix (do nothing)', '29n', ['29n']) + assertAnalysis('wrong suffix (do nothing)', '29nd', ['29nd']) + + suite.run(t.end) + }) +} + +module.exports.tests.tokenizer = function (test, common) { + test('tokenizer', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + var expected = [ '0:bedell', '1:street', '1:st', '2:133', '3:avenue', '3:ave', '3:av' ] // specify 2 streets with a delimeter - assertAnalysis( 'forward slash', 'Bedell Street/133rd Avenue', expected ); - assertAnalysis( 'forward slash', 'Bedell Street /133rd Avenue', expected ); - assertAnalysis( 'forward slash', 'Bedell Street/ 133rd Avenue', expected ); - assertAnalysis( 'back slash', 'Bedell Street\\133rd Avenue', expected ); - assertAnalysis( 'back slash', 'Bedell Street \\133rd Avenue', expected ); - assertAnalysis( 'back slash', 'Bedell Street\\ 133rd Avenue', expected ); - assertAnalysis( 'comma', 'Bedell Street,133rd Avenue', expected ); - assertAnalysis( 'comma', 'Bedell Street ,133rd Avenue', expected ); - assertAnalysis( 'comma', 'Bedell Street, 133rd Avenue', expected ); - - suite.run( t.end ); - }); -}; + assertAnalysis('forward slash', 'Bedell Street/133rd Avenue', expected) + assertAnalysis('forward slash', 'Bedell Street /133rd Avenue', expected) + assertAnalysis('forward slash', 'Bedell Street/ 133rd Avenue', expected) + assertAnalysis('back slash', 'Bedell Street\\133rd Avenue', expected) + assertAnalysis('back slash', 'Bedell Street \\133rd Avenue', expected) + assertAnalysis('back slash', 'Bedell Street\\ 133rd Avenue', expected) + assertAnalysis('comma', 'Bedell Street,133rd Avenue', expected) + assertAnalysis('comma', 'Bedell Street ,133rd Avenue', expected) + assertAnalysis('comma', 'Bedell Street, 133rd Avenue', expected) + + suite.run(t.end) + }) +} // @see: https://github.com/pelias/api/issues/600 -module.exports.tests.unicode = function(test, common){ - test( 'normalization', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasStreet' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - var latin_large_letter_e_with_acute = String.fromCodePoint(0x00C9); - var latin_small_letter_e_with_acute = String.fromCodePoint(0x00E9); - var combining_acute_accent = String.fromCodePoint(0x0301); - var latin_large_letter_e = String.fromCodePoint(0x0045); - var latin_small_letter_e = String.fromCodePoint(0x0065); +module.exports.tests.unicode = function (test, common) { + test('normalization', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasQuery') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + const latinLargeLetterEWithAcute = String.fromCodePoint(0x00C9) + const latinSmallLetterEWithAcute = String.fromCodePoint(0x00E9) + const combiningAcuteAccent = String.fromCodePoint(0x0301) + const latinLargeLetterE = String.fromCodePoint(0x0045) + const latinSmallLetterE = String.fromCodePoint(0x0065) // Chambéry (both forms appear the same) - var composed = "Chamb" + latin_small_letter_e_with_acute + "ry"; - var decomposed = "Chamb" + combining_acute_accent + latin_small_letter_e + "ry" + let composed = 'Chamb' + latinSmallLetterEWithAcute + 'ry' + let decomposed = 'Chamb' + combiningAcuteAccent + latinSmallLetterE + 'ry' - assertAnalysis( 'composed', composed, ['chambery'] ); - assertAnalysis( 'decomposed', decomposed, ['chambery'] ); + assertAnalysis('composed', composed, ['chambery']) + assertAnalysis('decomposed', decomposed, ['chambery']) // Één (both forms appear the same) - var composed = latin_large_letter_e_with_acute + latin_small_letter_e_with_acute + "n"; - var decomposed = combining_acute_accent + latin_large_letter_e + combining_acute_accent + latin_small_letter_e + "n" + composed = latinLargeLetterEWithAcute + latinSmallLetterEWithAcute + 'n' + decomposed = combiningAcuteAccent + latinLargeLetterE + combiningAcuteAccent + latinSmallLetterE + 'n' - assertAnalysis( 'composed', composed, ['een'] ); - assertAnalysis( 'decomposed', decomposed, ['een'] ); + assertAnalysis('composed', composed, ['een']) + assertAnalysis('decomposed', decomposed, ['een']) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} module.exports.tests.germanic_street_suffixes = function (test, common) { test('germanic_street_suffixes', function (t) { - - var suite = new elastictest.Suite(common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet'); - suite.action(function (done) { setTimeout(done, 500); }); // wait for es to bring some shards up + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasStreet') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // Germanic street suffixes - assertAnalysis('straße', 'straße', ['0:strasse', '0:str']); - assertAnalysis('strasse', 'strasse', ['0:strasse', '0:str']); - assertAnalysis('str.', 'str.', ['0:str', '0:strasse']); - assertAnalysis('str', 'str', ['0:str', '0:strasse']); - assertAnalysis('brücke', 'brücke', [ '0:bruecke', '0:brucke', '0:br' ]); - assertAnalysis('bruecke', 'bruecke', [ '0:bruecke', '0:brucke', '0:br' ]); - assertAnalysis('brucke', 'brucke', ['0:brucke', '0:bruecke', '0:br']); - assertAnalysis('br.', 'br.', ['0:br', '0:branch', '0:bruecke', '0:brucke']); - assertAnalysis('br', 'br', ['0:br', '0:branch', '0:bruecke', '0:brucke']); - - suite.run(t.end); - }); -}; + assertAnalysis('straße', 'straße', ['0:strasse', '0:str']) + assertAnalysis('strasse', 'strasse', ['0:strasse', '0:str']) + assertAnalysis('str.', 'str.', ['0:str', '0:strasse']) + assertAnalysis('str', 'str', ['0:str', '0:strasse']) + assertAnalysis('brücke', 'brücke', [ '0:bruecke', '0:brucke', '0:br' ]) + assertAnalysis('bruecke', 'bruecke', [ '0:bruecke', '0:brucke', '0:br' ]) + assertAnalysis('brucke', 'brucke', ['0:brucke', '0:bruecke', '0:br']) + assertAnalysis('br.', 'br.', ['0:br', '0:branch', '0:bruecke', '0:brucke']) + assertAnalysis('br', 'br', ['0:br', '0:branch', '0:bruecke', '0:brucke']) + + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('peliasStreet: ' + name, testFunction); + function test (name, testFunction) { + return tape('peliasStreet: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/analyzer_peliasZip.js b/integration/analyzer_peliasZip.js index 4788547a..307a50ad 100644 --- a/integration/analyzer_peliasZip.js +++ b/integration/analyzer_peliasZip.js @@ -1,52 +1,46 @@ // validate analyzer is behaving as expected - -var tape = require('tape'), - elastictest = require('elastictest'), - punctuation = require('../punctuation'); - -module.exports.tests = {}; - -module.exports.tests.analyze = function(test, common){ - test( 'analyze', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasZip' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'lowercase', 'F', ['f']); - assertAnalysis( 'trim', ' f ', ['f'] ); - assertAnalysis( 'alphanumeric', 'a-f g', ['afg'] ); - - suite.run( t.end ); - }); -}; - -module.exports.tests.functional = function(test, common){ - test( 'functional', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasZip' ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up - - assertAnalysis( 'usa zip', '10010', [ '10010' ]); - assertAnalysis( 'usa zip', 10010, [ '10010' ]); - assertAnalysis( 'usa zip - punctuation', '10-010', [ '10010' ]); - assertAnalysis( 'usa zip - whitespace', '10 010', [ '10010' ]); - assertAnalysis( 'uk postcode', 'E24DN', [ 'e24dn' ]); - assertAnalysis( 'uk postcode - punctuation', 'E2-4DN', [ 'e24dn' ]); - assertAnalysis( 'uk postcode - whitespace', 'E 24 DN', [ 'e24dn' ]); - - suite.run( t.end ); - }); -}; +const elastictest = require('elastictest') + +module.exports.tests = {} + +module.exports.tests.analyze = function (test, common) { + test('analyze', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasZip') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('lowercase', 'F', ['f']) + assertAnalysis('trim', ' f ', ['f']) + assertAnalysis('alphanumeric', 'a-f g', ['afg']) + + suite.run(t.end) + }) +} + +module.exports.tests.functional = function (test, common) { + test('functional', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + var assertAnalysis = common.analyze.bind(null, suite, t, 'peliasZip') + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up + + assertAnalysis('usa zip', '10010', [ '10010' ]) + assertAnalysis('usa zip', 10010, [ '10010' ]) + assertAnalysis('usa zip - punctuation', '10-010', [ '10010' ]) + assertAnalysis('usa zip - whitespace', '10 010', [ '10010' ]) + assertAnalysis('uk postcode', 'E24DN', [ 'e24dn' ]) + assertAnalysis('uk postcode - punctuation', 'E2-4DN', [ 'e24dn' ]) + assertAnalysis('uk postcode - whitespace', 'E 24 DN', [ 'e24dn' ]) + + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('peliasZip: ' + name, testFunction); + function test (name, testFunction) { + return tape('peliasZip: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/autocomplete_abbreviated_street_names.js b/integration/autocomplete_abbreviated_street_names.js index a2f5f8fb..b6a29cb5 100644 --- a/integration/autocomplete_abbreviated_street_names.js +++ b/integration/autocomplete_abbreviated_street_names.js @@ -4,31 +4,30 @@ // The greater issue is descriped in: https://github.com/pelias/pelias/issues/211 // The cases tested here are described in: https://github.com/pelias/schema/issues/105 -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); -const getTotalHits = require('./_hits_total_helper'); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() +const getTotalHits = require('./_hits_total_helper') -module.exports.tests = {}; +module.exports.tests = {} // index the name as 'Grolmanstraße' and then retrieve with partially complete token 'Grolmanstr.' -module.exports.tests.index_expanded_form_search_contracted = function(test, common){ - test( 'index expanded and retrieve contracted form', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_expanded_form_search_contracted = function (test, common) { + test('index expanded and retrieve contracted form', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'Grolmanstraße' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -37,17 +36,17 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm 'analyzer': 'peliasQuery', 'query': 'Grolmanstr.' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // Note: this test is commented out, it's a behaviour we would like to have but currently // do not support. @@ -115,12 +114,11 @@ module.exports.tests.index_expanded_form_search_contracted = function(test, comm // }; module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('autocomplete abbreviated street names: ' + name, testFunction); + function test (name, testFunction) { + return tape('autocomplete abbreviated street names: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/autocomplete_directional_synonym_expansion.js b/integration/autocomplete_directional_synonym_expansion.js index 30f0a10d..862c5b23 100644 --- a/integration/autocomplete_directional_synonym_expansion.js +++ b/integration/autocomplete_directional_synonym_expansion.js @@ -4,31 +4,30 @@ // The greater issue is descriped in: https://github.com/pelias/pelias/issues/211 // The cases tested here are described in: https://github.com/pelias/schema/issues/105 -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); -const getTotalHits = require('./_hits_total_helper'); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() +const getTotalHits = require('./_hits_total_helper') -module.exports.tests = {}; +module.exports.tests = {} // index the name as 'north' and then retrieve with partially complete token 'nor' -module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ - test( 'index and retrieve expanded form', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_expanded_form = function (test, common) { + test('index and retrieve expanded form', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'north' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -37,16 +36,16 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'nor' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -55,37 +54,36 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'north' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'n' and then retrieve with 'n' -module.exports.tests.index_and_retrieve_contracted_form = function(test, common){ - test( 'index and retrieve contracted form', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_contracted_form = function (test, common) { + test('index and retrieve contracted form', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'n' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -94,37 +92,36 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) 'analyzer': 'peliasQuery', 'query': 'n' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'n' and then retrieve with partially complete token 'nor' -module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ - test( 'index and retrieve mixed form 1', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_mixed_form_1 = function (test, common) { + test('index and retrieve mixed form 1', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'n' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -133,16 +130,16 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'nor' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -151,37 +148,36 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'north' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'north' and then retrieve with 'n' -module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ - test( 'index and retrieve mixed form 2', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_mixed_form_2 = function (test, common) { + test('index and retrieve mixed form 2', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'north' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -190,25 +186,24 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'n' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('autocomplete directional synonym expansion: ' + name, testFunction); + function test (name, testFunction) { + return tape('autocomplete directional synonym expansion: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/autocomplete_street_synonym_expansion.js b/integration/autocomplete_street_synonym_expansion.js index 55e1719c..bb133ab2 100644 --- a/integration/autocomplete_street_synonym_expansion.js +++ b/integration/autocomplete_street_synonym_expansion.js @@ -4,32 +4,31 @@ // The greater issue is descriped in: https://github.com/pelias/pelias/issues/211 // The cases tested here are described in: https://github.com/pelias/schema/issues/105 -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() -const getTotalHits = require('./_hits_total_helper'); +const getTotalHits = require('./_hits_total_helper') -module.exports.tests = {}; +module.exports.tests = {} // index the name as 'center' and then retrieve with partially complete token 'cent' -module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ - test( 'index and retrieve expanded form', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_expanded_form = function (test, common) { + test('index and retrieve expanded form', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'center' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -38,16 +37,16 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'cent' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -56,37 +55,36 @@ module.exports.tests.index_and_retrieve_expanded_form = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'center' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'ctr' and then retrieve with 'ctr' -module.exports.tests.index_and_retrieve_contracted_form = function(test, common){ - test( 'index and retrieve contracted form', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_contracted_form = function (test, common) { + test('index and retrieve contracted form', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'ctr' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -95,37 +93,36 @@ module.exports.tests.index_and_retrieve_contracted_form = function(test, common) 'analyzer': 'peliasQuery', 'query': 'ctr' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'ctr' and then retrieve with partially complete token 'cent' -module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ - test( 'index and retrieve mixed form 1', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_mixed_form_1 = function (test, common) { + test('index and retrieve mixed form 1', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'ctr' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -134,16 +131,16 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'cent' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -152,37 +149,36 @@ module.exports.tests.index_and_retrieve_mixed_form_1 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'center' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} // index the name as 'center' and then retrieve with 'ctr' -module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ - test( 'index and retrieve mixed form 2', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve_mixed_form_2 = function (test, common) { + test('index and retrieve mixed form 2', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a name which contains a synonym (center) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, id: '1', body: { name: { default: 'center' } } - }, done); - }); + }, done) + }) // search using 'peliasQuery' - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -191,25 +187,24 @@ module.exports.tests.index_and_retrieve_mixed_form_2 = function(test, common){ 'analyzer': 'peliasQuery', 'query': 'ctr' } - }}} - }, function( err, res ){ - t.equal( err, undefined ); - t.equal( getTotalHits(res.hits), 1, 'document found' ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } } + }, function (err, res) { + t.equal(err, undefined) + t.equal(getTotalHits(res.hits), 1, 'document found') + done() + }) + }) + + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('autocomplete street synonym expansion: ' + name, testFunction); + function test (name, testFunction) { + return tape('autocomplete street synonym expansion: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/bounding_box.js b/integration/bounding_box.js index ecd23d0a..7c281b01 100644 --- a/integration/bounding_box.js +++ b/integration/bounding_box.js @@ -1,18 +1,17 @@ // validate bounding box behaves as expected -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.index_and_retrieve = function(test, common){ - test( 'index and retrieve', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.index_and_retrieve = function (test, common) { + test('index and retrieve', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with a bbox - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, @@ -20,11 +19,11 @@ module.exports.tests.index_and_retrieve = function(test, common){ body: { bounding_box: '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}' } - }, done); - }); + }, done) + }) // retrieve document by id - suite.assert( function( done ) { + suite.assert(function (done) { suite.client.get( { index: suite.props.index, @@ -32,24 +31,23 @@ module.exports.tests.index_and_retrieve = function(test, common){ id: '1' }, function (err, res) { - t.equal(err, undefined); - t.deepEqual(res._source.bounding_box, '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}'); - done(); + t.equal(err, undefined) + t.deepEqual(res._source.bounding_box, '{"min_lat":-47.75,"max_lat":-33.9,"min_lon":163.82,"max_lon":179.42}') + done() } - ); - }); + ) + }) - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('bounding box: ' + name, testFunction); + function test (name, testFunction) { + return tape('bounding box: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/dynamic_templates.js b/integration/dynamic_templates.js index 4ece07b7..fee8df0a 100644 --- a/integration/dynamic_templates.js +++ b/integration/dynamic_templates.js @@ -1,170 +1,163 @@ // http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-root-object-type.html#_dynamic_templates +const elastictest = require('elastictest') +const config = require('pelias-config').generate() -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); +module.exports.tests = {} -module.exports.tests = {}; - -module.exports.tests.dynamic_templates_name = function(test, common){ - test( 'document->name', nameAssertion( 'peliasIndexOneEdgeGram', common ) ); -}; +module.exports.tests.dynamic_templates_name = function (test, common) { + test('document->name', nameAssertion('peliasIndexOneEdgeGram', common)) +} -module.exports.tests.dynamic_templates_phrase = function(test, common){ - test( 'document->phrase', phraseAssertion( 'peliasPhrase', common ) ); -}; +module.exports.tests.dynamic_templates_phrase = function (test, common) { + test('document->phrase', phraseAssertion('peliasPhrase', common)) +} -module.exports.tests.dynamic_templates_addendum = function(test, common){ - test( 'addendum', addendumAssertion( 'wikipedia', JSON.stringify({ slug: 'Wikipedia' }), common ) ); -}; +module.exports.tests.dynamic_templates_addendum = function (test, common) { + test('addendum', addendumAssertion('wikipedia', JSON.stringify({ slug: 'Wikipedia' }), common)) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('dynamic_templates: ' + name, testFunction); + function test (name, testFunction) { + return tape('dynamic_templates: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; - -function nameAssertion( analyzer, common ){ - return function(t){ +} - var suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; +function nameAssertion (analyzer, common) { + return function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + const _type = config.schema.typeName // index a document from a normal document layer - suite.action( done => { + suite.action(done => { suite.client.index({ index: suite.props.index, type: _type, id: '1', body: { name: { default: 'foo', alt: 'bar' } } - }, done ); - }); + }, done) + }) // check dynamically created mapping has // inherited from the dynamic_template - suite.assert( done => { - + suite.assert(done => { suite.client.indices.getMapping({ index: suite.props.index, include_type_name: false }, (err, res) => { - - const properties = res[suite.props.index].mappings.properties; - t.equal( properties.name.dynamic, 'true' ); - - const nameProperties = properties.name.properties; - t.equal( nameProperties.default.analyzer, analyzer ); - t.equal( nameProperties.alt.analyzer, analyzer ); - done(); - }); - }); - - suite.run( t.end ); - }; + t.false(err) + const properties = res[suite.props.index].mappings.properties + t.equal(properties.name.dynamic, 'true') + + const nameProperties = properties.name.properties + t.equal(nameProperties.default.analyzer, analyzer) + t.equal(nameProperties.alt.analyzer, analyzer) + done() + }) + }) + + suite.run(t.end) + } } -function phraseAssertion( analyzer, common ){ - return function(t){ - - const suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; +function phraseAssertion (analyzer, common) { + return function (t) { + const suite = new elastictest.Suite(common.clientOpts, common.create) + const _type = config.schema.typeName // index a document from a normal document layer - suite.action( done => { + suite.action(done => { suite.client.index({ index: suite.props.index, type: _type, id: '1', body: { phrase: { default: 'foo', alt: 'bar' } } - }, done ); - }); + }, done) + }) // check dynamically created mapping has // inherited from the dynamic_template - suite.assert( done => { - + suite.assert(done => { suite.client.indices.getMapping({ index: suite.props.index, include_type_name: false - }, ( err, res ) => { - - const properties = res[suite.props.index].mappings.properties; - t.equal( properties.phrase.dynamic, 'true' ); - - const phraseProperties = properties.phrase.properties; - t.equal( phraseProperties.default.analyzer, analyzer ); - t.equal( phraseProperties.alt.analyzer, analyzer ); - done(); - }); - }); - - suite.run( t.end ); - }; + }, (err, res) => { + t.false(err) + const properties = res[suite.props.index].mappings.properties + t.equal(properties.phrase.dynamic, 'true') + + const phraseProperties = properties.phrase.properties + t.equal(phraseProperties.default.analyzer, analyzer) + t.equal(phraseProperties.alt.analyzer, analyzer) + done() + }) + }) + + suite.run(t.end) + } } -function addendumAssertion( namespace, value, common ){ - return function(t){ - - const suite = new elastictest.Suite( common.clientOpts, common.create ); - const _type = config.schema.typeName; +function addendumAssertion (namespace, value, common) { + return function (t) { + const suite = new elastictest.Suite(common.clientOpts, common.create) + const _type = config.schema.typeName // index a document including the addendum - suite.action( done => { + suite.action(done => { suite.client.index({ index: suite.props.index, type: _type, id: '1', - body: { addendum: { [namespace]: value } }, - }, done ); - }); + body: { addendum: { [namespace]: value } } + }, done) + }) // check dynamically created mapping has // inherited from the dynamic_template - suite.assert( done => { + suite.assert(done => { suite.client.indices.getMapping({ index: suite.props.index, - include_type_name: false, - }, ( err, res ) => { - - const properties = res[suite.props.index].mappings.properties; - t.equal( properties.addendum.dynamic, 'true' ); + include_type_name: false + }, (err, res) => { + t.false(err) + const properties = res[suite.props.index].mappings.properties + t.equal(properties.addendum.dynamic, 'true') - const addendumProperties = properties.addendum.properties; + const addendumProperties = properties.addendum.properties t.true([ 'keyword' // elasticsearch 5.6 - ].includes( addendumProperties[namespace].type )); + ].includes(addendumProperties[namespace].type)) t.true([ false // elasticsearch 5.6 - ].includes( addendumProperties[namespace].index )); + ].includes(addendumProperties[namespace].index)) // elasticsearch 5.6 - if( addendumProperties[namespace].doc_values ){ - t.equal( addendumProperties[namespace].doc_values, false ); + if (addendumProperties[namespace].doc_values) { + t.equal(addendumProperties[namespace].doc_values, false) } - done(); - }); - }); + done() + }) + }) // retrieve document and check addendum was stored verbatim - suite.assert( done => { + suite.assert(done => { suite.client.get({ index: suite.props.index, type: _type, id: 1 - }, ( err, res ) => { - t.false( err ); - t.equal( res._source.addendum[namespace], value ); - done(); - }); - }); - - suite.run( t.end ); - }; + }, (err, res) => { + t.false(err) + t.equal(res._source.addendum[namespace], value) + done() + }) + }) + + suite.run(t.end) + } } diff --git a/integration/multi_token_synonyms.js b/integration/multi_token_synonyms.js index 721b01f4..30ee1b4f 100644 --- a/integration/multi_token_synonyms.js +++ b/integration/multi_token_synonyms.js @@ -1,17 +1,16 @@ // validate analyzer is behaving as expected -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); +const elastictest = require('elastictest') +const config = require('pelias-config').generate() -module.exports.tests = {}; +module.exports.tests = {} // simple test to cover the issue noted in: // https://github.com/pelias/schema/issues/381#issuecomment-548305594 module.exports.tests.functional = function (test, common) { test('functional', function (t) { - - var suite = new elastictest.Suite(common.clientOpts, common.create); - suite.action(function (done) { setTimeout(done, 500); }); // wait for es to bring some shards up + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index a document with all admin values // note: this will return an error if multi-token synonyms are @@ -20,7 +19,8 @@ module.exports.tests.functional = function (test, common) { suite.client.index({ index: suite.props.index, type: config.schema.typeName, - id: '1', body: { + id: '1', + body: { name: { default: 'set' }, phrase: { default: 'set' }, address_parts: { @@ -31,20 +31,19 @@ module.exports.tests.functional = function (test, common) { country: 'set' } } - }, done); - }); + }, done) + }) - suite.run(t.end); - }); -}; + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('multi token synonyms: ' + name, testFunction); + function test (name, testFunction) { + return tape('multi token synonyms: ' + name, testFunction) } for (var testCase in module.exports.tests) { - module.exports.tests[testCase](test, common); + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/run.js b/integration/run.js index 5b312aca..28566851 100644 --- a/integration/run.js +++ b/integration/run.js @@ -1,8 +1,8 @@ -const _ = require('lodash'); -const tape = require('tape'); -const config = require('pelias-config').generate(); +const _ = require('lodash') +const tape = require('tape') +const config = require('pelias-config').generate() -const schema = require('../schema'); +const schema = require('../schema') const common = { clientOpts: { @@ -22,40 +22,40 @@ const common = { _id: h._id, _score: h._score, name: h._source.name - }; - }); + } + }) }, summary: (res) => { - common.summaryMap( res ) - .forEach( console.dir ); + common.summaryMap(res) + .forEach(console.dir) }, bucketTokens: tokens => { - const positions = {}; + const positions = {} tokens.forEach((t, i) => { // format returned by elasticsearch if (_.isPlainObject(t)) { - const pos = '@pos' + t.position; - if (!positions[pos]) { positions[pos] = []; } - positions[pos].push(t.token); - } + const pos = '@pos' + t.position + if (!positions[pos]) { positions[pos] = [] } + positions[pos].push(t.token) + // 'simple tokens' format // eg '1:foo' - else if (_.isString(t)) { - const match = t.match(/^(\d+):(.+)$/); - let pos = '@pos' + i; + } else if (_.isString(t)) { + const match = t.match(/^(\d+):(.+)$/) + let pos = '@pos' + i if (match) { - pos = '@pos' + match[1]; - t = match[2]; + pos = '@pos' + match[1] + t = match[2] } - if (!positions[pos]) { positions[pos] = []; } - positions[pos].push(t); + if (!positions[pos]) { positions[pos] = [] } + positions[pos].push(t) } - }); + }) // sort all the arrays so that order is irrelevant - for (var attr in positions){ - positions[attr] = positions[attr].sort(); + for (var attr in positions) { + positions[attr] = positions[attr].sort() } - return positions; + return positions }, // the 'analyze' assertion indexes $text using the analyzer specified // in the $analyzer var and then checks that all of the tokens in @@ -72,25 +72,25 @@ const common = { text: text.toString() } }, (err, res) => { - if (err) { console.error(err); } + if (err) { console.error(err) } t.deepEqual({}, removeIndexTokensFromExpectedTokens( common.bucketTokens(res.tokens), common.bucketTokens(expected) - ), comment); - done(); - }); - }); + ), comment) + done() + }) + }) } -}; +} -function removeIndexTokensFromExpectedTokens(index, expected){ +function removeIndexTokensFromExpectedTokens (index, expected) { for (var pos in index) { - if (!_.isArray(expected[pos])) { continue; } - expected[pos] = expected[pos].filter(token => !index[pos].includes(token)); - if (_.isEmpty(expected[pos])) { delete expected[pos]; } + if (!_.isArray(expected[pos])) { continue } + expected[pos] = expected[pos].filter(token => !index[pos].includes(token)) + if (_.isEmpty(expected[pos])) { delete expected[pos] } } - return expected; + return expected } var tests = [ @@ -111,8 +111,8 @@ var tests = [ require('./autocomplete_directional_synonym_expansion.js'), require('./autocomplete_abbreviated_street_names.js'), require('./multi_token_synonyms.js') -]; +] -tests.map(function(t) { - t.all(tape, common); -}); +tests.map(function (t) { + t.all(tape, common) +}) diff --git a/integration/source_layer_sourceid_filtering.js b/integration/source_layer_sourceid_filtering.js index 8fb17238..ab4c2fca 100644 --- a/integration/source_layer_sourceid_filtering.js +++ b/integration/source_layer_sourceid_filtering.js @@ -1,48 +1,54 @@ // validate analyzer is behaving as expected +const elastictest = require('elastictest') +const config = require('pelias-config').generate() +const getTotalHits = require('./_hits_total_helper') -const elastictest = require('elastictest'); -const config = require('pelias-config').generate(); -const getTotalHits = require('./_hits_total_helper'); +module.exports.tests = {} -module.exports.tests = {}; - -module.exports.tests.source_filter = function(test, common){ - test( 'source filter', function(t){ - - var suite = new elastictest.Suite( common.clientOpts, common.create ); - suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up +module.exports.tests.source_filter = function (test, common) { + test('source filter', function (t) { + var suite = new elastictest.Suite(common.clientOpts, common.create) + suite.action(function (done) { setTimeout(done, 500) }) // wait for es to bring some shards up // index some docs - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ - index: suite.props.index, type: config.schema.typeName, - id: '1', body: { source: 'osm', layer: 'node', source_id: 'dataset/1' } - }, done ); - }); + index: suite.props.index, + type: config.schema.typeName, + id: '1', + body: { source: 'osm', layer: 'node', source_id: 'dataset/1' } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ - index: suite.props.index, type: config.schema.typeName, - id: '2', body: { source: 'osm', layer: 'address', source_id: 'dataset/2' } - }, done ); - }); + index: suite.props.index, + type: config.schema.typeName, + id: '2', + body: { source: 'osm', layer: 'address', source_id: 'dataset/2' } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ - index: suite.props.index, type: config.schema.typeName, - id: '3', body: { source: 'geonames', layer: 'address', source_id: 'dataset/1' } - }, done ); - }); + index: suite.props.index, + type: config.schema.typeName, + id: '3', + body: { source: 'geonames', layer: 'address', source_id: 'dataset/1' } + }, done) + }) - suite.action( function( done ){ + suite.action(function (done) { suite.client.index({ - index: suite.props.index, type: config.schema.typeName, - id: '4', body: { source: 'foo bar baz' } - }, done ); - }); + index: suite.props.index, + type: config.schema.typeName, + id: '4', + body: { source: 'foo bar baz' } + }, done) + }) // find all 'osm' sources - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -50,15 +56,16 @@ module.exports.tests.source_filter = function(test, common){ term: { source: 'osm' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); - done(); - }); - }); + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 2) + done() + }) + }) // find all 'address' layers - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -66,15 +73,16 @@ module.exports.tests.source_filter = function(test, common){ term: { layer: 'address' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); - done(); - }); - }); + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 2) + done() + }) + }) // find all 'shop' source_ids - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -82,30 +90,32 @@ module.exports.tests.source_filter = function(test, common){ term: { source_id: 'dataset/1' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 2 ); - done(); - }); - }); + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 2) + done() + }) + }) // find all 'shop' source_ids from 'osm' source - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, body: { query: { bool: { must: [ { term: { source: 'osm' } }, { term: { source_id: 'dataset/1' } } - ]}}} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 1 ); - done(); - }); - }); + ] } } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 1) + done() + }) + }) // case sensitive - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -113,15 +123,16 @@ module.exports.tests.source_filter = function(test, common){ term: { source: 'OSM' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 0 ); - done(); - }); - }); + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 0) + done() + }) + }) // keyword analysis - no partial matching - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -129,15 +140,16 @@ module.exports.tests.source_filter = function(test, common){ term: { source: 'foo' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 0 ); - done(); - }); - }); + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 0) + done() + }) + }) // keyword analysis - allows spaces - suite.assert( function( done ){ + suite.assert(function (done) { suite.client.search({ index: suite.props.index, type: config.schema.typeName, @@ -145,24 +157,24 @@ module.exports.tests.source_filter = function(test, common){ term: { source: 'foo bar baz' } - }} - }, function( err, res ){ - t.equal( getTotalHits(res.hits), 1 ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + } } + }, function (err, res) { + t.false(err) + t.equal(getTotalHits(res.hits), 1) + done() + }) + }) + + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('field filtering: ' + name, testFunction); + function test (name, testFunction) { + return tape('field filtering: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/integration/validate.js b/integration/validate.js index ee35640e..409a731d 100644 --- a/integration/validate.js +++ b/integration/validate.js @@ -1,33 +1,31 @@ // simply validate that the schema doesn't error when inserted in to // your local elasticsearch server, useful to sanity check version upgrades. +const elastictest = require('elastictest') -const elastictest = require('elastictest'); +module.exports.tests = {} -module.exports.tests = {}; +module.exports.tests.validate = function (test, common) { + test('schema', t => { + var suite = new elastictest.Suite(common.clientOpts, common.create) -module.exports.tests.validate = function(test, common){ - test( 'schema', t => { + suite.assert(done => { + suite.client.info({}, (err, res, status) => { + t.false(err) + t.equal(status, 200) + done() + }) + }) - var suite = new elastictest.Suite( common.clientOpts, common.create ); - - suite.assert( done => { - suite.client.info({}, ( err, res, status ) => { - t.equal( status, 200 ); - done(); - }); - }); - - suite.run( t.end ); - }); -}; + suite.run(t.end) + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('validate: ' + name, testFunction); + function test (name, testFunction) { + return tape('validate: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/mappings/document.js b/mappings/document.js index 402c92da..2f0ff0f2 100644 --- a/mappings/document.js +++ b/mappings/document.js @@ -1,16 +1,16 @@ -const admin = require('./partial/admin'); -const postalcode = require('./partial/postalcode'); -const hash = require('./partial/hash'); -const multiplier = require('./partial/multiplier'); -const keyword = require('./partial/keyword'); -const keyword_with_doc_values = require('./partial/keyword_with_doc_values'); +const admin = require('./partial/admin') +const postalcode = require('./partial/postalcode') +const hash = require('./partial/hash') +const multiplier = require('./partial/multiplier') +const keyword = require('./partial/keyword') +const keywordWithDocValues = require('./partial/keyword_with_doc_values') var schema = { properties: { // data partitioning - source: keyword_with_doc_values, - layer: keyword_with_doc_values, + source: keywordWithDocValues, + layer: keywordWithDocValues, // place name (ngram analysis) name: hash, @@ -58,7 +58,7 @@ var schema = { analyzer: 'peliasZip', search_analyzer: 'peliasZip', similarity: 'peliasDefaultSimilarity' - }, + } } }, @@ -168,8 +168,8 @@ var schema = { search_analyzer: 'peliasQuery', similarity: 'peliasDefaultSimilarity' } - }, - },{ + } + }, { phrase: { path_match: 'phrase.*', match_mapping_type: 'string', @@ -180,7 +180,7 @@ var schema = { similarity: 'peliasDefaultSimilarity' } } - },{ + }, { addendum: { path_match: 'addendum.*', match_mapping_type: 'string', @@ -192,9 +192,9 @@ var schema = { } }], _source: { - excludes : ['shape','phrase'] + excludes: ['shape', 'phrase'] }, dynamic: 'strict' -}; +} -module.exports = schema; +module.exports = schema diff --git a/package.json b/package.json index 1a749ec5..1626e06b 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,10 @@ "drop_index": "node scripts/drop_index", "reset_type": "node scripts/reset_type", "update_settings": "node scripts/update_settings", - "travis": "npm run test && npm run integration" + "travis": "npm run test && npm run integration", + "format_check": "./node_modules/standard/bin/cmd.js", + "format_fix": "./node_modules/standard/bin/cmd.js --fix", + "validate": "npm ls" }, "repository": { "type": "git", @@ -43,10 +46,13 @@ "difflet": "^1.0.1", "elastictest": "^3.0.0", "proxyquire": "^2.0.0", + "standard": "^14.3.4", "tap-spec": "^5.0.0", "tape": "^4.5.0" }, - "pre-commit": [], + "pre-commit": [ + "format_check" + ], "release": { "branch": "master", "success": [] diff --git a/punctuation.js b/punctuation.js index 0c9fd970..67823087 100644 --- a/punctuation.js +++ b/punctuation.js @@ -2,22 +2,22 @@ // @see: org/apache/lucene/analysis/cn/smart/stopwords.txt module.exports.all = [ - ".","`","‘","-","_","=","?","'","|","\"","(",")","{","}","[","]","<",">","*", - "#","&","^","$","@","!","~",":",";","+","《","》","—","-",",","。", - "、", ":",";","!","·","?","“","”",")","(","【","】","[","]","●" -]; + '.', '`', '‘', '-', '_', '=', '?', "'", '|', '"', '(', ')', '{', '}', '[', ']', '<', '>', '*', + '#', '&', '^', '$', '@', '!', '~', ':', ';', '+', '《', '》', '—', '-', ',', '。', + '、', ':', ';', '!', '·', '?', '“', '”', ')', '(', '【', '】', '[', ']', '●' +] module.exports.allowed = [ - "-", // allow hypens - "&" // allow ampersands -]; + '-', // allow hypens + '&' // allow ampersands +] -module.exports.blacklist = module.exports.all.slice(); +module.exports.blacklist = module.exports.all.slice() // remove alowed chars from blacklist -module.exports.allowed.forEach(function(item){ - var index = module.exports.blacklist.indexOf(item); - if( index > -1 ){ - module.exports.blacklist.splice(index, 1); +module.exports.allowed.forEach(function (item) { + var index = module.exports.blacklist.indexOf(item) + if (index > -1) { + module.exports.blacklist.splice(index, 1) } -}); +}) diff --git a/schema.js b/schema.js index 5ff23128..3aeffd72 100644 --- a/schema.js +++ b/schema.js @@ -1,6 +1,6 @@ const schema = { settings: require('./settings')(), - mappings: require('./mappings/document'), -}; + mappings: require('./mappings/document') +} -module.exports = schema; +module.exports = schema diff --git a/scripts/check_plugins.js b/scripts/check_plugins.js index 6ab738cf..84695993 100644 --- a/scripts/check_plugins.js +++ b/scripts/check_plugins.js @@ -1,87 +1,84 @@ -const colors = require('colors/safe'); -const config = require('pelias-config').generate(); -const es = require('elasticsearch'); -const client = new es.Client(config.esclient); -const cli = require('./cli'); +const colors = require('colors/safe') +const config = require('pelias-config').generate() +const es = require('elasticsearch') +const client = new es.Client(config.esclient) +const cli = require('./cli') // mandatory plugins -const required = ['analysis-icu']; +const required = ['analysis-icu'] // list of failures -let failures = []; +let failures = [] // returns the appropriate plugin name for the configured Elasticsearch version -function elasticsearchPluginUtility() { +function elasticsearchPluginUtility () { if (config.esclient.apiVersion === '2.4') { - return 'plugin'; + return 'plugin' } else { - return 'elasticsearch-plugin'; + return 'elasticsearch-plugin' } } -cli.header("checking elasticsearch plugins"); +cli.header('checking elasticsearch plugins') client.nodes.info(null, (err, res) => { - - if( err ){ - console.error(err); - process.exit(1); + if (err) { + console.error(err) + process.exit(1) } - if( !res || !res.nodes ){ - console.error("no nodes found"); - process.exit(1); + if (!res || !res.nodes) { + console.error('no nodes found') + process.exit(1) } // iterate over all nodes in cluster - for( const uid in res.nodes ){ - - const node = res.nodes[uid]; + for (const uid in res.nodes) { + const node = res.nodes[uid] // Amazon's hosted Elasticsearch does not have the plugins property // but has the plugins we need if (!node.plugins) { - continue; + continue } - console.log( colors.bold(`node '${node.name}' [${uid}]`) ); + console.log(colors.bold(`node '${node.name}' [${uid}]`)) // per node failures - let node_failures = []; + let nodeFailures = [] // iterate over all required plugins required.forEach(plugin => { - // bool, is the plugin currently installed? - const isInstalled = node.plugins.some(installed => installed.name === plugin); + const isInstalled = node.plugins.some(installed => installed.name === plugin) // output status to terminal - console.log( ` checking plugin '${plugin}': ${isInstalled ? cli.status.success : cli.status.failure}` ); + console.log(` checking plugin '${plugin}': ${isInstalled ? cli.status.success : cli.status.failure}`) // record this plugin as not installed yet - if( !isInstalled ){ - node_failures.push(plugin); + if (!isInstalled) { + nodeFailures.push(plugin) } - }); + }) // node had at least one failure - if( node_failures.length ){ - failures.push({ node: node, plugins: node_failures }); + if (nodeFailures.length) { + failures.push({ node: node, plugins: nodeFailures }) } } // pretty print error message - if( failures.length ){ - console.error( colors.red(`${failures.length} required plugin(s) are not installed on the node(s) shown above.` ) ); - console.error( "you must install the plugins before continuing with the installation."); + if (failures.length) { + console.error(colors.red(`${failures.length} required plugin(s) are not installed on the node(s) shown above.`)) + console.error('you must install the plugins before continuing with the installation.') failures.forEach(failure => { - console.error( `\nyou can install the missing packages on '${failure.node.name}' [${failure.node.ip}] with the following command(s):\n` ); + console.error(`\nyou can install the missing packages on '${failure.node.name}' [${failure.node.ip}] with the following command(s):\n`) failure.plugins.forEach(plugin => { - console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/${elasticsearchPluginUtility()} install ${plugin}`) ); - }); - }); - console.error( colors.white("\nnote:") + "some plugins may require you to restart elasticsearch.\n"); + console.error(colors.green(`sudo ${failure.node.settings.path.home}/bin/${elasticsearchPluginUtility()} install ${plugin}`)) + }) + }) + console.error(colors.white('\nnote:') + 'some plugins may require you to restart elasticsearch.\n') process.exit(1) } - console.log(); -}); + console.log() +}) diff --git a/scripts/check_version.js b/scripts/check_version.js index 2b7d4155..794e8f4d 100644 --- a/scripts/check_version.js +++ b/scripts/check_version.js @@ -1,33 +1,32 @@ -const _ = require('lodash'); -const semver = require('semver'); -const es = require('elasticsearch'); -const config = require('pelias-config').generate(); -const client = new es.Client(config.esclient); -const cli = require('./cli'); +const _ = require('lodash') +const semver = require('semver') +const es = require('elasticsearch') +const config = require('pelias-config').generate() +const client = new es.Client(config.esclient) +const cli = require('./cli') // pass target elastic version semver as the first CLI arg -const targetVersion = process.argv[2]; -if(!targetVersion){ - console.error('you must pass a target elasticsearch version semver as the first argument'); - process.exit(1); +const targetVersion = process.argv[2] +if (!targetVersion) { + console.error('you must pass a target elasticsearch version semver as the first argument') + process.exit(1) } -cli.header(`checking elasticsearch server version matches "${targetVersion}"`); +cli.header(`checking elasticsearch server version matches "${targetVersion}"`) client.info(null, (err, res) => { - if (err) { - console.error(err); - process.exit(1); + console.error(err) + process.exit(1) } - const version = _.get(res, 'version.number', '0.0.0'); + const version = _.get(res, 'version.number', '0.0.0') // pretty print error message if (!semver.satisfies(version, targetVersion)) { - console.log(`${cli.status.failure} ${version}\n`); + console.log(`${cli.status.failure} ${version}\n`) process.exit(1) } - console.log(`${cli.status.success} ${version}\n`); - console.log(); -}); + console.log(`${cli.status.success} ${version}\n`) + console.log() +}) diff --git a/scripts/cli.js b/scripts/cli.js index fb0d7b56..b6c8931a 100644 --- a/scripts/cli.js +++ b/scripts/cli.js @@ -1,10 +1,10 @@ +/* eslint no-octal-escape: "error" */ +const util = require('util') +const colors = require('colors/safe') -const util = require('util'); -const colors = require('colors/safe'); - -module.exports.header = function( text ){ - var rule = new Array( text.length + 3 ).join("-"); - console.log( util.format("\n\033[0;33m%s\n %s \n%s\033[0m\n", rule, text, rule ) ); +module.exports.header = function (text) { + var rule = new Array(text.length + 3).join('-') + console.log(util.format('\n\x1b[0;33m%s\n %s \n%s\x1b[0m\n', rule, text, rule)) } module.exports.status = { diff --git a/scripts/create_index.js b/scripts/create_index.js index 0db08b5f..7dfb08c2 100644 --- a/scripts/create_index.js +++ b/scripts/create_index.js @@ -1,43 +1,43 @@ -const child_process = require('child_process'); -const config = require('pelias-config').generate(); -const es = require('elasticsearch'); -const SUPPORTED_ES_VERSIONS = '>=6.5.4 || >=7.4.2'; +const child = require('child_process') +const config = require('pelias-config').generate() +const es = require('elasticsearch') +const SUPPORTED_ES_VERSIONS = '>=6.5.4 || >=7.4.2' -const cli = require('./cli'); -const schema = require('../schema'); +const cli = require('./cli') +const schema = require('../schema') -cli.header("create index"); +cli.header('create index') -const client = new es.Client(config.esclient); +const client = new es.Client(config.esclient) // check minimum elasticsearch versions before continuing try { - child_process.execSync(`node ${__dirname}/check_version.js "${SUPPORTED_ES_VERSIONS}"`); + child.execSync(`node ${__dirname}/check_version.js "${SUPPORTED_ES_VERSIONS}"`) } catch (e) { - console.error(`unsupported elasticsearch version. try: ${SUPPORTED_ES_VERSIONS}\n`); - process.exit(1); + console.error(`unsupported elasticsearch version. try: ${SUPPORTED_ES_VERSIONS}\n`) + process.exit(1) } // check mandatory plugins are installed before continuing try { - child_process.execSync(`node ${__dirname}/check_plugins.js`); + child.execSync(`node ${__dirname}/check_plugins.js`) } catch (e) { - console.error("please install mandatory plugins before continuing.\n"); - process.exit(1); + console.error('please install mandatory plugins before continuing.\n') + process.exit(1) } -const indexName = config.schema.indexName; +const indexName = config.schema.indexName const req = { index: indexName, body: schema, include_type_name: false -}; +} client.indices.create(req, (err, res) => { if (err) { - console.error(err.message || err, '\n'); - process.exit(1); + console.error(err.message || err, '\n') + process.exit(1) } - console.log('[put mapping]', '\t', indexName, res, '\n'); - process.exit(!!err); -}); + console.log('[put mapping]', '\t', indexName, res, '\n') + process.exit(!!err) +}) diff --git a/scripts/drop_index.js b/scripts/drop_index.js index 100ad2ec..2ae6e689 100644 --- a/scripts/drop_index.js +++ b/scripts/drop_index.js @@ -1,42 +1,41 @@ -const colors = require('colors/safe'); -const config = require('pelias-config').generate(); -const es = require('elasticsearch'); -const client = new es.Client(config.esclient); -const readline = require('readline'); -const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); +const colors = require('colors/safe') +const config = require('pelias-config').generate() +const es = require('elasticsearch') +const client = new es.Client(config.esclient) +const readline = require('readline') +const rl = readline.createInterface({ input: process.stdin, output: process.stdout }) // use -f or --force-yes to skip the prompt -if (isForced()) { drop(); } -else { prompt(drop, fail); } +if (isForced()) { drop() } else { prompt(drop, fail) } -function drop() { - const req = { index: config.schema.indexName }; +function drop () { + const req = { index: config.schema.indexName } client.indices.delete(req, (err, res) => { - console.log('\n[delete mapping]', '\t', config.schema.indexName, err || '\t', res); - process.exit(!!err); - }); + console.log('\n[delete mapping]', '\t', config.schema.indexName, err || '\t', res) + process.exit(!!err) + }) } // check all hosts to see if any is not localhost -function warnIfNotLocal() { - if (config.esclient.hosts.some((env) => { return env.host !== 'localhost'; })) { - console.log(colors.red(`WARNING: DROPPING SCHEMA NOT ON LOCALHOST: ${config.esclient.hosts[0].host}`)); +function warnIfNotLocal () { + if (config.esclient.hosts.some((env) => { return env.host !== 'localhost' })) { + console.log(colors.red(`WARNING: DROPPING SCHEMA NOT ON LOCALHOST: ${config.esclient.hosts[0].host}`)) } } -function prompt(yes, no) { - warnIfNotLocal(); +function prompt (yes, no) { + warnIfNotLocal() rl.question(`Are you sure you want to drop the ${config.schema.indexName} index and delete ALL records? `, answer => { - if (!answer.match(/^y(es)?$/i)) { return no(); } - return yes(); - }); + if (!answer.match(/^y(es)?$/i)) { return no() } + return yes() + }) } -function fail() { - console.log(`you must answer 'y' to confirm. aborting.`); - process.exit(0); +function fail () { + console.log(`you must answer 'y' to confirm. aborting.`) + process.exit(0) } -function isForced() { - return process.argv.length > 2 && ['--force-yes', '-f'].indexOf(process.argv[2]) > -1; -} \ No newline at end of file +function isForced () { + return process.argv.length > 2 && ['--force-yes', '-f'].indexOf(process.argv[2]) > -1 +} diff --git a/scripts/info.js b/scripts/info.js index 373dbccf..de020129 100644 --- a/scripts/info.js +++ b/scripts/info.js @@ -1,5 +1,5 @@ -var config = require('pelias-config').generate().esclient; -var es = require('elasticsearch'); -var client = new es.Client(config); +var config = require('pelias-config').generate().esclient +var es = require('elasticsearch') +var client = new es.Client(config) -client.info( {}, console.log.bind(console) ); +client.info({}, console.log.bind(console)) diff --git a/scripts/list_analyzers.js b/scripts/list_analyzers.js index ee386f56..b98bf843 100644 --- a/scripts/list_analyzers.js +++ b/scripts/list_analyzers.js @@ -1,30 +1,29 @@ const _ = require('lodash') const colors = require('colors') -const cli = require('./cli'); -const config = require('pelias-config').generate(); -const schema = require('../schema'); -const DEFAULT_ANALYZER = 'standard'; -const NOT_APPLICABLE_ANALYZER = 'n/a'; -const DEFAULT_NORMALIZER = '{none}'; -const NOT_APPLICABLE_NORMALIZER = 'n/a'; +const cli = require('./cli') +const config = require('pelias-config').generate() +const schema = require('../schema') +const DEFAULT_ANALYZER = 'standard' +const NOT_APPLICABLE_ANALYZER = 'n/a' +const DEFAULT_NORMALIZER = '{none}' -const defaultAnalyzerFor = function(mapping){ - switch (mapping.type){ - case 'text': return DEFAULT_ANALYZER; - case 'string': return DEFAULT_ANALYZER; +const defaultAnalyzerFor = function (mapping) { + switch (mapping.type) { + case 'text': return DEFAULT_ANALYZER + case 'string': return DEFAULT_ANALYZER } - return NOT_APPLICABLE_ANALYZER; + return NOT_APPLICABLE_ANALYZER } const defaultNormalizerFor = function (mapping) { switch (mapping.type) { - case 'keyword': return DEFAULT_NORMALIZER; + case 'keyword': return DEFAULT_NORMALIZER } - return NOT_APPLICABLE_ANALYZER; + return NOT_APPLICABLE_ANALYZER } // pretty print a single analyzer/normalizer label -const pretty = function(analyzer){ +const pretty = function (analyzer) { if (analyzer === DEFAULT_ANALYZER) { return colors.blue(analyzer) } @@ -38,27 +37,27 @@ const pretty = function(analyzer){ } // pretty print a single line -const print = function(vals) { +const print = function (vals) { console.error( colors.brightBlue(vals.field.padEnd(32)), vals.type.padEnd(32), pretty(vals.analyzer).padEnd(35), pretty(vals.search_analyzer).padEnd(35), pretty(vals.normalizer).padEnd(35) - ); + ) } // pretty print an error -const error = function(vals) { - console.error(Object.values(vals).map(v => colors.red(v)).join(' | ')); +const error = function (vals) { + console.error(Object.values(vals).map(v => colors.red(v)).join(' | ')) } // parse mapping -const mapping = schema.mappings[config.schema.typeName]; -const dynamic = mapping.dynamic_templates.map(t => _.first(_.map(t, v => v))); +const mapping = schema.mappings[config.schema.typeName] +const dynamic = mapping.dynamic_templates.map(t => _.first(_.map(t, v => v))) // process and single mapping property (recursively) -const property = function(prop, field){ +const property = function (prop, field) { // properties with subfields if (prop.type === 'object') { // recurse the object properties @@ -71,7 +70,7 @@ const property = function(prop, field){ const matches = dynamic.filter(t => field.startsWith(t.path_match.replace('.*', ''))) // a dynamic template matches - if (matches.length === 1){ + if (matches.length === 1) { let prop = matches[0].mapping print({ field: matches[0].path_match, @@ -108,7 +107,7 @@ const property = function(prop, field){ } } -cli.header("list analyzers"); +cli.header('list analyzers') console.error( colors.bgWhite([ colors.black('field').padEnd(43), @@ -117,7 +116,7 @@ console.error( colors.black('search_analyzer').padEnd(36), colors.black('normalizer').padEnd(36) ].join('')) -); +) -_.each(mapping.properties, property); +_.each(mapping.properties, property) console.log() diff --git a/scripts/output_mapping.js b/scripts/output_mapping.js index c05da280..1f83feae 100644 --- a/scripts/output_mapping.js +++ b/scripts/output_mapping.js @@ -1,21 +1,18 @@ -var config = require('pelias-config').generate(); -var es = require('elasticsearch'); -var client = new es.Client(config.esclient); -var schema = require('../schema'); +var config = require('pelias-config').generate() +var schema = require('../schema') -var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName; -var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args +var _index = (process.argv.length > 3) ? process.argv[3] : config.schema.indexName +var _type = (process.argv.length > 2) ? process.argv[2] : null // get type from cli args // print out mapping for just one type -if ( _type ) { - var mapping = schema.mappings[_type]; - if( !mapping ){ - console.error( 'could not find a mapping in the schema file for', _index+'/'+_type ); - process.exit(1); +if (_type) { + var mapping = schema.mappings[_type] + if (!mapping) { + console.error('could not find a mapping in the schema file for', _index + '/' + _type) + process.exit(1) } - console.log( JSON.stringify( mapping, null, 2 ) ); -//print out the entire schema mapping + console.log(JSON.stringify(mapping, null, 2)) +// print out the entire schema mapping } else { - console.log( JSON.stringify( schema, null, 2 ) ); + console.log(JSON.stringify(schema, null, 2)) } - diff --git a/scripts/reset_type.js b/scripts/reset_type.js index 91485237..aac03b8c 100644 --- a/scripts/reset_type.js +++ b/scripts/reset_type.js @@ -1,26 +1,26 @@ -var config = require('pelias-config').generate().esclient; -var es = require('elasticsearch'); -var client = new es.Client(config); -var schema = require('../schema'); +var config = require('pelias-config').generate().esclient +var es = require('elasticsearch') +var client = new es.Client(config) +var schema = require('../schema') -var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName; -var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args +var _index = (process.argv.length > 3) ? process.argv[3] : config.schema.indexName +var _type = (process.argv.length > 2) ? process.argv[2] : null // get type from cli args -if( !_type ){ - console.error( 'you must provide the target es \'type\' as the first cli argument' ); - process.exit(1); +if (!_type) { + console.error('you must provide the target es \'type\' as the first cli argument') + process.exit(1) } -var mapping = schema.mappings[_type]; -if( !mapping ){ - console.error( 'could not find a mapping in the schema file for', _index+'/'+_type ); - process.exit(1); +var mapping = schema.mappings[_type] +if (!mapping) { + console.error('could not find a mapping in the schema file for', _index + '/' + _type) + process.exit(1) } -client.indices.deleteMapping( { index: _index, type: _type }, function( err, res ){ - console.log( '[delete mapping]', '\t', _index+'/'+_type, err || '\t', res ); - client.indices.putMapping( { index: _index, type: _type, body:mapping }, function( err, res ){ - console.log( '[put mapping]', '\t\t', _index+'/'+_type, err || '\t', res ); - process.exit( !!err ); - }); -}); +client.indices.deleteMapping({ index: _index, type: _type }, function (err, res) { + console.log('[delete mapping]', '\t', _index + '/' + _type, err || '\t', res) + client.indices.putMapping({ index: _index, type: _type, body: mapping }, function (err, res) { + console.log('[put mapping]', '\t\t', _index + '/' + _type, err || '\t', res) + process.exit(!!err) + }) +}) diff --git a/scripts/update_settings.js b/scripts/update_settings.js index 22d7b9c5..cd56e23b 100644 --- a/scripts/update_settings.js +++ b/scripts/update_settings.js @@ -1,24 +1,24 @@ -var config = require('pelias-config').generate(); -var es = require('elasticsearch'); -var client = new es.Client(config.esclient); -var schema = require('../schema'); +var config = require('pelias-config').generate() +var es = require('elasticsearch') +var client = new es.Client(config.esclient) +var schema = require('../schema') -var _index = config.schema.indexName; +var _index = config.schema.indexName // Error: ElasticsearchIllegalArgumentException[can't change the number of shards for an index -if( schema.settings.hasOwnProperty('index') && - schema.settings.index.hasOwnProperty('number_of_shards') ){ - delete schema.settings.index.number_of_shards; - delete schema.settings.index.number_of_replicas; +if (schema.settings.hasOwnProperty('index') && + schema.settings.index.hasOwnProperty('number_of_shards')) { + delete schema.settings.index.number_of_shards + delete schema.settings.index.number_of_replicas } -client.indices.close( { index: _index }, function( err, res ){ - console.log( '[close index]', '\t', _index, err || '\t', res ); - client.indices.putSettings( { index: _index, body: schema.settings }, function( err, res ){ - console.log( '[put settings]', '\t', _index, err || '\t', res ); - client.indices.open( { index: _index }, function( err, res ){ - console.log( '[open index]', '\t', _index, err || '\t', res ); - process.exit( !!err ); - }); - }); -}); +client.indices.close({ index: _index }, function (err, res) { + console.log('[close index]', '\t', _index, err || '\t', res) + client.indices.putSettings({ index: _index, body: schema.settings }, function (err, res) { + console.log('[put settings]', '\t', _index, err || '\t', res) + client.indices.open({ index: _index }, function (err, res) { + console.log('[open index]', '\t', _index, err || '\t', res) + process.exit(!!err) + }) + }) +}) diff --git a/settings.js b/settings.js index dca59cd3..5caf6cc9 100644 --- a/settings.js +++ b/settings.js @@ -1,243 +1,243 @@ -const _ = require('lodash'); -const peliasConfig = require('pelias-config'); -const punctuation = require('./punctuation'); -const synonyms = require('./synonyms/loader').load(); +const _ = require('lodash') +const peliasConfig = require('pelias-config') +const punctuation = require('./punctuation') +const synonyms = require('./synonyms/loader').load() -require('./configValidation').validate(peliasConfig.generate()); +require('./configValidation').validate(peliasConfig.generate()) -function generate(){ - var config = peliasConfig.generate(); +function generate () { + var config = peliasConfig.generate() // Default settings var settings = { - "index": { - "similarity": { - "peliasDefaultSimilarity": { - "type": "BM25", - "k1": 1.2, - "b": 0.75 + 'index': { + 'similarity': { + 'peliasDefaultSimilarity': { + 'type': 'BM25', + 'k1': 1.2, + 'b': 0.75 } } }, - "analysis": { - "tokenizer": { - "peliasTokenizer": { - "type": "pattern", - "pattern": "[\\s,/\\\\-]+" + 'analysis': { + 'tokenizer': { + 'peliasTokenizer': { + 'type': 'pattern', + 'pattern': '[\\s,/\\\\-]+' } }, - "analyzer": { - "peliasAdmin": { - "type": "custom", - "tokenizer": "peliasTokenizer", - "char_filter" : ["punctuation", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "admin_synonyms_multiplexer", - "icu_folding", - "word_delimiter", - "unique_only_same_position", - "notnull", - "flatten_graph" + 'analyzer': { + 'peliasAdmin': { + 'type': 'custom', + 'tokenizer': 'peliasTokenizer', + 'char_filter': ['punctuation', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'admin_synonyms_multiplexer', + 'icu_folding', + 'word_delimiter', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' ] }, - "peliasIndexOneEdgeGram" : { - "type": "custom", - "tokenizer" : "peliasTokenizer", - "char_filter" : ["punctuation", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "name_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "removeAllZeroNumericPrefix", - "peliasOneEdgeGramFilter", - "unique_only_same_position", - "notnull", - "flatten_graph" + 'peliasIndexOneEdgeGram': { + 'type': 'custom', + 'tokenizer': 'peliasTokenizer', + 'char_filter': ['punctuation', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'name_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'removeAllZeroNumericPrefix', + 'peliasOneEdgeGramFilter', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' ] }, - "peliasQuery": { - "type": "custom", - "tokenizer": "peliasTokenizer", - "char_filter": ["punctuation", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "icu_folding", - "remove_ordinals", - "removeAllZeroNumericPrefix", - "unique_only_same_position", - "notnull" + 'peliasQuery': { + 'type': 'custom', + 'tokenizer': 'peliasTokenizer', + 'char_filter': ['punctuation', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'icu_folding', + 'remove_ordinals', + 'removeAllZeroNumericPrefix', + 'unique_only_same_position', + 'notnull' ] }, - "peliasPhrase": { - "type": "custom", - "tokenizer":"peliasTokenizer", - "char_filter" : ["punctuation", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "remove_duplicate_spaces", - "name_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "unique_only_same_position", - "notnull", - "flatten_graph" + 'peliasPhrase': { + 'type': 'custom', + 'tokenizer': 'peliasTokenizer', + 'char_filter': ['punctuation', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'remove_duplicate_spaces', + 'name_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' ] }, - "peliasZip": { - "type": "custom", - "tokenizer":"keyword", - "char_filter": ["alphanumeric", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "icu_folding", - "unique_only_same_position", - "notnull" + 'peliasZip': { + 'type': 'custom', + 'tokenizer': 'keyword', + 'char_filter': ['alphanumeric', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'icu_folding', + 'unique_only_same_position', + 'notnull' ] }, - "peliasUnit": { - "type": "custom", - "tokenizer":"keyword", - "char_filter": ["alphanumeric", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "icu_folding", - "unique_only_same_position", - "notnull" + 'peliasUnit': { + 'type': 'custom', + 'tokenizer': 'keyword', + 'char_filter': ['alphanumeric', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'icu_folding', + 'unique_only_same_position', + 'notnull' ] }, - "peliasHousenumber": { - "type": "custom", - "tokenizer":"standard", - "char_filter" : ["numeric"] - }, - "peliasStreet": { - "type": "custom", - "tokenizer":"peliasTokenizer", - "char_filter" : ["punctuation", "nfkc_normalizer"], - "filter": [ - "lowercase", - "trim", - "remove_duplicate_spaces", - "street_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "trim", - "unique_only_same_position", - "notnull", - "flatten_graph" + 'peliasHousenumber': { + 'type': 'custom', + 'tokenizer': 'standard', + 'char_filter': ['numeric'] + }, + 'peliasStreet': { + 'type': 'custom', + 'tokenizer': 'peliasTokenizer', + 'char_filter': ['punctuation', 'nfkc_normalizer'], + 'filter': [ + 'lowercase', + 'trim', + 'remove_duplicate_spaces', + 'street_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'trim', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' ] } }, - "filter" : { - "street_synonyms_multiplexer": { - "type": "multiplexer", - "preserve_original": false, - "filters": [ - "synonyms/custom_street", - "synonyms/personal_titles", - "synonyms/streets", - "synonyms/directionals" + 'filter': { + 'street_synonyms_multiplexer': { + 'type': 'multiplexer', + 'preserve_original': false, + 'filters': [ + 'synonyms/custom_street', + 'synonyms/personal_titles', + 'synonyms/streets', + 'synonyms/directionals' ] }, - "name_synonyms_multiplexer": { - "type": "multiplexer", - "preserve_original": false, - "filters": [ - "synonyms/custom_name", - "synonyms/personal_titles", - "synonyms/place_names", - "synonyms/streets", - "synonyms/directionals", - "synonyms/punctuation" + 'name_synonyms_multiplexer': { + 'type': 'multiplexer', + 'preserve_original': false, + 'filters': [ + 'synonyms/custom_name', + 'synonyms/personal_titles', + 'synonyms/place_names', + 'synonyms/streets', + 'synonyms/directionals', + 'synonyms/punctuation' ] }, - "admin_synonyms_multiplexer": { - "type": "multiplexer", - "preserve_original": false, - "filters": [ - "synonyms/custom_admin", - "synonyms/personal_titles", - "synonyms/place_names" + 'admin_synonyms_multiplexer': { + 'type': 'multiplexer', + 'preserve_original': false, + 'filters': [ + 'synonyms/custom_admin', + 'synonyms/personal_titles', + 'synonyms/place_names' ] }, - "notnull" :{ - "type" : "length", - "min" : 1 - }, - "unique_only_same_position": { - "type": "unique", - "only_on_same_position": "true" - }, - "peliasOneEdgeGramFilter": { - "type" : "edgeNGram", - "min_gram" : 1, - "max_gram" : 24 - }, - "removeAllZeroNumericPrefix" :{ - "type" : "pattern_replace", - "pattern" : "^(0*)", - "replacement" : "" - }, - "remove_ordinals" : { - "type" : "pattern_replace", - "pattern": "(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))", - "replacement": "$2$4$5$6$7$9$10$12$14$15$16$17$18" - }, - "remove_duplicate_spaces" : { - "type" : "pattern_replace", - "pattern": " +", - "replacement": " " - }, + 'notnull': { + 'type': 'length', + 'min': 1 + }, + 'unique_only_same_position': { + 'type': 'unique', + 'only_on_same_position': 'true' + }, + 'peliasOneEdgeGramFilter': { + 'type': 'edgeNGram', + 'min_gram': 1, + 'max_gram': 24 + }, + 'removeAllZeroNumericPrefix': { + 'type': 'pattern_replace', + 'pattern': '^(0*)', + 'replacement': '' + }, + 'remove_ordinals': { + 'type': 'pattern_replace', + 'pattern': '(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))', + 'replacement': '$2$4$5$6$7$9$10$12$14$15$16$17$18' + }, + 'remove_duplicate_spaces': { + 'type': 'pattern_replace', + 'pattern': ' +', + 'replacement': ' ' + } // more generated below }, - "char_filter": { - "punctuation" : { - "type" : "mapping", - "mappings" : punctuation.blacklist.map(function(c){ - return c + '=>'; + 'char_filter': { + 'punctuation': { + 'type': 'mapping', + 'mappings': punctuation.blacklist.map(function (c) { + return c + '=>' }) }, - "alphanumeric" : { - "type" : "pattern_replace", - "pattern": "[^a-zA-Z0-9]", - "replacement": "" + 'alphanumeric': { + 'type': 'pattern_replace', + 'pattern': '[^a-zA-Z0-9]', + 'replacement': '' }, - "numeric" : { - "type" : "pattern_replace", - "pattern": "[^0-9]", - "replacement": " " + 'numeric': { + 'type': 'pattern_replace', + 'pattern': '[^0-9]', + 'replacement': ' ' }, - "nfkc_normalizer": { - "type": "icu_normalizer", - "name": "nfkc", - "mode": "compose" + 'nfkc_normalizer': { + 'type': 'icu_normalizer', + 'name': 'nfkc', + 'mode': 'compose' } } } - }; + } // dynamically create filters for all synonym files in the ./synonyms directory. // each filter is given the same name as the file, paths separators are replaced with // underscores and the file extension is removed. _.each(synonyms, (synonym, name) => { settings.analysis.filter[`synonyms/${name}`] = { - "type": "synonym", - "synonyms": !_.isEmpty(synonym) ? synonym : [''] - }; - }); + 'type': 'synonym', + 'synonyms': !_.isEmpty(synonym) ? synonym : [''] + } + }) // Merge settings from pelias/config - settings = _.merge({}, settings, _.get(config, 'elasticsearch.settings', {})); + settings = _.merge({}, settings, _.get(config, 'elasticsearch.settings', {})) - return settings; + return settings } -module.exports = generate; +module.exports = generate diff --git a/synonyms/linter.js b/synonyms/linter.js index f4417899..ad519ee4 100644 --- a/synonyms/linter.js +++ b/synonyms/linter.js @@ -1,6 +1,6 @@ -const _ = require('lodash'); -const logger = require('pelias-logger').get('schema-synonyms'); -const punctuation = require('../punctuation'); +const _ = require('lodash') +const logger = require('pelias-logger').get('schema-synonyms') +const punctuation = require('../punctuation') /** * The synonyms linter attempts to warn the user when making @@ -13,73 +13,73 @@ const punctuation = require('../punctuation'); * - Multi Word: Multi-word synonyms can generate unexpected token positions */ -function linter(synonyms) { +function linter (synonyms) { _.each(synonyms, (lines, filename) => { - logger.debug(`[lint] ${filename}`); + logger.debug(`[lint] ${filename}`) lines.forEach((line, idx) => { - const logprefix = `[${filename} line ${idx+1}]`; - logger.debug(`[line] ${line}`); + const logprefix = `[${filename} line ${idx + 1}]` + logger.debug(`[line] ${line}`) // split the lines by delimeter - let tokens = line.split(/,|=>/g).map(t => t.trim()); + let tokens = line.split(/,|=>/g).map(t => t.trim()) // strip blacklisted punctuation from synonyms // the 'punctuation.blacklist' contains a list of characters which are // stripped from the tokens before indexing. tokens = _.map(tokens, token => { punctuation.blacklist.forEach(char => { - let replacement = token.split(char).join(''); - if(replacement.length != token.length){ - logger.warn(`${logprefix} punctunation removed: ${token} --> ${replacement}`); + let replacement = token.split(char).join('') + if (replacement.length !== token.length) { + logger.warn(`${logprefix} punctunation removed: ${token} --> ${replacement}`) } - token = replacement; - }); + token = replacement + }) return token - }); + }) - letterCasing(line, logprefix, tokens); - tokensSanityCheck(line, logprefix, tokens); - multiWordCheck(line, logprefix, tokens); + letterCasing(line, logprefix, tokens) + tokensSanityCheck(line, logprefix, tokens) + multiWordCheck(line, logprefix, tokens) // tokenLengthCheck(line, logprefix, tokens); }) }) } -function letterCasing(line, logprefix){ +function letterCasing (line, logprefix) { if (line.toLowerCase() !== line) { - logger.warn(`${logprefix} should be lowercase:`, line); + logger.warn(`${logprefix} should be lowercase:`, line) } } -function tokensSanityCheck(line, logprefix, tokens) { - switch (tokens.length){ +function tokensSanityCheck (line, logprefix, tokens) { + switch (tokens.length) { case 0: - return logger.warn(`${logprefix} no tokens:`, line); + return logger.warn(`${logprefix} no tokens:`, line) case 1: - return logger.warn(`${logprefix} only one token:`, line); + return logger.warn(`${logprefix} only one token:`, line) default: - let dupes = _.filter(tokens, (val, i, t) => _.includes(t, val, i + 1)); - if (dupes.length){ - logger.warn(`${logprefix} duplicate tokens:`, dupes); + let dupes = _.filter(tokens, (val, i, t) => _.includes(t, val, i + 1)) + if (dupes.length) { + logger.warn(`${logprefix} duplicate tokens:`, dupes) } } } -function multiWordCheck(line, logprefix, tokens) { +function multiWordCheck (line, logprefix, tokens) { _.each(tokens, token => { - if (/\s/.test(token)){ - logger.warn(`${logprefix} multi word synonyms may cause issues with phrase queries:`, token); + if (/\s/.test(token)) { + logger.warn(`${logprefix} multi word synonyms may cause issues with phrase queries:`, token) } - }); + }) } -function tokenLengthCheck(line, logprefix, tokens) { - _.each(tokens, token => { - if (token.length <= 1) { - logger.warn(`${logprefix} short token:`, token); - } - }); -} +// function tokenLengthCheck (line, logprefix, tokens) { +// _.each(tokens, token => { +// if (token.length <= 1) { +// logger.warn(`${logprefix} short token:`, token) +// } +// }) +// } module.exports = linter diff --git a/synonyms/loader.js b/synonyms/loader.js index 19918a0b..f020402d 100644 --- a/synonyms/loader.js +++ b/synonyms/loader.js @@ -1,38 +1,37 @@ -const path = require('path'); -const glob = require('glob'); -const parse = require('./parser'); -const lint = require('./linter'); - -function load() { +const path = require('path') +const glob = require('glob') +const parse = require('./parser') +const lint = require('./linter') +function load () { // map containing all synonyms - const synonyms = {}; + const synonyms = {} // recursively find all files ending with .txt in this directory - const basepath = __dirname; - const pattern = path.join(basepath, '**', '*.txt'); - const files = glob.sync(pattern, { realpath: true }); + const basepath = __dirname + const pattern = path.join(basepath, '**', '*.txt') + const files = glob.sync(pattern, { realpath: true }) // load synonyms files and parse each files.forEach(filepath => { // for directories of synonyms we use the directory name as the key. // nested directories will have their path separators normalized to '/'. - let key = path.dirname(path.relative(basepath, filepath)).split(path.sep).join('/'); + let key = path.dirname(path.relative(basepath, filepath)).split(path.sep).join('/') // for synonym files at the root of the synonyms dir we use the basename as the key. if (key === '.') { - key = path.basename(filepath).replace('.txt', ''); + key = path.basename(filepath).replace('.txt', '') } - if (!synonyms.hasOwnProperty(key)) { synonyms[key] = []; } - synonyms[key] = synonyms[key].concat(parse(filepath)); - }); + if (!synonyms.hasOwnProperty(key)) { synonyms[key] = [] } + synonyms[key] = synonyms[key].concat(parse(filepath)) + }) // emit synonym warnings - lint(synonyms); + lint(synonyms) // return all synonyms - return synonyms; + return synonyms } -module.exports.load = load; +module.exports.load = load diff --git a/synonyms/parser.js b/synonyms/parser.js index abdc5256..53569639 100644 --- a/synonyms/parser.js +++ b/synonyms/parser.js @@ -1,34 +1,33 @@ -const fs = require('fs'); +const fs = require('fs') // https://www.elastic.co/guide/en/elasticsearch/reference/2.4/analysis-synonym-tokenfilter.html -function load( filename ){ - +function load (filename) { // path not specified / file does not exist try { - if( !fs.lstatSync(filename).isFile() ){ - throw new Error( 'invalid file' ); + if (!fs.lstatSync(filename).isFile()) { + throw new Error('invalid file') } - } catch(e){ - throw new Error( 'file not found' ); + } catch (e) { + throw new Error('file not found') } // parse solr synonyms format - return parse( fs.readFileSync( filename, 'utf8' ) ); + return parse(fs.readFileSync(filename, 'utf8')) } -function parse( contents ){ +function parse (contents) { return contents.split('\n') - .map( line => { - return line.trim().toLowerCase() // lowercase all tokens - .replace( /\s\s+/g, ' ' ) // squash double spaces - .replace(/(^,)|(,$)/g, '') // trim commas - .replace(/(\s*,\s*)/g,',') // trim spaces around commas - .replace(/(\s*=>\s*)/g,' => '); // trim spaces around arrows - }) - .filter( line => line.length > 0 ) // remove empty lines - .filter( line => '#' !== line[0] ); // remove comments + .map(line => { + return line.trim().toLowerCase() // lowercase all tokens + .replace(/\s\s+/g, ' ') // squash double spaces + .replace(/(^,)|(,$)/g, '') // trim commas + .replace(/(\s*,\s*)/g, ',') // trim spaces around commas + .replace(/(\s*=>\s*)/g, ' => ') // trim spaces around arrows + }) + .filter(line => line.length > 0) // remove empty lines + .filter(line => line[0] !== '#') // remove comments } -module.exports = load; -module.exports.parse = parse; +module.exports = load +module.exports.parse = parse diff --git a/test/compile.js b/test/compile.js index 719dde7a..0cd90cc7 100644 --- a/test/compile.js +++ b/test/compile.js @@ -1,114 +1,111 @@ -const _ = require('lodash'); -const path = require('path'); -const schema = require('../'); -const fixture = require('./fixtures/expected.json'); -const config = require('pelias-config').generate(); +const _ = require('lodash') +const path = require('path') +const schema = require('../') +const fixture = require('./fixtures/expected.json') const forEachDeep = (obj, cb) => _.forEach(obj, (val, key) => { - cb(val, key); - if (_.isPlainObject(val) || _.isArray(val)){ - forEachDeep(val, cb); + cb(val, key) + if (_.isPlainObject(val) || _.isArray(val)) { + forEachDeep(val, cb) } - }); + }) -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // admin indices are explicitly specified in order to specify a custom // dynamic_template and to avoid 'type not found' errors when deploying // the api codebase against an index without admin data -module.exports.tests.indices = function(test, common) { - test('explicitly specify some admin indices and their analyzer', function(t) { - t.equal(typeof schema.mappings, 'object', 'mappings present'); - t.equal(schema.mappings.dynamic_templates[0].nameGram.mapping.analyzer, 'peliasIndexOneEdgeGram'); - t.end(); - }); -}; +module.exports.tests.indices = function (test, common) { + test('explicitly specify some admin indices and their analyzer', function (t) { + t.equal(typeof schema.mappings, 'object', 'mappings present') + t.equal(schema.mappings.dynamic_templates[0].nameGram.mapping.analyzer, 'peliasIndexOneEdgeGram') + t.end() + }) +} // some 'admin' types allow single edgeNGrams and so have a different dynamic_template -module.exports.tests.dynamic_templates = function(test, common) { - test('dynamic_templates: nameGram', function(t) { - t.equal(typeof schema.mappings.dynamic_templates[0].nameGram, 'object', 'nameGram template specified'); - var template = schema.mappings.dynamic_templates[0].nameGram; - t.equal(template.path_match, 'name.*'); - t.equal(template.match_mapping_type, 'string'); +module.exports.tests.dynamic_templates = function (test, common) { + test('dynamic_templates: nameGram', function (t) { + t.equal(typeof schema.mappings.dynamic_templates[0].nameGram, 'object', 'nameGram template specified') + var template = schema.mappings.dynamic_templates[0].nameGram + t.equal(template.path_match, 'name.*') + t.equal(template.match_mapping_type, 'string') t.deepEqual(template.mapping, { type: 'text', analyzer: 'peliasIndexOneEdgeGram', search_analyzer: 'peliasQuery', similarity: 'peliasDefaultSimilarity' - }); - t.end(); - }); + }) + t.end() + }) test('dynamic_templates: phrase', function (t) { - t.equal(typeof schema.mappings.dynamic_templates[1].phrase, 'object', 'phrase template specified'); - var template = schema.mappings.dynamic_templates[1].phrase; - t.equal(template.path_match, 'phrase.*'); - t.equal(template.match_mapping_type, 'string'); + t.equal(typeof schema.mappings.dynamic_templates[1].phrase, 'object', 'phrase template specified') + var template = schema.mappings.dynamic_templates[1].phrase + t.equal(template.path_match, 'phrase.*') + t.equal(template.match_mapping_type, 'string') t.deepEqual(template.mapping, { type: 'text', analyzer: 'peliasPhrase', search_analyzer: 'peliasQuery', similarity: 'peliasDefaultSimilarity' - }); - t.end(); - }); + }) + t.end() + }) test('dynamic_templates: addendum', function (t) { - t.equal(typeof schema.mappings.dynamic_templates[2].addendum, 'object', 'addendum template specified'); - var template = schema.mappings.dynamic_templates[2].addendum; - t.equal(template.path_match, 'addendum.*'); - t.equal(template.match_mapping_type, 'string'); + t.equal(typeof schema.mappings.dynamic_templates[2].addendum, 'object', 'addendum template specified') + var template = schema.mappings.dynamic_templates[2].addendum + t.equal(template.path_match, 'addendum.*') + t.equal(template.match_mapping_type, 'string') t.deepEqual(template.mapping, { type: 'keyword', index: false, doc_values: false - }); - t.end(); - }); -}; + }) + t.end() + }) +} // ensure both "analyzer" and "search_analyzer" are set for stringy fields module.exports.tests.analyzers = function (test, common) { test('analyzers: ensure "analyzer" and "search_analyzer" are set', function (t) { - - const stringyTypes = ['string', 'text']; - const stringyFields = []; + const stringyTypes = ['string', 'text'] + const stringyFields = [] forEachDeep(schema, (value, key) => { - if (!_.isPlainObject(value)) { return; } - if (!stringyTypes.includes(_.get(value, 'type', ''))) { return; } - stringyFields.push({ key: key, value: value }); - }); + if (!_.isPlainObject(value)) { return } + if (!stringyTypes.includes(_.get(value, 'type', ''))) { return } + stringyFields.push({ key: key, value: value }) + }) stringyFields.forEach(field => { t.true(_.has(field.value, 'analyzer'), `analyzer not set on ${field.key}`) t.true(_.has(field.value, 'search_analyzer'), `search_analyzer not set on ${field.key}`) }) - t.end(); - }); -}; + t.end() + }) +} // current schema (compiled) - requires schema to be copied and settings to // be regenerated from a fixture in order to pass in CI environments. -module.exports.tests.current_schema = function(test, common) { - test('current schema vs. fixture', function(t) { - +module.exports.tests.current_schema = function (test, common) { + test('current schema vs. fixture', function (t) { // copy schema - var schemaCopy = JSON.parse( JSON.stringify( schema ) ); + var schemaCopy = JSON.parse(JSON.stringify(schema)) // use the pelias config fixture instead of the local config - process.env.PELIAS_CONFIG = path.resolve( __dirname + '/fixtures/config.json' ); - schemaCopy.settings = require('../settings')(); - delete process.env.PELIAS_CONFIG; + process.env.PELIAS_CONFIG = path.resolve(path.join(__dirname, '/fixtures/config.json')) + schemaCopy.settings = require('../settings')() + delete process.env.PELIAS_CONFIG // code intentionally commented to allow quick debugging of expected.json // common.diff(schemaCopy, fixture); @@ -118,18 +115,17 @@ module.exports.tests.current_schema = function(test, common) { // const fs = require('fs'); // fs.writeFileSync(path.resolve( __dirname + '/fixtures/expected.json' ), JSON.stringify(schemaCopy, null, 2)); - t.deepEqual(schemaCopy, fixture); - t.end(); - }); -}; + t.deepEqual(schemaCopy, fixture) + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('compile: ' + name, testFunction); + function test (name, testFunction) { + return tape('compile: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/configValidation.js b/test/configValidation.js index d4a2a676..b685daef 100644 --- a/test/configValidation.js +++ b/test/configValidation.js @@ -1,49 +1,44 @@ -const tape = require('tape'); +const configValidation = require('../configValidation') -const configValidation = require('../configValidation'); +module.exports.tests = {} -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('config without schema should throw error', function(t) { +module.exports.tests.interface = function (test, common) { + test('config without schema should throw error', function (t) { var config = { esclient: {} - }; - - t.throws(function() { - configValidation.validate(config); - }, /"schema" is required/, 'schema should exist'); - t.end(); + } - }); + t.throws(function () { + configValidation.validate(config) + }, /"schema" is required/, 'schema should exist') + t.end() + }) - test('config without schema.indexName should throw error', function(t) { + test('config without schema.indexName should throw error', function (t) { var config = { schema: { typeName: 'example_type' }, esclient: {} - }; + } - t.throws(function() { - configValidation.validate(config); - }, /"schema.indexName" is required/, 'schema.indexName should exist'); - t.end(); - - }); + t.throws(function () { + configValidation.validate(config) + }, /"schema.indexName" is required/, 'schema.indexName should exist') + t.end() + }) test('config without schema.typeName should throw error', function (t) { var config = { schema: { indexName: 'example_index' }, esclient: {} - }; + } t.throws(function () { - configValidation.validate(config); - }, /"schema.typeName" is required/, 'schema.typeName should exist'); - t.end(); + configValidation.validate(config) + }, /"schema.typeName" is required/, 'schema.typeName should exist') + t.end() + }) - }); - - test('config with non-string schema.indexName should throw error', function(t) { + test('config with non-string schema.indexName should throw error', function (t) { [null, 17, {}, [], false].forEach((value) => { var config = { schema: { @@ -51,17 +46,15 @@ module.exports.tests.interface = function(test, common) { typeName: 'example_type' }, esclient: {} - }; - - t.throws(function() { - configValidation.validate(config); - }, /"schema.indexName" must be a string/, 'schema.indexName should be a string'); + } - }); - - t.end(); + t.throws(function () { + configValidation.validate(config) + }, /"schema.indexName" must be a string/, 'schema.indexName should be a string') + }) - }); + t.end() + }) test('config with non-string schema.typeName should throw error', function (t) { [null, 17, {}, [], false].forEach((value) => { @@ -71,19 +64,17 @@ module.exports.tests.interface = function(test, common) { typeName: value }, esclient: {} - }; + } t.throws(function () { - configValidation.validate(config); - }, /"schema.typeName" must be a string/, 'schema.typeName should be a string'); + configValidation.validate(config) + }, /"schema.typeName" must be a string/, 'schema.typeName should be a string') + }) - }); + t.end() + }) - t.end(); - - }); - - test('config with non-object esclient should throw error', function(t) { + test('config with non-object esclient should throw error', function (t) { [null, 17, [], 'string', true].forEach((value) => { var config = { schema: { @@ -91,44 +82,39 @@ module.exports.tests.interface = function(test, common) { typeName: 'example_type' }, esclient: value - }; - - t.throws(function() { - configValidation.validate(config); - }, /"esclient" must be of type object/, 'esclient should be an object'); - - }); + } - t.end(); + t.throws(function () { + configValidation.validate(config) + }, /"esclient" must be of type object/, 'esclient should be an object') + }) - }); + t.end() + }) - test('config with string schema.indexName and object esclient should not throw error', function(t) { + test('config with string schema.indexName and object esclient should not throw error', function (t) { var config = { schema: { indexName: 'example_index', typeName: 'example_type' }, esclient: {} - }; - - t.doesNotThrow(function() { - configValidation.validate(config); - }, 'no error should have been thrown'); + } - t.end(); + t.doesNotThrow(function () { + configValidation.validate(config) + }, 'no error should have been thrown') - }); - -}; + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('configValidation: ' + name, testFunction); + function test (name, testFunction) { + return tape('configValidation: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/document.js b/test/document.js index 8307a66f..b072b54e 100644 --- a/test/document.js +++ b/test/document.js @@ -1,231 +1,230 @@ -const _ = require('lodash'); -const schema = require('../mappings/document'); +const _ = require('lodash') +const schema = require('../mappings/document') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // properties should always be set -module.exports.tests.properties = function(test, common) { - test('has properties', function(t) { - t.equal(typeof schema.properties, 'object', 'properties specified'); - t.end(); - }); -}; +module.exports.tests.properties = function (test, common) { + test('has properties', function (t) { + t.equal(typeof schema.properties, 'object', 'properties specified') + t.end() + }) +} // should contain the correct field definitions -module.exports.tests.fields = function(test, common) { +module.exports.tests.fields = function (test, common) { var fields = ['source', 'layer', 'name', 'phrase', 'address_parts', 'parent', 'center_point', 'shape', 'bounding_box', 'source_id', 'category', - 'population', 'popularity', 'addendum']; - test('fields specified', function(t) { - t.deepEqual(Object.keys(schema.properties), fields); - t.end(); - }); -}; + 'population', 'popularity', 'addendum'] + test('fields specified', function (t) { + t.deepEqual(Object.keys(schema.properties), fields) + t.end() + }) +} // should contain the correct address field definitions -module.exports.tests.address_fields = function(test, common) { - var fields = ['name','unit','number','street','cross_street','zip']; - test('address fields specified', function(t) { - t.deepEqual(Object.keys(schema.properties.address_parts.properties), fields); - t.end(); - }); -}; +module.exports.tests.address_fields = function (test, common) { + var fields = ['name', 'unit', 'number', 'street', 'cross_street', 'zip'] + test('address fields specified', function (t) { + t.deepEqual(Object.keys(schema.properties.address_parts.properties), fields) + t.end() + }) +} // address field analysis // ref: https://github.com/pelias/schema/pull/77 -module.exports.tests.address_analysis = function(test, common) { - var prop = schema.properties.address_parts.properties; +module.exports.tests.address_analysis = function (test, common) { + var prop = schema.properties.address_parts.properties // $name analysis is pretty basic, work can be done to improve this, although // at time of writing this field was not used by any API queries. - test('name', function(t) { - t.equal(prop.name.type, 'text'); - t.equal(prop.name.analyzer, 'keyword'); - t.equal(prop.name.search_analyzer, 'keyword'); - t.end(); - }); + test('name', function (t) { + t.equal(prop.name.type, 'text') + t.equal(prop.name.analyzer, 'keyword') + t.equal(prop.name.search_analyzer, 'keyword') + t.end() + }) // $unit analysis - test('unit', function(t) { - t.equal(prop.unit.type, 'text', 'unit has full text type'); - t.equal(prop.unit.analyzer, 'peliasUnit', 'unit analyzer is peliasUnit'); - t.equal(prop.unit.search_analyzer, 'peliasUnit', 'unit search_analyzer is peliasUnit'); - t.end(); - }); + test('unit', function (t) { + t.equal(prop.unit.type, 'text', 'unit has full text type') + t.equal(prop.unit.analyzer, 'peliasUnit', 'unit analyzer is peliasUnit') + t.equal(prop.unit.search_analyzer, 'peliasUnit', 'unit search_analyzer is peliasUnit') + t.end() + }) // $number analysis is discussed in: https://github.com/pelias/schema/pull/77 - test('number', function(t) { - t.equal(prop.number.type, 'text'); - t.equal(prop.number.analyzer, 'peliasHousenumber'); - t.equal(prop.number.search_analyzer, 'peliasHousenumber'); - t.end(); - }); + test('number', function (t) { + t.equal(prop.number.type, 'text') + t.equal(prop.number.analyzer, 'peliasHousenumber') + t.equal(prop.number.search_analyzer, 'peliasHousenumber') + t.end() + }) // $street analysis is discussed in: https://github.com/pelias/schema/pull/77 // and https://github.com/pelias/api/pull/1444 - test('street', function(t) { - t.equal(prop.street.type, 'text'); - t.equal(prop.street.analyzer, 'peliasStreet'); - t.equal(prop.street.search_analyzer, 'peliasQuery'); - t.end(); - }); + test('street', function (t) { + t.equal(prop.street.type, 'text') + t.equal(prop.street.analyzer, 'peliasStreet') + t.equal(prop.street.search_analyzer, 'peliasQuery') + t.end() + }) // $cross_street analysis test('cross_street', function (t) { - t.equal(prop.cross_street.type, 'text'); - t.equal(prop.cross_street.analyzer, 'peliasStreet'); - t.equal(prop.cross_street.search_analyzer, 'peliasQuery'); - t.end(); - }); + t.equal(prop.cross_street.type, 'text') + t.equal(prop.cross_street.analyzer, 'peliasStreet') + t.equal(prop.cross_street.search_analyzer, 'peliasQuery') + t.end() + }) // $zip analysis is discussed in: https://github.com/pelias/schema/pull/77 // note: this is a poor name, it would be better to rename this field to a more // generic term such as $postalcode as it is not specific to the USA. - test('zip', function(t) { - t.equal(prop.zip.type, 'text'); - t.equal(prop.zip.analyzer, 'peliasZip'); - t.equal(prop.zip.search_analyzer, 'peliasZip'); - t.end(); - }); -}; + test('zip', function (t) { + t.equal(prop.zip.type, 'text') + t.equal(prop.zip.analyzer, 'peliasZip') + t.equal(prop.zip.search_analyzer, 'peliasZip') + t.end() + }) +} // should contain the correct parent field definitions -module.exports.tests.parent_fields = function(test, common) { +module.exports.tests.parent_fields = function (test, common) { var fields = [ - 'continent', 'continent_a', 'continent_id', 'continent.fields.ngram', - 'ocean', 'ocean_a', 'ocean_id', 'ocean.fields.ngram', - 'empire', 'empire_a', 'empire_id', 'empire.fields.ngram', - 'country', 'country_a', 'country_id', 'country.fields.ngram', - 'dependency', 'dependency_a', 'dependency_id', 'dependency.fields.ngram', - 'marinearea', 'marinearea_a', 'marinearea_id', 'marinearea.fields.ngram', - 'macroregion', 'macroregion_a', 'macroregion_id', 'macroregion.fields.ngram', - 'region', 'region_a', 'region_id', 'region.fields.ngram', - 'macrocounty', 'macrocounty_a', 'macrocounty_id', 'macrocounty.fields.ngram', - 'county', 'county_a', 'county_id', 'county.fields.ngram', - 'locality', 'locality_a', 'locality_id', 'locality.fields.ngram', - 'borough', 'borough_a', 'borough_id', 'borough.fields.ngram', - 'localadmin', 'localadmin_a', 'localadmin_id', 'localadmin.fields.ngram', - 'neighbourhood', 'neighbourhood_a', 'neighbourhood_id', 'neighbourhood.fields.ngram', - 'postalcode', 'postalcode_a', 'postalcode_id', 'postalcode.fields.ngram' - ]; - test('parent fields specified', function(t) { - fields.forEach( expected => { - t.true( _.has( schema.properties.parent.properties, expected ), expected ); - }); - t.end(); - }); -}; + 'continent', 'continent_a', 'continent_id', 'continent.fields.ngram', + 'ocean', 'ocean_a', 'ocean_id', 'ocean.fields.ngram', + 'empire', 'empire_a', 'empire_id', 'empire.fields.ngram', + 'country', 'country_a', 'country_id', 'country.fields.ngram', + 'dependency', 'dependency_a', 'dependency_id', 'dependency.fields.ngram', + 'marinearea', 'marinearea_a', 'marinearea_id', 'marinearea.fields.ngram', + 'macroregion', 'macroregion_a', 'macroregion_id', 'macroregion.fields.ngram', + 'region', 'region_a', 'region_id', 'region.fields.ngram', + 'macrocounty', 'macrocounty_a', 'macrocounty_id', 'macrocounty.fields.ngram', + 'county', 'county_a', 'county_id', 'county.fields.ngram', + 'locality', 'locality_a', 'locality_id', 'locality.fields.ngram', + 'borough', 'borough_a', 'borough_id', 'borough.fields.ngram', + 'localadmin', 'localadmin_a', 'localadmin_id', 'localadmin.fields.ngram', + 'neighbourhood', 'neighbourhood_a', 'neighbourhood_id', 'neighbourhood.fields.ngram', + 'postalcode', 'postalcode_a', 'postalcode_id', 'postalcode.fields.ngram' + ] + test('parent fields specified', function (t) { + fields.forEach(expected => { + t.true(_.has(schema.properties.parent.properties, expected), expected) + }) + t.end() + }) +} // parent field analysis // ref: https://github.com/pelias/schema/pull/95 -module.exports.tests.parent_analysis = function(test, common) { - var prop = schema.properties.parent.properties; +module.exports.tests.parent_analysis = function (test, common) { + var prop = schema.properties.parent.properties var fields = [ 'continent', 'ocean', 'empire', 'country', 'dependency', 'marinearea', 'macroregion', 'region', 'macrocounty', 'county', 'locality', 'borough', 'localadmin', 'neighbourhood' - ]; - fields.forEach( function( field ){ - test(field, function(t) { - t.equal(prop[field].type, 'text', `${field} is set to text type`); - t.equal(prop[field].analyzer, 'peliasAdmin', `${field} analyzer is peliasAdmin`); - t.equal(prop[field+'_a'].type, 'text', `${field}_a type is text`); - t.equal(prop[field+'_a'].analyzer, 'peliasAdmin', `${field}_a analyzer is peliasAdmin`); - t.equal(prop[field+'_a'].search_analyzer, 'peliasAdmin', `${field}_a analyzer is peliasAdmin`); - t.equal(prop[field+'_id'].type, 'keyword', `${field}_id type is keyword`); - t.equal(prop[field+'_id'].index, undefined, `${field}_id index left at default`); + ] + fields.forEach(function (field) { + test(field, function (t) { + t.equal(prop[field].type, 'text', `${field} is set to text type`) + t.equal(prop[field].analyzer, 'peliasAdmin', `${field} analyzer is peliasAdmin`) + t.equal(prop[field + '_a'].type, 'text', `${field}_a type is text`) + t.equal(prop[field + '_a'].analyzer, 'peliasAdmin', `${field}_a analyzer is peliasAdmin`) + t.equal(prop[field + '_a'].search_analyzer, 'peliasAdmin', `${field}_a analyzer is peliasAdmin`) + t.equal(prop[field + '_id'].type, 'keyword', `${field}_id type is keyword`) + t.equal(prop[field + '_id'].index, undefined, `${field}_id index left at default`) // subfields - t.equal(prop[field].fields.ngram.type, 'text', `${field}.ngram type is full text`); - t.equal(prop[field].fields.ngram.analyzer, 'peliasIndexOneEdgeGram', `${field}.ngram analyzer is peliasIndexOneEdgeGram`); - t.equal(prop[field].fields.ngram.search_analyzer, 'peliasAdmin', `${field}.ngram analyzer is peliasIndexOneEdgeGram`); - - t.end(); - }); - }); - - test('postalcode', function(t) { - t.equal(prop['postalcode'].type, 'text', 'postalcode is full text field'); - t.equal(prop['postalcode'].analyzer, 'peliasZip', 'postalcode analyzer is peliasZip'); - t.equal(prop['postalcode'].search_analyzer, 'peliasZip', 'postalcode analyzer is peliasZip'); - t.equal(prop['postalcode'+'_a'].type, 'text', 'postalcode_a is full text field'); - t.equal(prop['postalcode'+'_a'].analyzer, 'peliasZip', 'postalcode_a analyzer is peliasZip'); - t.equal(prop['postalcode'+'_a'].search_analyzer, 'peliasZip', 'postalcode_a analyzer is peliasZip'); - t.equal(prop['postalcode'+'_id'].type, 'keyword', 'postalcode_id field is keyword type'); - t.equal(prop['postalcode'+'_id'].index, undefined, 'postalcode_id index left at default'); - - t.end(); - }); -}; - -module.exports.tests.dynamic_templates = function(test, common) { - test('dynamic_templates: nameGram', function(t) { - t.equal(typeof schema.dynamic_templates[0].nameGram, 'object', 'nameGram template specified'); - var template = schema.dynamic_templates[0].nameGram; - t.equal(template.path_match, 'name.*'); - t.equal(template.match_mapping_type, 'string'); - t.equal(template.mapping.type, 'text', 'set to full text type'); - t.equal(template.mapping.fielddata, undefined, 'fielddata is left to default (disabled)'); - t.equal(template.mapping.analyzer, 'peliasIndexOneEdgeGram', 'analyzer set'); - t.equal(template.mapping.search_analyzer, 'peliasQuery', 'search_analyzer set'); - t.end(); - }); - test('dynamic_templates: phrase', function(t) { - t.equal(typeof schema.dynamic_templates[1].phrase, 'object', 'phrase template specified'); - var template = schema.dynamic_templates[1].phrase; - t.equal(template.path_match, 'phrase.*'); - t.equal(template.match_mapping_type, 'string'); - t.equal(template.mapping.type, 'text', 'set to full text type'); - t.equal(template.mapping.fielddata, undefined, 'fielddata is left to default (disabled)'); - t.equal(template.mapping.analyzer, 'peliasPhrase', 'analyzer set'); - t.equal(template.mapping.search_analyzer, 'peliasQuery', 'search_analyzer set'); - t.end(); - }); -}; + t.equal(prop[field].fields.ngram.type, 'text', `${field}.ngram type is full text`) + t.equal(prop[field].fields.ngram.analyzer, 'peliasIndexOneEdgeGram', `${field}.ngram analyzer is peliasIndexOneEdgeGram`) + t.equal(prop[field].fields.ngram.search_analyzer, 'peliasAdmin', `${field}.ngram analyzer is peliasIndexOneEdgeGram`) + + t.end() + }) + }) + + test('postalcode', function (t) { + t.equal(prop['postalcode'].type, 'text', 'postalcode is full text field') + t.equal(prop['postalcode'].analyzer, 'peliasZip', 'postalcode analyzer is peliasZip') + t.equal(prop['postalcode'].search_analyzer, 'peliasZip', 'postalcode analyzer is peliasZip') + t.equal(prop['postalcode' + '_a'].type, 'text', 'postalcode_a is full text field') + t.equal(prop['postalcode' + '_a'].analyzer, 'peliasZip', 'postalcode_a analyzer is peliasZip') + t.equal(prop['postalcode' + '_a'].search_analyzer, 'peliasZip', 'postalcode_a analyzer is peliasZip') + t.equal(prop['postalcode' + '_id'].type, 'keyword', 'postalcode_id field is keyword type') + t.equal(prop['postalcode' + '_id'].index, undefined, 'postalcode_id index left at default') + + t.end() + }) +} + +module.exports.tests.dynamic_templates = function (test, common) { + test('dynamic_templates: nameGram', function (t) { + t.equal(typeof schema.dynamic_templates[0].nameGram, 'object', 'nameGram template specified') + var template = schema.dynamic_templates[0].nameGram + t.equal(template.path_match, 'name.*') + t.equal(template.match_mapping_type, 'string') + t.equal(template.mapping.type, 'text', 'set to full text type') + t.equal(template.mapping.fielddata, undefined, 'fielddata is left to default (disabled)') + t.equal(template.mapping.analyzer, 'peliasIndexOneEdgeGram', 'analyzer set') + t.equal(template.mapping.search_analyzer, 'peliasQuery', 'search_analyzer set') + t.end() + }) + test('dynamic_templates: phrase', function (t) { + t.equal(typeof schema.dynamic_templates[1].phrase, 'object', 'phrase template specified') + var template = schema.dynamic_templates[1].phrase + t.equal(template.path_match, 'phrase.*') + t.equal(template.match_mapping_type, 'string') + t.equal(template.mapping.type, 'text', 'set to full text type') + t.equal(template.mapping.fielddata, undefined, 'fielddata is left to default (disabled)') + t.equal(template.mapping.analyzer, 'peliasPhrase', 'analyzer set') + t.equal(template.mapping.search_analyzer, 'peliasQuery', 'search_analyzer set') + t.end() + }) +} // _all should be disabled -module.exports.tests.all_disabled = function(test, common) { - test('_all disabled', function(t) { - t.false(schema._all, '_all undefined'); - t.end(); - }); -}; +module.exports.tests.all_disabled = function (test, common) { + test('_all disabled', function (t) { + t.false(schema._all, '_all undefined') + t.end() + }) +} // dynamic should be true in order for dynamic_templates to function properly // @see: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-dynamic-mapping.html // strict ensures extra fields cannot be added: https://www.elastic.co/guide/en/elasticsearch/guide/current/dynamic-mapping.html -module.exports.tests.dynamic_disabled = function(test, common) { - test('dynamic strict', function(t) { - t.equal(schema.dynamic, 'strict', 'dynamic true'); - t.end(); - }); -}; +module.exports.tests.dynamic_disabled = function (test, common) { + test('dynamic strict', function (t) { + t.equal(schema.dynamic, 'strict', 'dynamic true') + t.end() + }) +} // shape field should be exluded from _source because it's massive -module.exports.tests._source = function(test, common) { - test('_source', function(t) { - t.ok(Array.isArray(schema._source.excludes), 'exclusions specified'); - t.equal(schema._source.excludes[0], 'shape', 'exclude shape'); - t.equal(schema._source.excludes[1], 'phrase', 'exclude phrase'); - t.end(); - }); -}; +module.exports.tests._source = function (test, common) { + test('_source', function (t) { + t.ok(Array.isArray(schema._source.excludes), 'exclusions specified') + t.equal(schema._source.excludes[0], 'shape', 'exclude shape') + t.equal(schema._source.excludes[1], 'phrase', 'exclude phrase') + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('document: ' + name, testFunction); + function test (name, testFunction) { + return tape('document: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/partial-admin.js b/test/partial-admin.js index 09f52d53..141095b3 100644 --- a/test/partial-admin.js +++ b/test/partial-admin.js @@ -1,54 +1,53 @@ -var schema = require('../mappings/partial/admin'); +var schema = require('../mappings/partial/admin') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // this should never need to change -module.exports.tests.type = function(test, common) { - test('correct type', function(t) { - t.equal(schema.type, 'text', 'set to text field for full text search'); - t.end(); - }); -}; - -module.exports.tests.store = function(test, common) { - test('store unset (will not be stored)', function(t) { - t.equal(schema.store, undefined, 'unset'); - t.end(); - }); -}; +module.exports.tests.type = function (test, common) { + test('correct type', function (t) { + t.equal(schema.type, 'text', 'set to text field for full text search') + t.end() + }) +} + +module.exports.tests.store = function (test, common) { + test('store unset (will not be stored)', function (t) { + t.equal(schema.store, undefined, 'unset') + t.end() + }) +} // this should be enabled to allow 'exists' filters to work -module.exports.tests.index = function(test, common) { - test('index enabled', function(t) { - t.notEqual(schema.index, 'no', 'should not be disabled'); - t.end(); - }); -}; +module.exports.tests.index = function (test, common) { + test('index enabled', function (t) { + t.notEqual(schema.index, 'no', 'should not be disabled') + t.end() + }) +} // pelias analysis does not ensure that we get ['Great Britain'] instead of ['Great','Britain'] // TODO this needs to be addressed -module.exports.tests.analysis = function(test, common) { - test('index analysis', function(t) { - t.equal(schema.analyzer, 'peliasAdmin', 'should be peliasAdmin'); - t.end(); - }); -}; +module.exports.tests.analysis = function (test, common) { + test('index analysis', function (t) { + t.equal(schema.analyzer, 'peliasAdmin', 'should be peliasAdmin') + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('admin: ' + name, testFunction); + function test (name, testFunction) { + return tape('admin: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/partial-centroid.js b/test/partial-centroid.js index 0b745269..64aa077e 100644 --- a/test/partial-centroid.js +++ b/test/partial-centroid.js @@ -1,31 +1,30 @@ -var schema = require('../mappings/partial/centroid'); +var schema = require('../mappings/partial/centroid') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // this should never need to change -module.exports.tests.type = function(test, common) { - test('correct type', function(t) { - t.equal(schema.type, 'geo_point', 'correct value'); - t.end(); - }); -}; +module.exports.tests.type = function (test, common) { + test('correct type', function (t) { + t.equal(schema.type, 'geo_point', 'correct value') + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('centroid: ' + name, testFunction); + function test (name, testFunction) { + return tape('centroid: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/partial-hash.js b/test/partial-hash.js index bb69261c..e8767fc9 100644 --- a/test/partial-hash.js +++ b/test/partial-hash.js @@ -1,39 +1,38 @@ -var schema = require('../mappings/partial/hash'); +var schema = require('../mappings/partial/hash') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // this should never need to change -module.exports.tests.type = function(test, common) { - test('correct type', function(t) { - t.equal(schema.type, 'object', 'correct value'); - t.end(); - }); -}; +module.exports.tests.type = function (test, common) { + test('correct type', function (t) { + t.equal(schema.type, 'object', 'correct value') + t.end() + }) +} // if dynamic=false you won't be able to // query the properties of the object! -module.exports.tests.dynamic = function(test, common) { - test('dynamic true', function(t) { - t.equal(schema.dynamic, true, 'correct value'); - t.end(); - }); -}; +module.exports.tests.dynamic = function (test, common) { + test('dynamic true', function (t) { + t.equal(schema.dynamic, true, 'correct value') + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('hash: ' + name, testFunction); + function test (name, testFunction) { + return tape('hash: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/partial-keyword.js b/test/partial-keyword.js index 03878009..718a82f3 100644 --- a/test/partial-keyword.js +++ b/test/partial-keyword.js @@ -1,45 +1,44 @@ -var schema = require('../mappings/partial/keyword'); +var schema = require('../mappings/partial/keyword') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.compile = function(test, common) { - test('valid schema file', function(t) { - t.equal(typeof schema, 'object', 'schema generated'); - t.equal(Object.keys(schema).length>0, true, 'schema has body'); - t.end(); - }); -}; +module.exports.tests.compile = function (test, common) { + test('valid schema file', function (t) { + t.equal(typeof schema, 'object', 'schema generated') + t.equal(Object.keys(schema).length > 0, true, 'schema has body') + t.end() + }) +} // this should never need to change -module.exports.tests.type = function(test, common) { - test('correct type', function(t) { - t.equal(schema.type, 'keyword', 'correct value'); - t.end(); - }); -}; - -module.exports.tests.store = function(test, common) { - test('store unset (will not be stored)', function(t) { - t.equal(schema.store, undefined, 'unset'); - t.end(); - }); -}; +module.exports.tests.type = function (test, common) { + test('correct type', function (t) { + t.equal(schema.type, 'keyword', 'correct value') + t.end() + }) +} + +module.exports.tests.store = function (test, common) { + test('store unset (will not be stored)', function (t) { + t.equal(schema.store, undefined, 'unset') + t.end() + }) +} // do not perform analysis on categories -module.exports.tests.analysis = function(test, common) { - test('index analysis disabled', function(t) { - t.equal(schema.index, undefined, 'should be set to default'); - t.end(); - }); -}; +module.exports.tests.analysis = function (test, common) { + test('index analysis disabled', function (t) { + t.equal(schema.index, undefined, 'should be set to default') + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('keyword: ' + name, testFunction); + function test (name, testFunction) { + return tape('keyword: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/run.js b/test/run.js index 185d144f..6046937a 100644 --- a/test/run.js +++ b/test/run.js @@ -1,12 +1,13 @@ -var tape = require('tape'), - diff = require('difflet')({ indent : 2, comment : true }); +var tape = require('tape') + +var diff = require('difflet')({ indent: 2, comment: true }) var common = { // a visual deep diff rendered using console.error() - diff: function( actual, expected ){ - console.error( diff.compare( actual, expected ) ); + diff: function (actual, expected) { + console.error(diff.compare(actual, expected)) } -}; +} var tests = [ require('./compile.js'), @@ -17,9 +18,9 @@ var tests = [ require('./partial-hash.js'), require('./settings.js'), require('./configValidation.js'), - require('./synonyms/parser.js'), -]; + require('./synonyms/parser.js') +] -tests.map(function(t) { - t.all(tape, common); -}); +tests.map(function (t) { + t.all(tape, common) +}) diff --git a/test/settings.js b/test/settings.js index badb401c..ca3ad058 100644 --- a/test/settings.js +++ b/test/settings.js @@ -1,52 +1,49 @@ -var path = require('path'), - settings = require('../settings'), - fs = require('fs'); - -module.exports.tests = {}; - -module.exports.tests.interface = function(test, common) { - test('valid interface', function(t) { - t.equal(typeof settings, 'function', 'settings is a function'); - t.end(); - }); -}; - -module.exports.tests.configValidation = function(test, common) { - test('configValidation throwing error should rethrow', function(t) { - t.throws(function() { - const proxyquire = require('proxyquire').noCallThru(); +const path = require('path') +const settings = require('../settings') + +module.exports.tests = {} + +module.exports.tests.interface = function (test, common) { + test('valid interface', function (t) { + t.equal(typeof settings, 'function', 'settings is a function') + t.end() + }) +} + +module.exports.tests.configValidation = function (test, common) { + test('configValidation throwing error should rethrow', function (t) { + t.throws(function () { + const proxyquire = require('proxyquire').noCallThru() proxyquire('../settings', { './configValidation': { validate: () => { - throw Error('config is not valid'); + throw Error('config is not valid') } } - })(); - - }, /config is not valid/); - - t.end(); - - }); -}; - -module.exports.tests.compile = function(test, common) { - test('valid settings file', function(t) { - var s = settings(); - t.equal(typeof s, 'object', 'settings generated'); - t.equal(Object.keys(s).length>0, true, 'settings has body'); - t.end(); - }); -}; + })() + }, /config is not valid/) + + t.end() + }) +} + +module.exports.tests.compile = function (test, common) { + test('valid settings file', function (t) { + var s = settings() + t.equal(typeof s, 'object', 'settings generated') + t.equal(Object.keys(s).length > 0, true, 'settings has body') + t.end() + }) +} // analysis should always be set -module.exports.tests.analysis = function(test, common) { - test('has analysis settings', function(t) { - var s = settings(); - t.equal(typeof s.analysis, 'object', 'analysis specified'); - t.end(); - }); -}; +module.exports.tests.analysis = function (test, common) { + test('has analysis settings', function (t) { + var s = settings() + t.equal(typeof s.analysis, 'object', 'analysis specified') + t.end() + }) +} // -- analyzers -- @@ -54,54 +51,54 @@ module.exports.tests.analysis = function(test, common) { // note: the multiplexer ensures than we do not virally generate synonyms of synonyms. module.exports.tests.nameSynonymsMultiplexerFilter = function (test, common) { test('has admin_synonyms_multiplexer filter', function (t) { - var s = settings(); - t.equal(typeof s.analysis.filter.admin_synonyms_multiplexer, 'object', 'there is a admin_synonyms_multiplexer filter'); - var filter = s.analysis.filter.admin_synonyms_multiplexer; - t.equal(filter.type, 'multiplexer'); + var s = settings() + t.equal(typeof s.analysis.filter.admin_synonyms_multiplexer, 'object', 'there is a admin_synonyms_multiplexer filter') + var filter = s.analysis.filter.admin_synonyms_multiplexer + t.equal(filter.type, 'multiplexer') t.deepEqual(filter.filters, [ 'synonyms/custom_admin', 'synonyms/personal_titles', 'synonyms/place_names' - ]); - t.end(); - }); -}; - -module.exports.tests.peliasAdminAnalyzer = function(test, common) { - test('has pelias admin analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasAdmin, 'object', 'there is a pelias admin analyzer'); - var analyzer = s.analysis.analyzer.peliasAdmin; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); + ]) + t.end() + }) +} + +module.exports.tests.peliasAdminAnalyzer = function (test, common) { + test('has pelias admin analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasAdmin, 'object', 'there is a pelias admin analyzer') + var analyzer = s.analysis.analyzer.peliasAdmin + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) test('peliasAdmin token filters', function (t) { - var analyzer = settings().analysis.analyzer.peliasAdmin; + var analyzer = settings().analysis.analyzer.peliasAdmin t.deepEqual(analyzer.filter, [ - "lowercase", - "trim", - "admin_synonyms_multiplexer", - "icu_folding", - "word_delimiter", - "unique_only_same_position", - "notnull", - "flatten_graph" - ]); - t.end(); - }); -}; + 'lowercase', + 'trim', + 'admin_synonyms_multiplexer', + 'icu_folding', + 'word_delimiter', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' + ]) + t.end() + }) +} // this multiplexer filter provides all the synonyms used by the peliasPhrase and peliasIndexOneEdgeGram analyzers // note: the multiplexer ensures than we do not virally generate synonyms of synonyms. module.exports.tests.nameSynonymsMultiplexerFilter = function (test, common) { test('has name_synonyms_multiplexer filter', function (t) { - var s = settings(); - t.equal(typeof s.analysis.filter.name_synonyms_multiplexer, 'object', 'there is a name_synonyms_multiplexer filter'); - var filter = s.analysis.filter.name_synonyms_multiplexer; - t.equal(filter.type, 'multiplexer'); + var s = settings() + t.equal(typeof s.analysis.filter.name_synonyms_multiplexer, 'object', 'there is a name_synonyms_multiplexer filter') + var filter = s.analysis.filter.name_synonyms_multiplexer + t.equal(filter.type, 'multiplexer') t.deepEqual(filter.filters, [ 'synonyms/custom_name', 'synonyms/personal_titles', @@ -109,53 +106,53 @@ module.exports.tests.nameSynonymsMultiplexerFilter = function (test, common) { 'synonyms/streets', 'synonyms/directionals', 'synonyms/punctuation' - ]); - t.end(); - }); -}; - -module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) { - test('has peliasIndexOneEdgeGram analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasIndexOneEdgeGram, 'object', 'there is a peliasIndexOneEdgeGram analyzer'); - var analyzer = s.analysis.analyzer.peliasIndexOneEdgeGram; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); - test('peliasIndexOneEdgeGram token filters', function(t) { - var analyzer = settings().analysis.analyzer.peliasIndexOneEdgeGram; - t.deepEqual( analyzer.filter, [ - "lowercase", - "trim", - "name_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "removeAllZeroNumericPrefix", - "peliasOneEdgeGramFilter", - "unique_only_same_position", - "notnull", - "flatten_graph" - ]); - t.end(); - }); -}; + ]) + t.end() + }) +} + +module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function (test, common) { + test('has peliasIndexOneEdgeGram analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasIndexOneEdgeGram, 'object', 'there is a peliasIndexOneEdgeGram analyzer') + var analyzer = s.analysis.analyzer.peliasIndexOneEdgeGram + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) + test('peliasIndexOneEdgeGram token filters', function (t) { + var analyzer = settings().analysis.analyzer.peliasIndexOneEdgeGram + t.deepEqual(analyzer.filter, [ + 'lowercase', + 'trim', + 'name_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'removeAllZeroNumericPrefix', + 'peliasOneEdgeGramFilter', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' + ]) + t.end() + }) +} module.exports.tests.peliasQueryAnalyzer = function (test, common) { test('has peliasQuery analyzer', function (t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasQuery, 'object', 'there is a peliasQuery analyzer'); - var analyzer = s.analysis.analyzer.peliasQuery; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasQuery, 'object', 'there is a peliasQuery analyzer') + var analyzer = s.analysis.analyzer.peliasQuery + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) test('peliasQuery token filters', function (t) { - var analyzer = settings().analysis.analyzer.peliasQuery; + var analyzer = settings().analysis.analyzer.peliasQuery t.deepEqual(analyzer.filter, [ 'lowercase', 'trim', @@ -164,441 +161,438 @@ module.exports.tests.peliasQueryAnalyzer = function (test, common) { 'removeAllZeroNumericPrefix', 'unique_only_same_position', 'notnull' - ]); - t.end(); - }); -}; - -module.exports.tests.peliasPhraseAnalyzer = function(test, common) { - test('has peliasPhrase analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasPhrase, 'object', 'there is a peliasPhrase analyzer'); - var analyzer = s.analysis.analyzer.peliasPhrase; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); - test('peliasPhrase token filters', function(t) { - var analyzer = settings().analysis.analyzer.peliasPhrase; - t.deepEqual( analyzer.filter, [ - "lowercase", - "trim", - "remove_duplicate_spaces", - "name_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "unique_only_same_position", - "notnull", - "flatten_graph" - ]); - t.end(); - }); -}; - -module.exports.tests.peliasZipAnalyzer = function(test, common) { - test('has peliasZip analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasZip, 'object', 'there is a peliasZip analyzer'); - var analyzer = s.analysis.analyzer.peliasZip; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ['alphanumeric', 'nfkc_normalizer'], 'alphanumeric filter specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); - test('peliasZip token filters', function(t) { - var analyzer = settings().analysis.analyzer.peliasZip; - t.deepEqual( analyzer.filter, [ - "lowercase", - "trim", - "icu_folding", - "unique_only_same_position", - "notnull" - ]); - t.end(); - }); -}; - -module.exports.tests.peliasUnitAnalyzer = function(test, common) { - test('has peliasUnit analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasUnit, 'object', 'there is a peliasUnit analyzer'); - var analyzer = s.analysis.analyzer.peliasUnit; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ['alphanumeric', 'nfkc_normalizer'], 'alphanumeric filter specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); - test('peliasUnit token filters', function(t) { - var analyzer = settings().analysis.analyzer.peliasUnit; - t.deepEqual( analyzer.filter, [ - "lowercase", - "trim", - "icu_folding", - "unique_only_same_position", - "notnull" - ]); - t.end(); - }); -}; - -module.exports.tests.peliasHousenumberAnalyzer = function(test, common) { - test('has peliasHousenumber analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasHousenumber, 'object', 'there is a peliasHousenumber analyzer'); - var analyzer = s.analysis.analyzer.peliasHousenumber; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ["numeric"], 'numeric filter specified'); - t.false(Array.isArray(analyzer.filter), 'no filters specified'); - t.end(); - }); -}; + ]) + t.end() + }) +} + +module.exports.tests.peliasPhraseAnalyzer = function (test, common) { + test('has peliasPhrase analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasPhrase, 'object', 'there is a peliasPhrase analyzer') + var analyzer = s.analysis.analyzer.peliasPhrase + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) + test('peliasPhrase token filters', function (t) { + var analyzer = settings().analysis.analyzer.peliasPhrase + t.deepEqual(analyzer.filter, [ + 'lowercase', + 'trim', + 'remove_duplicate_spaces', + 'name_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' + ]) + t.end() + }) +} + +module.exports.tests.peliasZipAnalyzer = function (test, common) { + test('has peliasZip analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasZip, 'object', 'there is a peliasZip analyzer') + var analyzer = s.analysis.analyzer.peliasZip + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['alphanumeric', 'nfkc_normalizer'], 'alphanumeric filter specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) + test('peliasZip token filters', function (t) { + var analyzer = settings().analysis.analyzer.peliasZip + t.deepEqual(analyzer.filter, [ + 'lowercase', + 'trim', + 'icu_folding', + 'unique_only_same_position', + 'notnull' + ]) + t.end() + }) +} + +module.exports.tests.peliasUnitAnalyzer = function (test, common) { + test('has peliasUnit analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasUnit, 'object', 'there is a peliasUnit analyzer') + var analyzer = s.analysis.analyzer.peliasUnit + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['alphanumeric', 'nfkc_normalizer'], 'alphanumeric filter specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) + test('peliasUnit token filters', function (t) { + var analyzer = settings().analysis.analyzer.peliasUnit + t.deepEqual(analyzer.filter, [ + 'lowercase', + 'trim', + 'icu_folding', + 'unique_only_same_position', + 'notnull' + ]) + t.end() + }) +} + +module.exports.tests.peliasHousenumberAnalyzer = function (test, common) { + test('has peliasHousenumber analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasHousenumber, 'object', 'there is a peliasHousenumber analyzer') + var analyzer = s.analysis.analyzer.peliasHousenumber + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['numeric'], 'numeric filter specified') + t.false(Array.isArray(analyzer.filter), 'no filters specified') + t.end() + }) +} // this multiplexer filter provides all the synonyms used by the peliasStreet analyzer // note: the multiplexer ensures than we do not virally generate synonyms of synonyms. module.exports.tests.streetSynonymsMultiplexerFilter = function (test, common) { test('has street_synonyms_multiplexer filter', function (t) { - var s = settings(); - t.equal(typeof s.analysis.filter.street_synonyms_multiplexer, 'object', 'there is a street_synonyms_multiplexer filter'); - var filter = s.analysis.filter.street_synonyms_multiplexer; - t.equal(filter.type, 'multiplexer'); + var s = settings() + t.equal(typeof s.analysis.filter.street_synonyms_multiplexer, 'object', 'there is a street_synonyms_multiplexer filter') + var filter = s.analysis.filter.street_synonyms_multiplexer + t.equal(filter.type, 'multiplexer') t.deepEqual(filter.filters, [ 'synonyms/custom_street', 'synonyms/personal_titles', 'synonyms/streets', 'synonyms/directionals' - ]); - t.end(); - }); -}; - -module.exports.tests.peliasStreetAnalyzer = function(test, common) { - test('has peliasStreet analyzer', function(t) { - var s = settings(); - t.equal(typeof s.analysis.analyzer.peliasStreet, 'object', 'there is a peliasStreet analyzer'); - var analyzer = s.analysis.analyzer.peliasStreet; - t.equal(analyzer.type, 'custom', 'custom analyzer'); - t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified'); - t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified'); - t.true(Array.isArray(analyzer.filter), 'filters specified'); - t.end(); - }); - test('peliasStreet token filters', function(t) { - var analyzer = settings().analysis.analyzer.peliasStreet; - t.deepEqual( analyzer.filter, [ - "lowercase", - "trim", - "remove_duplicate_spaces", - "street_synonyms_multiplexer", - "icu_folding", - "remove_ordinals", - "trim", - "unique_only_same_position", - "notnull", - "flatten_graph" - ]); - t.end(); - }); -}; + ]) + t.end() + }) +} + +module.exports.tests.peliasStreetAnalyzer = function (test, common) { + test('has peliasStreet analyzer', function (t) { + var s = settings() + t.equal(typeof s.analysis.analyzer.peliasStreet, 'object', 'there is a peliasStreet analyzer') + var analyzer = s.analysis.analyzer.peliasStreet + t.equal(analyzer.type, 'custom', 'custom analyzer') + t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified') + t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified') + t.true(Array.isArray(analyzer.filter), 'filters specified') + t.end() + }) + test('peliasStreet token filters', function (t) { + var analyzer = settings().analysis.analyzer.peliasStreet + t.deepEqual(analyzer.filter, [ + 'lowercase', + 'trim', + 'remove_duplicate_spaces', + 'street_synonyms_multiplexer', + 'icu_folding', + 'remove_ordinals', + 'trim', + 'unique_only_same_position', + 'notnull', + 'flatten_graph' + ]) + t.end() + }) +} // cycle through all analyzers and ensure the corrsponding token filters are globally defined -module.exports.tests.allTokenFiltersPresent = function(test, common) { +module.exports.tests.allTokenFiltersPresent = function (test, common) { var ES_INBUILT_FILTERS = [ 'lowercase', 'icu_folding', 'trim', 'word_delimiter', 'unique', 'flatten_graph' - ]; - test('all token filters present', function(t) { - var s = settings(); - for( var analyzerName in s.analysis.analyzer ){ - var analyzer = s.analysis.analyzer[analyzerName]; - if( Array.isArray( analyzer.filter ) ){ - analyzer.filter.forEach( function( tokenFilterName ){ + ] + test('all token filters present', function (t) { + var s = settings() + for (var analyzerName in s.analysis.analyzer) { + var analyzer = s.analysis.analyzer[analyzerName] + if (Array.isArray(analyzer.filter)) { + analyzer.filter.forEach(function (tokenFilterName) { var filterExists = ( - s.analysis.filter.hasOwnProperty( tokenFilterName ) || - ES_INBUILT_FILTERS.includes( tokenFilterName ) - ); - t.true( filterExists, 'token filter exists: ' + tokenFilterName ); - }); + s.analysis.filter.hasOwnProperty(tokenFilterName) || + ES_INBUILT_FILTERS.includes(tokenFilterName) + ) + t.true(filterExists, 'token filter exists: ' + tokenFilterName) + }) } } - t.end(); - }); -}; + t.end() + }) +} // cycle through all analyzers and ensure the mandatory token filters are defined on each module.exports.tests.mandatoryTokenFiltersPresent = function (test, common) { var MANDATORY_FILTERS = [ 'lowercase', 'icu_folding', 'trim', 'unique_only_same_position', 'notnull' - ]; + ] test('mandatory filters present', function (t) { - var s = settings(); + var s = settings() for (var analyzerName in s.analysis.analyzer) { - var analyzer = s.analysis.analyzer[analyzerName]; + var analyzer = s.analysis.analyzer[analyzerName] if (Array.isArray(analyzer.filter)) { MANDATORY_FILTERS.forEach(function (filterName) { t.true( analyzer.filter.includes(filterName), `mandatory token filter ${filterName} not defined on ${analyzerName}` - ); - }); + ) + }) } } - t.end(); - }); -}; + t.end() + }) +} // cycle through all analyzers and ensure the corrsponding character filters are defined -module.exports.tests.allCharacterFiltersPresent = function(test, common) { - var ES_INBUILT_FILTERS = []; - test('all character filters present', function(t) { - var s = settings(); - for( var analyzerName in s.analysis.analyzer ){ - var analyzer = s.analysis.analyzer[analyzerName]; - if( Array.isArray( analyzer.char_filter ) ){ - analyzer.char_filter.forEach( function( charFilterName ){ - var filterExists = s.analysis.char_filter.hasOwnProperty( charFilterName ); - if( !filterExists && -1 < ES_INBUILT_FILTERS.indexOf( charFilterName ) ){ - filterExists = true; +module.exports.tests.allCharacterFiltersPresent = function (test, common) { + var ES_INBUILT_FILTERS = [] + test('all character filters present', function (t) { + var s = settings() + for (var analyzerName in s.analysis.analyzer) { + var analyzer = s.analysis.analyzer[analyzerName] + if (Array.isArray(analyzer.char_filter)) { + analyzer.char_filter.forEach(function (charFilterName) { + var filterExists = s.analysis.char_filter.hasOwnProperty(charFilterName) + if (!filterExists && ES_INBUILT_FILTERS.indexOf(charFilterName) > -1) { + filterExists = true } - t.true( filterExists, 'missing character filter: ' + charFilterName ); - }); + t.true(filterExists, 'missing character filter: ' + charFilterName) + }) } } - t.end(); - }); -}; + t.end() + }) +} // -- filters -- // note: pattern/replace should not have surrounding whitespace // we convert and->& rather than &->and to save memory/disk -module.exports.tests.punctuationFilter = function(test, common) { - test('has punctuation filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter['synonyms/punctuation'], 'object', 'there is a punctuation filter'); - var filter = s.analysis.filter['synonyms/punctuation']; - t.equal(filter.type, 'synonym'); +module.exports.tests.punctuationFilter = function (test, common) { + test('has punctuation filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter['synonyms/punctuation'], 'object', 'there is a punctuation filter') + var filter = s.analysis.filter['synonyms/punctuation'] + t.equal(filter.type, 'synonym') t.deepEqual(filter.synonyms, [ - "&,and", - "&,und" - ]); - t.end(); - }); -}; + '&,and', + '&,und' + ]) + t.end() + }) +} // this filter simply removes empty tokens which can occur when other // filters do weird things, so just to be sure, we explicitly get rid of them -module.exports.tests.notnullFilter = function(test, common) { - test('has notnull filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter.notnull, 'object', 'there is a notnull filter'); - var filter = s.analysis.filter.notnull; - t.equal(filter.type, 'length'); - t.equal(filter.min, 1); - t.end(); - }); -}; +module.exports.tests.notnullFilter = function (test, common) { + test('has notnull filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter.notnull, 'object', 'there is a notnull filter') + var filter = s.analysis.filter.notnull + t.equal(filter.type, 'length') + t.equal(filter.min, 1) + t.end() + }) +} // this filter creates edgeNGrams with the minimum size of 1 -module.exports.tests.peliasOneEdgeGramFilter = function(test, common) { - test('has peliasIndexOneEdgeGram filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter.peliasOneEdgeGramFilter, 'object', 'there is a peliasIndexOneEdgeGram filter'); - var filter = s.analysis.filter.peliasOneEdgeGramFilter; - t.equal(filter.type, 'edgeNGram'); - t.equal(filter.min_gram, 1); - t.equal(filter.max_gram, 24); - t.end(); - }); -}; +module.exports.tests.peliasOneEdgeGramFilter = function (test, common) { + test('has peliasIndexOneEdgeGram filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter.peliasOneEdgeGramFilter, 'object', 'there is a peliasIndexOneEdgeGram filter') + var filter = s.analysis.filter.peliasOneEdgeGramFilter + t.equal(filter.type, 'edgeNGram') + t.equal(filter.min_gram, 1) + t.equal(filter.max_gram, 24) + t.end() + }) +} // this filter removed leading 0 characters. eg. 0001 -> 1 -module.exports.tests.removeAllZeroNumericPrefixFilter = function(test, common) { - test('has removeAllZeroNumericPrefix filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter.removeAllZeroNumericPrefix, 'object', 'there is a removeAllZeroNumericPrefix filter'); - var filter = s.analysis.filter.removeAllZeroNumericPrefix; - t.equal(filter.type, 'pattern_replace'); - t.equal(filter.pattern, '^(0*)'); - t.equal(filter.replacement, ''); - t.end(); - }); -}; +module.exports.tests.removeAllZeroNumericPrefixFilter = function (test, common) { + test('has removeAllZeroNumericPrefix filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter.removeAllZeroNumericPrefix, 'object', 'there is a removeAllZeroNumericPrefix filter') + var filter = s.analysis.filter.removeAllZeroNumericPrefix + t.equal(filter.type, 'pattern_replace') + t.equal(filter.pattern, '^(0*)') + t.equal(filter.replacement, '') + t.end() + }) +} // this filter provides synonyms for street suffixes // eg. road=>rd -module.exports.tests.streetSynonymFilter = function(test, common) { - test('has synonyms/streets filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter['synonyms/streets'], 'object', 'there is a synonyms/streets filter'); - var filter = s.analysis.filter['synonyms/streets']; - t.equal(filter.type, 'synonym'); - t.true(Array.isArray(filter.synonyms)); - t.equal(filter.synonyms.length, 809); - t.end(); - }); -}; +module.exports.tests.streetSynonymFilter = function (test, common) { + test('has synonyms/streets filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter['synonyms/streets'], 'object', 'there is a synonyms/streets filter') + var filter = s.analysis.filter['synonyms/streets'] + t.equal(filter.type, 'synonym') + t.true(Array.isArray(filter.synonyms)) + t.equal(filter.synonyms.length, 809) + t.end() + }) +} // this filter stems common directional terms // eg. north=>n and south=>s -module.exports.tests.directionalSynonymFilter = function(test, common) { - test('has directionals filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter['synonyms/directionals'], 'object', 'there is a synonyms/directionals filter'); - var filter = s.analysis.filter['synonyms/directionals']; - t.equal(filter.type, 'synonym'); - t.true(Array.isArray(filter.synonyms)); - t.equal(filter.synonyms.length, 69); - t.end(); - }); -}; +module.exports.tests.directionalSynonymFilter = function (test, common) { + test('has directionals filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter['synonyms/directionals'], 'object', 'there is a synonyms/directionals filter') + var filter = s.analysis.filter['synonyms/directionals'] + t.equal(filter.type, 'synonym') + t.true(Array.isArray(filter.synonyms)) + t.equal(filter.synonyms.length, 69) + t.end() + }) +} // this filter provides common synonyms for personal titles // eg. doctor=>dr module.exports.tests.personalTitleSynonymFilter = function (test, common) { test('has personal_titles filter', function (t) { - var s = settings(); - t.equal(typeof s.analysis.filter['synonyms/personal_titles'], 'object', 'there is a synonyms/personal_titles filter'); - var filter = s.analysis.filter['synonyms/personal_titles']; - t.equal(filter.type, 'synonym'); - t.true(Array.isArray(filter.synonyms)); - t.equal(filter.synonyms.length, 191); - t.end(); - }); -}; + var s = settings() + t.equal(typeof s.analysis.filter['synonyms/personal_titles'], 'object', 'there is a synonyms/personal_titles filter') + var filter = s.analysis.filter['synonyms/personal_titles'] + t.equal(filter.type, 'synonym') + t.true(Array.isArray(filter.synonyms)) + t.equal(filter.synonyms.length, 191) + t.end() + }) +} // this filter provides common synonyms for place names // eg. park=>pk module.exports.tests.placeNameSynonymFilter = function (test, common) { test('has place_names filter', function (t) { - var s = settings(); - t.equal(typeof s.analysis.filter['synonyms/place_names'], 'object', 'there is a synonyms/place_names filter'); - var filter = s.analysis.filter['synonyms/place_names']; - t.equal(filter.type, 'synonym'); - t.true(Array.isArray(filter.synonyms)); - t.equal(filter.synonyms.length, 314); - t.end(); - }); -}; + var s = settings() + t.equal(typeof s.analysis.filter['synonyms/place_names'], 'object', 'there is a synonyms/place_names filter') + var filter = s.analysis.filter['synonyms/place_names'] + t.equal(filter.type, 'synonym') + t.true(Array.isArray(filter.synonyms)) + t.equal(filter.synonyms.length, 314) + t.end() + }) +} // this filter removes number ordinals // eg. 26th => 26, 1st => 1 -module.exports.tests.removeOrdinalsFilter = function(test, common) { - test('has remove_ordinals filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.filter.remove_ordinals, 'object', 'there is an remove_ordinals filter'); - var filter = s.analysis.filter.remove_ordinals; - t.equal(filter.type, 'pattern_replace'); - t.equal(filter.pattern, '(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))'); - t.equal(filter.replacement, '$2$4$5$6$7$9$10$12$14$15$16$17$18'); - t.end(); - }); -}; +module.exports.tests.removeOrdinalsFilter = function (test, common) { + test('has remove_ordinals filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.filter.remove_ordinals, 'object', 'there is an remove_ordinals filter') + var filter = s.analysis.filter.remove_ordinals + t.equal(filter.type, 'pattern_replace') + t.equal(filter.pattern, '(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))') + t.equal(filter.replacement, '$2$4$5$6$7$9$10$12$14$15$16$17$18') + t.end() + }) +} // -- char filters -- // we use a custom punctuation filter in order to allow the ampersand // character which would otherwise be stripped by the standard tokenizer -module.exports.tests.punctuationCharFilter = function(test, common) { - test('has punctuation char_filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.char_filter.punctuation, 'object', 'there is a punctuation char_filter'); - var char_filter = s.analysis.char_filter.punctuation; - t.equal(char_filter.type, 'mapping'); - t.true(Array.isArray(char_filter.mappings)); - t.equal(char_filter.mappings.length, 48); - t.end(); - }); -}; +module.exports.tests.punctuationCharFilter = function (test, common) { + test('has punctuation char_filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.char_filter.punctuation, 'object', 'there is a punctuation char_filter') + var charFilter = s.analysis.char_filter.punctuation + t.equal(charFilter.type, 'mapping') + t.true(Array.isArray(charFilter.mappings)) + t.equal(charFilter.mappings.length, 48) + t.end() + }) +} // remove non alphanumeric characters -module.exports.tests.alphanumericCharFilter = function(test, common) { - test('has alphanumeric char_filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.char_filter.alphanumeric, 'object', 'there is a alphanumeric char_filter'); - var char_filter = s.analysis.char_filter.alphanumeric; - t.equal(char_filter.type, 'pattern_replace'); - t.equal(char_filter.pattern, '[^a-zA-Z0-9]'); - t.equal(char_filter.replacement, ''); - t.end(); - }); -}; +module.exports.tests.alphanumericCharFilter = function (test, common) { + test('has alphanumeric char_filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.char_filter.alphanumeric, 'object', 'there is a alphanumeric char_filter') + var charFilter = s.analysis.char_filter.alphanumeric + t.equal(charFilter.type, 'pattern_replace') + t.equal(charFilter.pattern, '[^a-zA-Z0-9]') + t.equal(charFilter.replacement, '') + t.end() + }) +} // replace non-numeric chars with a space -module.exports.tests.numericCharFilter = function(test, common) { - test('has numeric char_filter', function(t) { - var s = settings(); - t.equal(typeof s.analysis.char_filter.numeric, 'object', 'there is a numeric char_filter'); - var char_filter = s.analysis.char_filter.numeric; - t.equal(char_filter.type, 'pattern_replace'); - t.equal(char_filter.pattern, '[^0-9]'); - t.equal(char_filter.replacement, ' '); - t.end(); - }); -}; +module.exports.tests.numericCharFilter = function (test, common) { + test('has numeric char_filter', function (t) { + var s = settings() + t.equal(typeof s.analysis.char_filter.numeric, 'object', 'there is a numeric char_filter') + var charFilter = s.analysis.char_filter.numeric + t.equal(charFilter.type, 'pattern_replace') + t.equal(charFilter.pattern, '[^0-9]') + t.equal(charFilter.replacement, ' ') + t.end() + }) +} // -- etc -- // index should always be set -module.exports.tests.index = function(test, common) { - test('has index settings', function(t) { - var s = settings(); - t.equal(typeof s.index, 'object', 'index specified'); - t.equal(s.index.number_of_replicas, "0", 'replicas will increase index time'); - t.equal(s.index.number_of_shards, "5", 'sharding value should use the elasticsearch default'); - t.end(); - }); -}; +module.exports.tests.index = function (test, common) { + test('has index settings', function (t) { + var s = settings() + t.equal(typeof s.index, 'object', 'index specified') + t.equal(s.index.number_of_replicas, '0', 'replicas will increase index time') + t.equal(s.index.number_of_shards, '5', 'sharding value should use the elasticsearch default') + t.end() + }) +} // allow overrides from pelias/config -module.exports.tests.overrides = function(test, common) { - test('override defaults', function(t) { - - process.env['PELIAS_CONFIG'] = path.resolve(__dirname + '/fixtures/empty.json'); +module.exports.tests.overrides = function (test, common) { + test('override defaults', function (t) { + process.env['PELIAS_CONFIG'] = path.resolve(path.join(__dirname, '/fixtures/empty.json')) - var s = settings(); - t.equal(s.index['number_of_replicas'], '0', 'unchanged'); + var s = settings() + t.equal(s.index['number_of_replicas'], '0', 'unchanged') // set the PELIAS_CONFIG env var - process.env['PELIAS_CONFIG'] = path.resolve( __dirname + '/fixtures/config.json' ); + process.env['PELIAS_CONFIG'] = path.resolve(path.join(__dirname, '/fixtures/config.json')) - s = settings(); - t.equal(s.index['number_of_replicas'], '999', 'changed'); - t.end(); + s = settings() + t.equal(s.index['number_of_replicas'], '999', 'changed') + t.end() // unset the PELIAS_CONFIG env var - delete process.env['PELIAS_CONFIG']; - }); + delete process.env['PELIAS_CONFIG'] + }) test('override similarity', function (t) { + process.env['PELIAS_CONFIG'] = path.resolve(path.join(__dirname, '/fixtures/empty.json')) - process.env['PELIAS_CONFIG'] = path.resolve(__dirname + '/fixtures/empty.json'); - - var s = settings(); - t.equal(s.index.similarity.peliasDefaultSimilarity.k1, 1.2, 'unchanged'); + var s = settings() + t.equal(s.index.similarity.peliasDefaultSimilarity.k1, 1.2, 'unchanged') // set the PELIAS_CONFIG env var - process.env['PELIAS_CONFIG'] = path.resolve(__dirname + '/fixtures/similarity.json'); + process.env['PELIAS_CONFIG'] = path.resolve(path.join(__dirname, '/fixtures/similarity.json')) - s = settings(); - t.equal(s.index.similarity.peliasDefaultSimilarity.k1, 0, 'changed'); - t.equal(s.index.similarity.peliasDefaultSimilarity.b, 0.75, 'changed'); - t.end(); + s = settings() + t.equal(s.index.similarity.peliasDefaultSimilarity.k1, 0, 'changed') + t.equal(s.index.similarity.peliasDefaultSimilarity.b, 0.75, 'changed') + t.end() // unset the PELIAS_CONFIG env var - delete process.env['PELIAS_CONFIG']; - }); -}; + delete process.env['PELIAS_CONFIG'] + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('settings: ' + name, testFunction); + function test (name, testFunction) { + return tape('settings: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +} diff --git a/test/synonyms/parser.js b/test/synonyms/parser.js index 6ce2d201..43846dd5 100644 --- a/test/synonyms/parser.js +++ b/test/synonyms/parser.js @@ -1,85 +1,84 @@ -const parser = require('../../synonyms/parser'); +const parser = require('../../synonyms/parser') -module.exports.tests = {}; +module.exports.tests = {} -module.exports.tests.load = function(test, common) { - test('load: invalid file', function(t) { - t.throws(() => parser('/invalid/path'), /file not found/, 'invalid file'); - t.throws(() => parser('/tmp'), /file not found/, 'directory'); - t.end(); - }); -}; +module.exports.tests.load = function (test, common) { + test('load: invalid file', function (t) { + t.throws(() => parser('/invalid/path'), /file not found/, 'invalid file') + t.throws(() => parser('/tmp'), /file not found/, 'directory') + t.end() + }) +} -module.exports.tests.parse = function(test, common) { - test('empty file', function(t) { - t.deepEqual( parser.parse(``), [] ); - t.end(); - }); - test('comments and newlines', function(t) { - t.deepEqual( parser.parse(` +module.exports.tests.parse = function (test, common) { + test('empty file', function (t) { + t.deepEqual(parser.parse(``), []) + t.end() + }) + test('comments and newlines', function (t) { + t.deepEqual(parser.parse(` # foo bar # baz -`), [] ); - t.end(); - }); - test('lowercase', function(t) { - t.deepEqual( parser.parse(` +`), []) + t.end() + }) + test('lowercase', function (t) { + t.deepEqual(parser.parse(` Foo => BaR Foo,Bar,Baz `), [ - 'foo => bar', - 'foo,bar,baz' -] ); - t.end(); - }); - test('squash spaces', function(t) { - t.deepEqual( parser.parse(` + 'foo => bar', + 'foo,bar,baz' + ]) + t.end() + }) + test('squash spaces', function (t) { + t.deepEqual(parser.parse(` foo bar => foo Foo Bar, Foo `), [ - 'foo bar => foo', - 'foo bar,foo' -] ); - t.end(); - }); - test('trim commas', function(t) { - t.deepEqual( parser.parse(` + 'foo bar => foo', + 'foo bar,foo' + ]) + t.end() + }) + test('trim commas', function (t) { + t.deepEqual(parser.parse(` ,foo => bar ,foo, bar, `), [ - 'foo => bar', - 'foo,bar' -] ); - t.end(); - }); - test('trim around commas', function(t) { - t.deepEqual( parser.parse(` + 'foo => bar', + 'foo,bar' + ]) + t.end() + }) + test('trim around commas', function (t) { + t.deepEqual(parser.parse(` ,foo, bar , baz `), [ - 'foo,bar,baz' -] ); - t.end(); - }); - test('trim around arrows', function(t) { - t.deepEqual( parser.parse(` + 'foo,bar,baz' + ]) + t.end() + }) + test('trim around arrows', function (t) { + t.deepEqual(parser.parse(` foo => bar `), [ - 'foo => bar' -] ); - t.end(); - }); -}; + 'foo => bar' + ]) + t.end() + }) +} module.exports.all = function (tape, common) { - - function test(name, testFunction) { - return tape('synonyms parser: ' + name, testFunction); + function test (name, testFunction) { + return tape('synonyms parser: ' + name, testFunction) } - for( var testCase in module.exports.tests ){ - module.exports.tests[testCase](test, common); + for (var testCase in module.exports.tests) { + module.exports.tests[testCase](test, common) } -}; +}