Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
104 commits
Select commit Hold shift + click to select a range
b1b4547
Merge pull request #491 from NYPL/main
charmingduchess Jun 2, 2025
7685722
Override agg-self-filtering behavior for open search
nonword Jul 10, 2025
472331e
Merge branch 'NOREF-filter-self-filtering-fix' into qa2
nonword Jul 10, 2025
579eec5
Fix query structure
danamansana Dec 1, 2025
ae28b72
Add more permissive key structure
danamansana Dec 4, 2025
8b6000f
Remove console logs and commented code
danamansana Dec 4, 2025
e5b61dc
Fix linter errors
danamansana Dec 4, 2025
b85f3fb
Move esRangeValue
danamansana Dec 9, 2025
a02a058
Move parseParams
danamansana Dec 9, 2025
0beec4c
Refactor nyplSourc/id calculation and move nyplSourceAndId to utils'
danamansana Dec 9, 2025
a749878
Add bodybuilder methods for findByUri
danamansana Dec 9, 2025
683445f
Add nyplSourceAndId call to annotatedMarc
danamansana Dec 9, 2025
5ac57ee
Move itemsByFilter to utils
danamansana Dec 9, 2025
d50ea45
Move buildElasticQuery/Body to bodybuilder
danamansana Dec 9, 2025
2f6c887
Factor out body for search
danamansana Dec 9, 2025
1075d65
Move buildElasticAggregationsBody
danamansana Dec 9, 2025
a0237b7
Move aggregationQueriesForParams to bodybuilder
danamansana Dec 9, 2025
3340f5e
Move mergeAggregationsResposes to utils
danamansana Dec 9, 2025
e072d93
Factor out body for aggregation
danamansana Dec 9, 2025
054d1e4
Move findByUri to async/await
danamansana Dec 9, 2025
2d60b15
Make annotatedMarc async
danamansana Dec 9, 2025
2ec818a
Make deliveryLocationsByBarcode async
danamansana Dec 9, 2025
c784247
Merge pull request #597 from NYPL/NOREF/bnf-test
danamansana Dec 10, 2025
ba7fcf5
Pull search from promise chain in search
danamansana Dec 11, 2025
9189c1b
Pull massaged response from promise chain in resources#search
danamansana Dec 11, 2025
3297b63
Pull ResourceResultsSerializer.serialize from promise chain in search
danamansana Dec 11, 2025
03d57dc
Remove nested promise
danamansana Dec 11, 2025
1c34469
Remove promise from search
danamansana Dec 11, 2025
6e8af8d
Factor out relevance report
danamansana Dec 11, 2025
cf7edcd
Make aggregation endpoint async
danamansana Dec 11, 2025
6ca7e33
Exclude parentheses in query term
danamansana Dec 18, 2025
39c0920
Make keyphrase/non_ws_key lowercase
danamansana Dec 18, 2025
26e0a15
Merge pull request #606 from NYPL/NOREF/bnf-exclude-query-term-parent…
danamansana Dec 18, 2025
6714e24
Change callNumber to callnumber to enable callnumber searches
danamansana Dec 18, 2025
9534528
Merge pull request #607 from NYPL/NOREF/bnf-exclude-query-term-parent…
danamansana Dec 18, 2025
cd4c24f
Add finding text by key for atomic queries
danamansana Jan 7, 2026
eefeb44
Reorganize addInnerHits
danamansana Jan 9, 2026
d26553b
Reorganize bodyForFindByUri except for innerHits
danamansana Jan 9, 2026
7429ad6
Remove addInnerHits from findByUri
danamansana Jan 9, 2026
b386ec1
Reorganize buildElasticBody
danamansana Jan 9, 2026
36d56c5
Fix linting
danamansana Jan 13, 2026
d8249ac
Remove adding source to body in body for search
danamansana Jan 13, 2026
9353887
Add innerHits options
danamansana Jan 15, 2026
15a5e5c
Remove dependence on addInnerHits
danamansana Jan 15, 2026
b5530eb
Add tests for bodyForSearch; remove tests for addInnerHits
danamansana Jan 16, 2026
6cfbd49
Remove remaining references to addInnerHits
danamansana Jan 16, 2026
1c769c9
Add bodybuilder tests
danamansana Jan 16, 2026
c379ba5
Fix merge conflicts
danamansana Jan 20, 2026
a5e190e
Remove options from elastic-query
danamansana Jan 23, 2026
58a0d9f
Merge branch 'main' into scc-5168
danamansana Jan 29, 2026
8b3e4e1
Add initial bnf
danamansana Jan 30, 2026
87c1e32
Update packages
danamansana Jan 30, 2026
a55444d
Add alternate grammars and comment for atomic
danamansana Jan 30, 2026
05dfb3f
Add reverseGrammar and related methods
danamansana Feb 5, 2026
f373383
Apparently working left associating cql
danamansana Feb 5, 2026
951a8e8
Clean up grammar file
danamansana Feb 6, 2026
3a03d81
Use parseWithRightCql in query builder and tests
danamansana Feb 6, 2026
1952828
Remove console log and commented code'
danamansana Feb 6, 2026
7d1612e
Merge pull request #638 from NYPL/main
charmingduchess Feb 6, 2026
5d3bf7d
fix gha dep
charmingduchess Feb 6, 2026
b2a79b8
Merge branch 'SC-5164/delivery-inegration-test' into qa2
charmingduchess Feb 6, 2026
ff32619
gha
charmingduchess Feb 6, 2026
ffe1b30
update gha invocation
charmingduchess Feb 9, 2026
c389071
config aws in integration test
charmingduchess Feb 9, 2026
3a83857
Fix some param passing and start adding query tests
danamansana Feb 9, 2026
d2109f5
Add tests for atomic queries and some small corrections
danamansana Feb 10, 2026
baf803c
Add initial boolean tests
danamansana Feb 10, 2026
8842a88
Add tests for negation
danamansana Feb 11, 2026
fd8ff37
Fix linting/tests/small errors
danamansana Feb 11, 2026
271c692
Add date queries
danamansana Feb 19, 2026
ffe585f
Add filters to cql query builder
danamansana Feb 19, 2026
5990165
Remove irrelevant code
danamansana Feb 19, 2026
bd098d5
Merge branch 'main' into scc-5050-2
danamansana Feb 19, 2026
020c75d
Remove spurious merge code
danamansana Feb 19, 2026
7aec0dd
Merge pull request #649 from NYPL/merge-test
danamansana Feb 19, 2026
1473e3d
Fix some small errors
danamansana Feb 19, 2026
ed4c14a
Add initial filter implementation for cql
danamansana Feb 20, 2026
74f97a4
Fix tests
danamansana Feb 20, 2026
d5cee7f
Add date and filter features to cql
danamansana Feb 20, 2026
e2e43d0
Add some more useful display of parsing and errors
danamansana Feb 20, 2026
93c042e
Fix linting
danamansana Feb 20, 2026
9c98372
Merge pull request #650 from NYPL/scc-5203
danamansana Feb 27, 2026
0ebe3bd
Add new strategy for handling keyword vs text fields
danamansana Mar 2, 2026
b2ebcab
Add exact match query
danamansana Mar 2, 2026
15ddf08
Merge branch 'NOREF/nyql-for-qa2' into merge-nyql
danamansana Mar 10, 2026
d99276f
Merge pull request #665 from NYPL/qa2-nyql
danamansana Mar 10, 2026
f9a0250
Merge main
danamansana Mar 10, 2026
691b974
Fix deploy yaml
danamansana Mar 10, 2026
fd33073
Merge pull request #666 from NYPL/qa2-nyql
danamansana Mar 10, 2026
36e6302
Fixes in response to PR comments
danamansana Mar 13, 2026
a7d78e7
Merge branch 'main' into scc-5050-2
yossariano Mar 17, 2026
d5ec597
Add check for whether query has fields before adding
danamansana Mar 19, 2026
4f26b20
Remove console log
danamansana Mar 19, 2026
94d0d61
Fix tests
danamansana Mar 19, 2026
ca8a4d1
Merge pull request #686 from NYPL/scc-5168
danamansana Mar 19, 2026
e0e8f6b
Add reversing strings in nested array
danamansana Mar 19, 2026
34a399e
Merge pull request #687 from NYPL/scc-5168
danamansana Mar 19, 2026
d65a3d3
Remove double reversing in display
danamansana Mar 19, 2026
753ca86
Merge branch 'scc-5168' into qa2
danamansana Mar 19, 2026
330e170
Fix double nesting of should array for dates
danamansana Mar 20, 2026
b0f7585
Fix date test fixtures
danamansana Mar 20, 2026
d4c655a
Merge branch 'scc-5168' into qa2
danamansana Mar 20, 2026
14d7134
Merge branch 'qa2' into scc-5050-2
danamansana Mar 26, 2026
c59f33e
fix merge
yossariano Mar 27, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions .github/workflows/test-and-deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,37 @@ jobs:
run: npm ci
- name: Unit Tests
run: npm test
integration-test-qa:
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
needs:
- tests
if: github.ref == 'refs/heads/qa'
steps:
- uses: actions/checkout@v4
- name: Set Node version
uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
- name: Install dependencies
run: npm ci
- name: Start service
run: ENV=qa npm start &
- name: Run tests
run: node test/integration/delivery-locations-by-barcode.test.js
deploy-qa:
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
needs:
- tests
if: github.ref == 'refs/heads/qa'
steps:
- name: Checkout repo
uses: actions/checkout@v3

deploy:
permissions:
Expand Down
44 changes: 42 additions & 2 deletions lib/elasticsearch/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ const SEARCH_SCOPES = {
},
standard_number: {
// We do custom field matching for this search-scope
}
},
cql: {} // see cql/index_mapping for this search scope
}

const FILTER_CONFIG = {
Expand Down Expand Up @@ -127,8 +128,47 @@ const AGGREGATIONS_SPEC = {
collection: { terms: { field: 'collectionIds' } }
}

const ITEM_FILTER_AGGREGATIONS = {
item_location: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.holdingLocation_packed' } } } },
item_status: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.status_packed' } } } },
item_format: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.formatLiteral' } } } }
}

// Configure sort fields:
const SORT_FIELDS = {
title: {
initialDirection: 'asc',
field: 'title_sort'
},
date: {
initialDirection: 'desc',
field: 'dateStartYear'
},
creator: {
initialDirection: 'asc',
field: 'creator_sort'
},
relevance: {}
}

// The following fields can be excluded from ES responses because we don't pass them to client:
const EXCLUDE_FIELDS = [
'uris',
'*_packed',
'*_sort',
'items.*_packed',
'contentsTitle',
'suppressed',
// Hide contributor and creator transformed fields:
'*WithoutDates',
'*Normalized'
]

module.exports = {
SEARCH_SCOPES,
FILTER_CONFIG,
AGGREGATIONS_SPEC
AGGREGATIONS_SPEC,
ITEM_FILTER_AGGREGATIONS,
EXCLUDE_FIELDS,
SORT_FIELDS
}
138 changes: 138 additions & 0 deletions lib/elasticsearch/cql/index-mapping.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
const indexMapping = {
keyword: {
fields: [
'title',
'title.folded',
'description.foldedStemmed',
'subjectLiteral',
'subjectLiteral.folded',
'creatorLiteral',
'creatorLiteral.folded',
'contributorLiteral.folded',
'note.label.foldedStemmed',
'publisherLiteral.folded',
'seriesStatement.folded',
'titleAlt.folded',
'titleDisplay.folded',
'contentsTitle.folded',
'tableOfContents.folded',
'genreForm',
'donor.folded',
'parallelTitle.folded',
'parallelTitleDisplay.folded',
'parallelTitleAlt.folded',
'parallelSeriesStatement.folded',
'parallelCreatorLiteral.folded',
'parallelPublisher',
'parallelPublisherLiteral',
'uniformTitle.folded',
'parallelUniformTitle',
'formerTitle',
'addedAuthorTitle',
'placeOfPublication.folded',
// Try to detect shelfmark searches (e.g. JFD 16-5143)
{ field: 'items.shelfMark', on: (q) => /^[A-Z]{1,3} \d{2,}/.test(q) }
],
exact_fields: [
'title.keywordLowercasedStripped',
// missing description
'subjectLiteral.raw',
'creatorLiteral.keywordLowercased',
'contributorLiteral.keywordLowercased',
// note.label is missing
'publisherLiteral.raw',
'seriesStatement.raw',
'titleAlt.raw',
// titleDisplay missing
// contentsTitle missing
// tableOfContents missing
'genreForm.raw',
'donor.raw',
// parallelTitle missing
// parallelTitleDisplay missing
'parallelTitleAlt.raw',
'parallelSeriesStatement.raw',
'parallelCreatorLiteral.raw',
// parallelPublisher/parallelPublisherLiteral missing
'uniformTitle.raw',
'parallelUniformTitle.raw',
// formerTitle missing
'addedAuthorTitle.raw',
'placeOfPublication',
{ field: 'items.shelfMark.raw', on: (q) => /^[A-Z]{1,3} \d{2,}/.test(q) }
],
term: [
{ field: 'items.idBarcode', on: (q) => /\d{6,}/.test(q) }
]
},
title: {
fields: [
'title',
'title.folded',
'titleAlt.folded',
'uniformTitle.folded',
'titleDisplay.folded',
'seriesStatement.folded',
'contentsTitle.folded',
'donor.folded',
'parallelTitle.folded',
'parallelTitleDisplay.folded',
'parallelSeriesStatement.folded',
'parallelTitleAlt.folded',
'parallelCreatorLiteral.folded',
'parallelUniformTitle',
'formerTitle',
'addedAuthorTitle'
],
exact_fields: [
'title.keywordLowercasedStripped',
'seriesStatement.raw',
'titleAlt.raw',
// titleDisplay missing
// contentsTitle missing
// tableOfContents missing
'donor.raw',
// parallelTitle missing
// parallelTitleDisplay missing
'parallelTitleAlt.raw',
'parallelSeriesStatement.raw',
'parallelCreatorLiteral.raw',
'uniformTitle.raw',
'parallelUniformTitle.raw',
// formerTitle missing
'addedAuthorTitle.raw',
'placeOfPublication'
]
},
author: {
fields: ['creatorLiteral', 'creatorLiteral.folded', 'contributorLiteral.folded', 'parallelCreatorLiteral.folded', 'parallelContributorLiteral.folded'],
exact_fields: [
'creatorLiteral.keywordLowercased', 'contributorLiteral.keywordLowercased',
'parallelCreatorLiteral.raw', 'parallelContributorLiteral.raw'
]
},
callnumber: {
term: ['shelfMark.keywordLowercased', 'items.shelfMark.keywordLowercased']
},
identifier: {
prefix: ['identifierV2.value', 'items.shelfMark.keywordLowercased'],
term: ['uri', 'items.idBarcode', 'idIsbn.clean', 'idIssn.clean']
},
subject: {
fields: ['subjectLiteral', 'subjectLiteral.folded', 'parallelSubjectLiteral.folded'],
exact_fields: ['subjectLiteral.raw']
},
language: { term: ['language.id', 'language.label'] },
date: { fields: ['dates.range'] },
series: {
term: ['series', 'parallelSeries']
},
genre: { fields: ['genreForm'], exact_fields: ['genreForm.raw'] },
center: { term: ['buildingLocationIds'] },
division: { term: ['collectionIds'] },
format: { term: ['formatId'] }
}

module.exports = {
indexMapping
}
130 changes: 130 additions & 0 deletions lib/elasticsearch/cql_grammar.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
const { Grammars } = require('ebnf')

function reverseGrammar (grammar) {
return grammar.split('\n')
.map(line =>
(line.split('::=')
.map(side =>
(side.split('|')
.map(dis =>
(dis.split(' ')
.map(word =>
(word.includes('"') ? word.split('').reverse().join('') : word))
.reverse().join(' '))
).join('|'))).join('::= '))).join('\n')
}

const leftCql = `
query ::= query whitespace connective whitespace sub_query | sub_query
connective ::= "AND NOT" | "AND" | "OR" | "NOT"
sub_query ::= atomic_query | "(" query ")"
atomic_query ::= scope relation quoted_term
scope ::= scope_term whitespace | scope_term
relation ::= relation_term whitespace | relation_term
scope_term ::= "title" | "author" | "keyword" | "callnumber" | "identifier" | "subject" | "language" | "date" | "series"| "genre" | "center" | "division" | "format"
relation_term ::= "any" | "adj" | "all" | "<=" | ">=" | "<" | ">" | "==" | "=" | "within" | "encloses"
quoted_term ::= quote phrase quote
phrase ::= phrase whitespace word | word
whitespace ::= [#x20#x09#x0A#x0D]+
word ::= word escaped_char | word regular_char | escaped_char | regular_char
regular_char ::= [^#x22#x5c#x20#x09#x0A#x0D]
escaped_char ::= slash char
slash ::= [#x5c]
char ::= [a-z]|[^a-z]
quote ::= [#x22]
`

const rightCql = reverseGrammar(leftCql)

function simplify (ast) {
switch (ast.type) {
case 'query': {
const children = ast.children.filter(child => child.type !== 'whitespace').map(child => simplify(child))
return children.length > 1 ? children : children[0]
}
case 'connective':
return ast.text
case 'sub_query':
return simplify(ast.children.find(child => child.type.includes('query')))
case 'atomic_query':
return ast.children.map(child => simplify(child))
case 'scope':
return simplify(ast.children.find(child => child.type.includes('scope_term')))
case 'relation':
return simplify(ast.children.find(child => child.type.includes('relation_term')))
case 'scope_term':
return ast.text
case 'relation_term':
return ast.text
case 'quoted_term':
return simplify(ast.children.find(child => child.type.includes('phrase')))
case 'phrase': {
const word = ast.children.find(child => child.type === 'word')
const phrase = ast.children.find(child => child.type === 'phrase')
return [simplify(word)].concat(phrase ? simplify(phrase) : [])
}
case 'word':
return ast.text
default:
break
}
}

function reverseString (string) {
return string.split('').reverse().join('')
}

function reverseAST (tree) {
if (!tree) return null
tree.text = reverseString(tree.text)
tree.children = tree.children.map(child => reverseAST(child)).reverse()
return tree
}

const rightCqlParser = new Grammars.W3C.Parser(rightCql)

// we want to associate operators to the left, but we have a right parser.
// so: reverse the grammar and the input string, then reverse the output
function parseRight (string, parser) {
return reverseAST(parser.getAST(reverseString(string)))
}
function parseWithRightCql (string) {
return parseRight(string, rightCqlParser)
}

function parsedASTtoNestedArray (ast) {
if (!ast.type.includes('query')) {
return reverseString(ast.text)
}

const childTypes = [
'atomic_query', 'sub_query', 'query', 'connective',
'scope', 'relation', 'quoted_term'
]

const children = ast.children
.filter(child => childTypes.includes(child.type))
.map(child => parsedASTtoNestedArray(child))

if (children.length === 1) {
return children[0]
}

return children
}

// we need to reverse the error message since `parseWithRightCql` doesn't
function displayParsed (string) {
const parsed = parseWithRightCql(string)
if (!parsed) return {}
if (parsed.errors.length) {
return {
error: parsed.errors.map(error =>
`Parsing error likely near end of "${reverseString(error.token.rest)}"`
).join('\n')
}
}
return { parsed: parsedASTtoNestedArray(parsed) }
}

module.exports = { simplify, reverseAST, reverseGrammar, parseRight, parseWithRightCql, rightCqlParser, reverseString, displayParsed }
Loading
Loading