Scc 5050 2

.github/workflows/test-and-deploy.yml

@@ -20,6 +20,37 @@ jobs:
         run: npm ci
       - name: Unit Tests
         run: npm test
+  integration-test-qa:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    needs:
+      - tests
+    if: github.ref == 'refs/heads/qa'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set Node version
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+      - name: Install dependencies
+        run: npm ci
+      - name: Start service
+        run: ENV=qa npm start &
+      - name: Run tests
+        run: node test/integration/delivery-locations-by-barcode.test.js
+  deploy-qa:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    needs:
+      - tests
+    if: github.ref == 'refs/heads/qa'
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
 
   deploy:
     permissions:

lib/elasticsearch/config.js

@@ -73,7 +73,8 @@ const SEARCH_SCOPES = {
   },
   standard_number: {
     // We do custom field matching for this search-scope
-  }
+  },
+  cql: {} // see cql/index_mapping for this search scope
 }
 
 const FILTER_CONFIG = {
@@ -127,8 +128,47 @@ const AGGREGATIONS_SPEC = {
   collection: { terms: { field: 'collectionIds' } }
 }
 
+const ITEM_FILTER_AGGREGATIONS = {
+  item_location: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.holdingLocation_packed' } } } },
+  item_status: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.status_packed' } } } },
+  item_format: { nested: { path: 'items' }, aggs: { _nested: { terms: { size: 100, field: 'items.formatLiteral' } } } }
+}
+
+// Configure sort fields:
+const SORT_FIELDS = {
+  title: {
+    initialDirection: 'asc',
+    field: 'title_sort'
+  },
+  date: {
+    initialDirection: 'desc',
+    field: 'dateStartYear'
+  },
+  creator: {
+    initialDirection: 'asc',
+    field: 'creator_sort'
+  },
+  relevance: {}
+}
+
+// The following fields can be excluded from ES responses because we don't pass them to client:
+const EXCLUDE_FIELDS = [
+  'uris',
+  '*_packed',
+  '*_sort',
+  'items.*_packed',
+  'contentsTitle',
+  'suppressed',
+  // Hide contributor and creator transformed fields:
+  '*WithoutDates',
+  '*Normalized'
+]
+
 module.exports = {
   SEARCH_SCOPES,
   FILTER_CONFIG,
-  AGGREGATIONS_SPEC
+  AGGREGATIONS_SPEC,
+  ITEM_FILTER_AGGREGATIONS,
+  EXCLUDE_FIELDS,
+  SORT_FIELDS
 }

lib/elasticsearch/cql/index-mapping.js

@@ -0,0 +1,138 @@
+const indexMapping = {
+  keyword: {
+    fields: [
+      'title',
+      'title.folded',
+      'description.foldedStemmed',
+      'subjectLiteral',
+      'subjectLiteral.folded',
+      'creatorLiteral',
+      'creatorLiteral.folded',
+      'contributorLiteral.folded',
+      'note.label.foldedStemmed',
+      'publisherLiteral.folded',
+      'seriesStatement.folded',
+      'titleAlt.folded',
+      'titleDisplay.folded',
+      'contentsTitle.folded',
+      'tableOfContents.folded',
+      'genreForm',
+      'donor.folded',
+      'parallelTitle.folded',
+      'parallelTitleDisplay.folded',
+      'parallelTitleAlt.folded',
+      'parallelSeriesStatement.folded',
+      'parallelCreatorLiteral.folded',
+      'parallelPublisher',
+      'parallelPublisherLiteral',
+      'uniformTitle.folded',
+      'parallelUniformTitle',
+      'formerTitle',
+      'addedAuthorTitle',
+      'placeOfPublication.folded',
+      // Try to detect shelfmark searches (e.g. JFD 16-5143)
+      { field: 'items.shelfMark', on: (q) => /^[A-Z]{1,3} \d{2,}/.test(q) }
+    ],
+    exact_fields: [
+      'title.keywordLowercasedStripped',
+      // missing description
+      'subjectLiteral.raw',
+      'creatorLiteral.keywordLowercased',
+      'contributorLiteral.keywordLowercased',
+      // note.label is missing
+      'publisherLiteral.raw',
+      'seriesStatement.raw',
+      'titleAlt.raw',
+      // titleDisplay missing
+      // contentsTitle missing
+      // tableOfContents missing
+      'genreForm.raw',
+      'donor.raw',
+      // parallelTitle missing
+      // parallelTitleDisplay missing
+      'parallelTitleAlt.raw',
+      'parallelSeriesStatement.raw',
+      'parallelCreatorLiteral.raw',
+      // parallelPublisher/parallelPublisherLiteral missing
+      'uniformTitle.raw',
+      'parallelUniformTitle.raw',
+      // formerTitle missing
+      'addedAuthorTitle.raw',
+      'placeOfPublication',
+      { field: 'items.shelfMark.raw', on: (q) => /^[A-Z]{1,3} \d{2,}/.test(q) }
+    ],
+    term: [
+      { field: 'items.idBarcode', on: (q) => /\d{6,}/.test(q) }
+    ]
+  },
+  title: {
+    fields: [
+      'title',
+      'title.folded',
+      'titleAlt.folded',
+      'uniformTitle.folded',
+      'titleDisplay.folded',
+      'seriesStatement.folded',
+      'contentsTitle.folded',
+      'donor.folded',
+      'parallelTitle.folded',
+      'parallelTitleDisplay.folded',
+      'parallelSeriesStatement.folded',
+      'parallelTitleAlt.folded',
+      'parallelCreatorLiteral.folded',
+      'parallelUniformTitle',
+      'formerTitle',
+      'addedAuthorTitle'
+    ],
+    exact_fields: [
+      'title.keywordLowercasedStripped',
+      'seriesStatement.raw',
+      'titleAlt.raw',
+      // titleDisplay missing
+      // contentsTitle missing
+      // tableOfContents missing
+      'donor.raw',
+      // parallelTitle missing
+      // parallelTitleDisplay missing
+      'parallelTitleAlt.raw',
+      'parallelSeriesStatement.raw',
+      'parallelCreatorLiteral.raw',
+      'uniformTitle.raw',
+      'parallelUniformTitle.raw',
+      // formerTitle missing
+      'addedAuthorTitle.raw',
+      'placeOfPublication'
+    ]
+  },
+  author: {
+    fields: ['creatorLiteral', 'creatorLiteral.folded', 'contributorLiteral.folded', 'parallelCreatorLiteral.folded', 'parallelContributorLiteral.folded'],
+    exact_fields: [
+      'creatorLiteral.keywordLowercased', 'contributorLiteral.keywordLowercased',
+      'parallelCreatorLiteral.raw', 'parallelContributorLiteral.raw'
+    ]
+  },
+  callnumber: {
+    term: ['shelfMark.keywordLowercased', 'items.shelfMark.keywordLowercased']
+  },
+  identifier: {
+    prefix: ['identifierV2.value', 'items.shelfMark.keywordLowercased'],
+    term: ['uri', 'items.idBarcode', 'idIsbn.clean', 'idIssn.clean']
+  },
+  subject: {
+    fields: ['subjectLiteral', 'subjectLiteral.folded', 'parallelSubjectLiteral.folded'],
+    exact_fields: ['subjectLiteral.raw']
+  },
+  language: { term: ['language.id', 'language.label'] },
+  date: { fields: ['dates.range'] },
+  series: {
+    term: ['series', 'parallelSeries']
+  },
+  genre: { fields: ['genreForm'], exact_fields: ['genreForm.raw'] },
+  center: { term: ['buildingLocationIds'] },
+  division: { term: ['collectionIds'] },
+  format: { term: ['formatId'] }
+}
+
+module.exports = {
+  indexMapping
+}

lib/elasticsearch/cql_grammar.js

@@ -0,0 +1,130 @@
+const { Grammars } = require('ebnf')
+
+function reverseGrammar (grammar) {
+  return grammar.split('\n')
+    .map(line =>
+      (line.split('::=')
+        .map(side =>
+          (side.split('|')
+            .map(dis =>
+              (dis.split(' ')
+                .map(word =>
+                  (word.includes('"') ? word.split('').reverse().join('') : word))
+                .reverse().join(' '))
+            ).join('|'))).join('::= '))).join('\n')
+}
+
+const leftCql = `
+ query ::=  query whitespace connective whitespace sub_query | sub_query
+ connective ::= "AND NOT" | "AND" | "OR" | "NOT"
+ sub_query ::= atomic_query | "(" query ")"
+ atomic_query ::= scope relation quoted_term
+ scope ::= scope_term whitespace | scope_term
+ relation ::= relation_term whitespace | relation_term
+ scope_term ::= "title" | "author" | "keyword" | "callnumber" | "identifier" | "subject" | "language" | "date" | "series"| "genre" | "center" | "division" | "format"
+ relation_term ::= "any" | "adj" | "all" | "<=" | ">=" | "<" | ">" | "==" | "=" | "within" | "encloses"
+ quoted_term ::= quote phrase quote
+ phrase ::= phrase whitespace word | word
+ whitespace ::= [#x20#x09#x0A#x0D]+
+ word ::= word escaped_char | word regular_char | escaped_char | regular_char
+ regular_char ::= [^#x22#x5c#x20#x09#x0A#x0D]
+ escaped_char ::= slash char
+ slash ::= [#x5c]
+ char ::= [a-z]|[^a-z]
+ quote ::= [#x22]
+`
+
+const rightCql = reverseGrammar(leftCql)
+
+function simplify (ast) {
+  switch (ast.type) {
+    case 'query': {
+      const children = ast.children.filter(child => child.type !== 'whitespace').map(child => simplify(child))
+      return children.length > 1 ? children : children[0]
+    }
+    case 'connective':
+      return ast.text
+    case 'sub_query':
+      return simplify(ast.children.find(child => child.type.includes('query')))
+    case 'atomic_query':
+      return ast.children.map(child => simplify(child))
+    case 'scope':
+      return simplify(ast.children.find(child => child.type.includes('scope_term')))
+    case 'relation':
+      return simplify(ast.children.find(child => child.type.includes('relation_term')))
+    case 'scope_term':
+      return ast.text
+    case 'relation_term':
+      return ast.text
+    case 'quoted_term':
+      return simplify(ast.children.find(child => child.type.includes('phrase')))
+    case 'phrase': {
+      const word = ast.children.find(child => child.type === 'word')
+      const phrase = ast.children.find(child => child.type === 'phrase')
+      return [simplify(word)].concat(phrase ? simplify(phrase) : [])
+    }
+    case 'word':
+      return ast.text
+    default:
+      break
+  }
+}
+
+function reverseString (string) {
+  return string.split('').reverse().join('')
+}
+
+function reverseAST (tree) {
+  if (!tree) return null
+  tree.text = reverseString(tree.text)
+  tree.children = tree.children.map(child => reverseAST(child)).reverse()
+  return tree
+}
+
+const rightCqlParser = new Grammars.W3C.Parser(rightCql)
+
+// we want to associate operators to the left, but we have a right parser.
+// so: reverse the grammar and the input string, then reverse the output
+function parseRight (string, parser) {
+  return reverseAST(parser.getAST(reverseString(string)))
+}
+function parseWithRightCql (string) {
+  return parseRight(string, rightCqlParser)
+}
+
+function parsedASTtoNestedArray (ast) {
+  if (!ast.type.includes('query')) {
+    return reverseString(ast.text)
+  }
+
+  const childTypes = [
+    'atomic_query', 'sub_query', 'query', 'connective',
+    'scope', 'relation', 'quoted_term'
+  ]
+
+  const children = ast.children
+    .filter(child => childTypes.includes(child.type))
+    .map(child => parsedASTtoNestedArray(child))
+
+  if (children.length === 1) {
+    return children[0]
+  }
+
+  return children
+}
+
+// we need to reverse the error message since `parseWithRightCql` doesn't
+function displayParsed (string) {
+  const parsed = parseWithRightCql(string)
+  if (!parsed) return {}
+  if (parsed.errors.length) {
+    return {
+      error: parsed.errors.map(error =>
+        `Parsing error likely near end of "${reverseString(error.token.rest)}"`
+      ).join('\n')
+    }
+  }
+  return { parsed: parsedASTtoNestedArray(parsed) }
+}
+
+module.exports = { simplify, reverseAST, reverseGrammar, parseRight, parseWithRightCql, rightCqlParser, reverseString, displayParsed }