do not merge - integration test verification

.github/workflows/integration-tests.yml

@@ -0,0 +1,30 @@
+name: integration-test
+
+on:
+  pull_request:
+    branches:
+      - production
+
+jobs:
+  integration-test:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
+          aws-region: us-east-1
+      - uses: actions/checkout@v3
+      - name: Set Node version
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+      - name: npm install
+        run: npm i
+      - name: Run integration tests
+        env: 
+          ENV: 'qa'
+        run: node test/integration/delivery-locations-by-barcode.test.js

.github/workflows/test-and-deploy.yml

@@ -2,6 +2,11 @@ name: Unit Tests
 
 on: [push]
 
+# Global environment variable based on the branch name
+env:
+  ENV_TAG: ${{ github.ref_name }}
+  ECR_REPOSITORY: discovery-api
+
 jobs:
   tests:
     runs-on: ubuntu-latest
@@ -15,114 +20,15 @@ jobs:
         run: npm ci
       - name: Unit Tests
         run: npm test
-  integration-test-qa:
-    permissions:
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs: tests
-    if: github.ref == 'refs/heads/qa'
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set Node version
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: '.nvmrc'
-      - name: Install dependencies
-        run: npm ci
-      - name: Start service
-        run: ENV=qa npm start &
-      - name: Run tests
-        run: npm run test-integration
-  deploy-qa:
-    permissions:
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs:
-      - tests
-    if: github.ref == 'refs/heads/qa'
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
-          aws-region: us-east-1
-      - name: Log in to ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Back up previous image for rollback
-        env:
-          ECR_REPOSITORY: discovery-api
-        run: |
-          MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="qa-latest" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          PREVIOUS_MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="qa-previous" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          if [ "$MANIFEST" != "$PREVIOUS_MANIFEST" ]; then aws ecr put-image --repository-name $ECR_REPOSITORY --image-tag "qa-previous" --image-manifest "$MANIFEST"; fi
-      - name: Build, tag, and push image to Amazon ECR
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: discovery-api
-          IMAGE_TAG: ${{ github.sha }}
-        run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:qa-latest
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:qa-latest
-      - name: Force ECS Update
-        run: |
-          aws ecs update-service --cluster discovery-api-qa --service discovery-api-qa --force-new-deployment
-  deploy-qa2:
-    permissions:
-      id-token: write
-      contents: read
-    runs-on: ubuntu-latest
-    needs:
-      - tests
-    if: github.ref == 'refs/heads/qa2'
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v2
-        with:
-          role-to-assume: arn:aws:iam::946183545209:role/GithubActionsDeployerRole
-          aws-region: us-east-1
-
-      - name: Log in to ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
-      - name: Back up previous image for rollback
-        env:
-          ECR_REPOSITORY: discovery-api
-        run: |
-          MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="qa2-latest" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          PREVIOUS_MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="qa2-previous" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          if [ "$MANIFEST" != "$PREVIOUS_MANIFEST" ]; then aws ecr put-image --repository-name $ECR_REPOSITORY --image-tag "qa2-previous" --image-manifest "$MANIFEST"; fi
-      - name: Build, tag, and push image to Amazon ECR
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: discovery-api
-          IMAGE_TAG: ${{ github.sha }}
-        run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:qa2-latest
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:qa2-latest
-
-      - name: Force ECS Update
-        run: |
-          aws ecs update-service --cluster discovery-api-qa2 --service discovery-api-qa2 --force-new-deployment
-  deploy-production:
+  deploy:
     permissions:
       id-token: write
       contents: read
     runs-on: ubuntu-latest
     needs: tests
-    if: github.ref == 'refs/heads/production'
+    # Only run if it's one of our three deployment branches
+    if: contains(fromJSON('["qa", "qa2", "production"]'), github.ref_name)
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -138,22 +44,23 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Back up previous image for rollback
-        env:
-          ECR_REPOSITORY: discovery-api
         run: |
-          MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="production-latest" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          PREVIOUS_MANIFEST=$(aws ecr batch-get-image --repository-name $ECR_REPOSITORY --image-ids imageTag="production-previous" --output json | jq --raw-output --join-output '.images[0].imageManifest')
-          if [ "$MANIFEST" != "$PREVIOUS_MANIFEST" ]; then aws ecr put-image --repository-name $ECR_REPOSITORY --image-tag "production-previous" --image-manifest "$MANIFEST"; fi
+          MANIFEST=$(aws ecr batch-get-image --repository-name ${{ env.ECR_REPOSITORY }} --image-ids imageTag="${{ env.ENV_TAG }}-latest" --output json | jq --raw-output --join-output '.images[0].imageManifest')
+          PREVIOUS_MANIFEST=$(aws ecr batch-get-image --repository-name ${{ env.ECR_REPOSITORY }} --image-ids imageTag="${{ env.ENV_TAG }}-previous" --output json | jq --raw-output --join-output '.images[0].imageManifest')
+          if [ "$MANIFEST" != "$PREVIOUS_MANIFEST" ]; then 
+            aws ecr put-image --repository-name ${{ env.ECR_REPOSITORY }} --image-tag "${{ env.ENV_TAG }}-previous" --image-manifest "$MANIFEST"
+          fi
+
       - name: Build, tag, and push image to Amazon ECR
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: discovery-api
           IMAGE_TAG: ${{ github.sha }}
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:production-latest
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:production-latest
+          docker build -t $ECR_REGISTRY/${{ env.ECR_REPOSITORY }}:$IMAGE_TAG .
+          docker push $ECR_REGISTRY/${{ env.ECR_REPOSITORY }}:$IMAGE_TAG
+          docker tag $ECR_REGISTRY/${{ env.ECR_REPOSITORY }}:$IMAGE_TAG $ECR_REGISTRY/${{ env.ECR_REPOSITORY }}:${{ env.ENV_TAG }}-latest
+          docker push $ECR_REGISTRY/${{ env.ECR_REPOSITORY }}:${{ env.ENV_TAG }}-latest
+
       - name: Force ECS Update
         run: |
-          aws ecs update-service --cluster discovery-api-production --service discovery-api-production --force-new-deployment
+          aws ecs update-service --cluster discovery-api-${{ env.ENV_TAG }} --service discovery-api-${{ env.ENV_TAG }} --force-new-deployment

config/production.env

@@ -10,7 +10,7 @@ NYPL_OAUTH_URL=https://isso.nypl.org/
 ENCRYPTED_NYPL_OAUTH_ID=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGswaQYJKoZIhvcNAQcGoFwwWgIBADBVBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMLKVUQA58B6vprNcAIBEIAoaz0lI9EL2M9NyTuEwT8JDmPBt6aXfMiFs027DEuwsCN0wS0qWeFL1g==
 ENCRYPTED_NYPL_OAUTH_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIcwgYQGCSqGSIb3DQEHBqB3MHUCAQAwcAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyWz91LOP2YP5fg0q0CARCAQ9inO9SV1M8R0Pkkx84r7UdwlU1FxfXvIjk/z6Qs81KBAVELhby2iD5LawQyDrR9tjhuMbotS6QnydwwMR/p8+qJXHI=
 
-NYPL_CORE_VERSION=v2.37
+NYPL_CORE_VERSION=v2.39
 
 LOG_LEVEL=info
 FEATURES=on-site-edd

config/qa.env

@@ -12,7 +12,7 @@ NYPL_OAUTH_URL=https://isso.nypl.org/
 ENCRYPTED_NYPL_OAUTH_ID=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAGswaQYJKoZIhvcNAQcGoFwwWgIBADBVBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMLKVUQA58B6vprNcAIBEIAoaz0lI9EL2M9NyTuEwT8JDmPBt6aXfMiFs027DEuwsCN0wS0qWeFL1g==
 ENCRYPTED_NYPL_OAUTH_SECRET=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAIcwgYQGCSqGSIb3DQEHBqB3MHUCAQAwcAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyWz91LOP2YP5fg0q0CARCAQ9inO9SV1M8R0Pkkx84r7UdwlU1FxfXvIjk/z6Qs81KBAVELhby2iD5LawQyDrR9tjhuMbotS6QnydwwMR/p8+qJXHI=
 
-NYPL_CORE_VERSION=v2.37
+NYPL_CORE_VERSION=v2.39
 
 LOG_LEVEL=debug
 FEATURES=on-site-edd

config/test.env

@@ -10,7 +10,7 @@ NYPL_OAUTH_URL=http://oauth.example.com
 NYPL_OAUTH_ID=encrypted-nypl-oauth-id
 NYPL_OAUTH_SECRET=encrypted-nypl-oauth-id
 
-NYPL_CORE_VERSION=v2.37
+NYPL_CORE_VERSION=v2.39
 
 LOG_LEVEL=error
 FEATURES=on-site-edd

lib/contributors.js

@@ -100,21 +100,21 @@ module.exports = function (app, _private = null) {
 }
 
 /**
- * Builds an aggregation query that checks the resource index for counts on the contributorRoleLiteral field for a list of contributors.
+ * Builds an aggregation query that checks the resource index for counts on the browseableContributorRole_packed field for a list of contributors.
  */
 const buildElasticRoleCountQuery = function (contributorList) {
   return {
     size: 0,
     query: {
       terms: {
-        contributorRoleLiteral: contributorList
+        browseableContributorRole_packed: contributorList
       }
     },
     aggs: {
       contributor_role: {
         terms: {
           script: {
-            source: 'def results = []; for (val in doc["contributorRoleLiteral"]) { int pos = val.indexOf("||"); if (pos != -1) { String name = val.substring(0, pos); if (params.targets.contains(name)) { results.add(val); } } } return results;',
+            source: 'def results = []; for (val in doc["browseableContributorRole_packed"]) { int pos = val.indexOf("||"); if (pos != -1) { String name = val.substring(0, pos); if (params.targets.contains(name)) { results.add(val); } } } return results;',
             params: {
               targets: contributorList
             }

lib/display-field-unpacker.js

@@ -0,0 +1,25 @@
+const parseValueAndLabel = (delimitedString) => {
+  if (!delimitedString.includes('||')) {
+    return { value: delimitedString, display: null }
+  }
+  const [value, display] = delimitedString.split('||')
+  return { value, display }
+}
+
+module.exports = (elasticSearchResponse) => {
+  elasticSearchResponse.hits.hits.forEach((bib) => {
+    // Contributors and creators are packed like so <name>||<display label> where <display label>
+    // can have prefix, title, and roles. We'd like to unpack them in a friendly format for the frontend
+    // to display the full label and use the isolated name for link-building
+    Object.entries(bib._source).forEach(([key, value]) => {
+      if (key.endsWith('_displayPacked')) {
+        const fieldName = key.replace('_displayPacked', '')
+        bib._source[fieldName + 'Display'] = value.map((packedValue) => parseValueAndLabel(packedValue))
+        delete bib._source[key]
+      }
+    })
+
+    return bib
+  })
+  return elasticSearchResponse
+}

lib/elasticsearch/config.js

@@ -73,7 +73,8 @@ const SEARCH_SCOPES = {
   },
   standard_number: {
     // We do custom field matching for this search-scope
-  }
+  },
+  cql: {} // see cql/index_mapping for this search scope
 }
 
 const FILTER_CONFIG = {
@@ -89,7 +90,7 @@ const FILTER_CONFIG = {
   mediaType: { operator: 'match', field: ['mediaType.id', 'mediaType.label'], repeatable: true },
   carrierType: { operator: 'match', field: ['carrierType.id', 'carrierType.label'], repeatable: true },
   publisher: { operator: 'match', field: ['publisherLiteral.raw'], repeatable: true },
-  contributorLiteral: { operator: 'match', field: ['contributorLiteral.keywordLowercased', 'parallelContributor.raw', 'creatorLiteral.keywordLowercased', 'parallelCreatorLiteral.raw'], repeatable: true },
+  contributorLiteral: { operator: 'match', field: ['contributorLiteral.keywordLowercased', 'parallelContributorLiteral.raw', 'creatorLiteral.keywordLowercased', 'parallelCreatorLiteral.raw'], repeatable: true },
   creatorLiteral: { operator: 'match', field: ['creatorLiteral.raw', 'parallelCreatorLiteral.raw'], repeatable: true },
   issuance: { operator: 'match', field: ['issuance.id', 'issuance.label'], repeatable: true },
   createdYear: { operator: 'match', field: ['createdYear'], repeatable: true },