feat: get response also when no result was chosen (MAPCO-5804)

.github/workflows/build-and-push.yaml

@@ -1,32 +1,23 @@
-name: Publish Release
+name: Build and push artifacts
 
 on:
   push:
     tags:
-      - 'v*'
+      - "v*"
 
-jobs:
-
-  build_and_test:
-    runs-on: ubuntu-latest
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
+        type: string
+env:
+  HELM_EXPERIMENTAL_OCI: 1
 
-    steps:
-        - name: Azure Pipelines Action
-          uses: Azure/pipelines@v1
-          with:
-            azure-devops-project-url: https://dev.azure.com/Libot-Mipui-Org/osm-sync-tracker
-            azure-pipeline-name: 'build-and-push-to-ACR'
-            azure-devops-token: ${{ secrets.AZURE_DEVOPS_TOKEN }}
+permissions:
+  contents: write
+  pull-requests: write
 
-  publish_release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code for CHANGELOG.md
-        uses: actions/checkout@v2
-
-      - name: Publish Release to Github
-        uses: softprops/action-gh-release@v1
-        with:
-          body_path: CHANGELOG.md
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+jobs:
+  build_and_push_docker:
+    uses: MapColonies/shared-workflows/.github/workflows/build-and-push-docker.yaml@v2
+    secrets: inherit

.github/workflows/pull_request.yaml

@@ -23,7 +23,6 @@ jobs:
         uses: wearerequired/lint-action@v1
         with:
           github_token: ${{ secrets.github_token }}
-          # Enable linters
           eslint: true
           prettier: true
           eslint_extensions: ts
@@ -38,46 +37,17 @@ jobs:
   tests:
       name: Run Tests
       runs-on: ubuntu-latest
-      container: node:16
 
       services:
-        # Label used to access the service container
         redis:
-          # Docker Hub image
-          image: bitnami/redis:6.2.7
-          # Provide the password for postgres
+          image: redis:7.2.3
           env:
-            REDIS_ENABLE_SSL_AUTH: false
-            REDIS_DATABASE: 1
             ALLOW_EMPTY_PASSWORD: yes
           ports: 
             - 6379:6379
 
-          # Set health checks to wait until postgres has started
           options: >-
-            --health-cmd "redis-cli ping || exit 1"
-            --health-interval 10s
-            --health-timeout 5s
-            --health-retries 5
-
-        zookeeper:
-            image: bitnami/zookeeper:3.8.0
-            ports:
-              - 2181:2181
-            env:
-              ALLOW_ANONYMOUS_LOGIN: yes
-
-        kafka:
-          image: bitnami/kafka:3.8.0
-          ports:
-            - 9092:9092
-          env:
-            KAFKA_BROKER_ID: 1
-            KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
-            KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-            ALLOW_PLAINTEXT_LISTENER: yes
-          options: >-
-            --health-cmd "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"
+            --health-cmd "redis-cli ping"
             --health-interval 10s
             --health-timeout 5s
             --health-retries 5
@@ -98,22 +68,10 @@ jobs:
         - name: Install Node.js dependencies
           run: npm ci
 
-        - name: Install Docker
-          run: |
-            apt-get update
-            apt-get install -y docker.io
-
-        - name: Wait for Kafka
-          run: |
-            until docker exec $(docker ps -qf "name=kafka") kafka-topics.sh --list --bootstrap-server localhost:9092; do
-              echo "Waiting for Kafka to be ready...";
-              sleep 5;
-            done
-
-        - name: Create Kafka testTopic
+        - name: Create Redis PubSub
           run: |
-            docker exec $(docker ps -qf "name=kafka") \
-            kafka-topics.sh --create --topic testTopic --bootstrap-server localhost:9092
+            docker exec $(docker ps -qf "name=redis") \
+            redis-cli config set notify-keyspace-events KEA
 
         - name: Run tests
           run: npm run test
@@ -142,4 +100,4 @@ jobs:
         uses: actions/checkout@v2
 
       - name: build Docker image
-        run: docker build -t test-build:latest .
+        run: docker build -t test-build:latest .

README.md

@@ -1,8 +1,25 @@
 # Feedback-Api
 This is an API that connects to MapColonies geocoding service, and get's feedback from the users.
+Geocoding's feedback API collects usage data from the [Geocoding API](https://github.com/MapColonies/Geocoding) user's response and stores it for BI purposes. We use this data to better understand and measure the relevance of our responses and adjust the data and algorithm accordingly.
+
 ## API
 Checkout the OpenAPI spec [here](/openapi3.yaml)
 
+## Workflow
+![workflow img](https://github.com/user-attachments/assets/7ed767ad-02dd-46f3-9de7-d2be72205e2c)
+
+## How does it work?
+Once a Geocoding user searches for something using Geocoding's api, it enters Redis (in the geocoding DB index), and an event is triggered. This event adds the `requestId` (from geocoding) to a different Redis DB with a ttl of 5 minutes (TTL DB index).<br/><br/>
+When the Geocoding user chooses a result, the requesting system will then send the `request_id`, the `chosen_response_id`, and the `user_id` back to us using the [Feedback api](/openapi3.yaml).<br/>
+Once we get the chosen response from the api, we validate it and make sure it exists in the Redis geocoding DB, and once it is validated we add a `wasUsed = true` parameter to the geocoding response (in Redis) for later use. We then send the response to Kafka (where it will later be enriched and added to elastic -> see [Geocoding Enrichment](https://github.com/MapColonies/geocoding-enrichment) for more details).<br/>
+Once the TTL of the `requestId` expires an even is triggered, and there are two options:<br/>
+1. One or more responses were chosen.
+2. No response was chosen (only searched for).
+
+We now go back to the `wasUsed` parameter from earlier, and check to see its value (or even if it exists).<br/>
+- If `wasUsed = true` we know from earlier that this response was already chosen and sent to Kafka previously, therefore we can remove it from the Redis geocoding DB index.
+- If `wasUsed` equals to one of the following: `false`, `null` or `undefined` then we create a new feedback response and make the `chosenResultId` `null`, and then send this response to Kafka, and delete the request from the Redis geocoding DB index.
+
 ## Installation
 Install deps with npm
 

config/custom-environment-variables.json

@@ -56,8 +56,12 @@
       "key": "REDIS_KEY_PATH",
       "cert": "REDIS_CERT_PATH"
     },
-    "database": {
-      "__name": "REDIS_DATABASE",
+    "databases": {
+      "geocodingIndex": "REDIS_GEOCODING_INDEX",
+      "ttlIndex": "REDIS_TTL_INDEX"
+    },
+    "ttl": {
+      "__name": "REDIS_TTL",
       "__format": "number"
     }
   },

config/default.json

@@ -36,7 +36,11 @@
       "key": "",
       "cert": ""
     },
-    "database": 0
+    "databases": {
+      "geocodingIndex": 0,
+      "ttlIndex": 1
+    },
+    "ttl": 300
   },
   "kafka": {
     "brokers": "KAFKA_URL:9092",

config/test.json

@@ -1,6 +1,6 @@
 {
   "redis": {
-    "host": "redis",
+    "host": "localhost",
     "port": 6379,
     "username": "",
     "password": "",
@@ -10,7 +10,11 @@
       "key": "",
       "cert": ""
     },
-    "database": 1
+    "databases": {
+      "geocodingIndex": 2,
+      "ttlIndex": 3
+    },
+    "ttl": 2
   },
   "kafka": {
     "brokers": "kafka:9092"

package.json

@@ -5,7 +5,7 @@
   "main": "./src/index.ts",
   "scripts": {
     "test:unit": "jest --config=./tests/configurations/unit/jest.config.js",
-    "test:integration": "jest --config=./tests/configurations/integration/jest.config.js --forceExit",
+    "test:integration": "jest --config=./tests/configurations/integration/jest.config.js",
     "format": "prettier --check .",
     "format:fix": "prettier --write .",
     "prelint:fix": "npm run format:fix",

src/common/constants.ts

@@ -7,16 +7,19 @@ export const IGNORED_OUTGOING_TRACE_ROUTES = [/^.*\/v1\/metrics.*$/];
 export const IGNORED_INCOMING_TRACE_ROUTES = [/^.*\/docs.*$/];
 
 /* eslint-disable @typescript-eslint/naming-convention */
-export const SERVICES: Record<string, symbol> = {
+export const SERVICES = {
   LOGGER: Symbol('Logger'),
   CONFIG: Symbol('Config'),
   TRACER: Symbol('Tracer'),
   METER: Symbol('Meter'),
-  REDIS: Symbol('Redis'),
+  GEOCODING_REDIS: Symbol('GeocodingRedis'),
+  TTL_REDIS: Symbol('TTLRedis'),
   KAFKA: Symbol('Kafka'),
-};
+} satisfies Record<string, symbol>;
 /* eslint-enable @typescript-eslint/naming-convention */
 
 export const ON_SIGNAL = Symbol('onSignal');
 export const HEALTHCHECK = Symbol('healthcheck');
 export const CLEANUP_REGISTRY = Symbol('cleanupRegistry');
+export const REDIS_SUB = Symbol('redisSubClient');
+export const REDIS_CLIENT_FACTORY = Symbol('redisClientFactory');

src/common/dependencyRegistration.ts

@@ -8,6 +8,7 @@ export interface InjectionObject<T> {
   provider: Providers<T>;
   options?: RegistrationOptions;
   postInjectionHook?: (container: DependencyContainer) => Promise<void>;
+  afterAllInjectionHook?: (container: DependencyContainer) => void | Promise<void>;
 }
 
 export const registerDependencies = async (
@@ -23,5 +24,8 @@ export const registerDependencies = async (
     await injectionObj.postInjectionHook?.(container);
   }
 
+  for (const dep of dependencies) {
+    await dep.afterAllInjectionHook?.(container);
+  }
   return container;
 };

src/common/interfaces.ts

@@ -18,6 +18,7 @@ export type RedisConfig = {
   port: number;
   enableSslAuth: boolean;
   sslPaths: { ca: string; cert: string; key: string };
+  databases: { geocodingIndex: number; ttlIndex: number };
 } & RedisClientOptions;
 
 export type KafkaOptions = {
@@ -28,16 +29,17 @@ export type KafkaOptions = {
 
 export interface FeedbackResponse {
   requestId: string;
-  chosenResultId: number;
+  chosenResultId: number | null;
   userId: string;
   responseTime: Date; // from FeedbackApi
   geocodingResponse: GeocodingResponse;
 }
 
 export interface GeocodingResponse {
-  userId: string;
+  userId?: string;
   apiKey: string;
   site: string;
   response: JSON;
   respondedAt: Date; // from Geocoding
+  wasUsed?: boolean;
 }

src/common/utils.ts

@@ -1,14 +1,34 @@
-import { TimeoutError } from './errors';
+import { DependencyContainer, FactoryFunction } from 'tsyringe';
+import { Logger } from '@map-colonies/js-logger';
+import { RedisClient } from '../redis/index';
+import { SERVICES } from './constants';
 
-export const promiseTimeout = async <T>(ms: number, promise: Promise<T>): Promise<T> => {
-  // create a promise that rejects in <ms> milliseconds
-  const timeout = new Promise<T>((_, reject) => {
-    const id = setTimeout(() => {
-      clearTimeout(id);
-      reject(new TimeoutError(`Timed out in + ${ms} + ms.`));
-    }, ms);
-  });
+export const healthCheckFactory: FactoryFunction<void> = (container: DependencyContainer): void => {
+  const logger = container.resolve<Logger>(SERVICES.LOGGER);
+  const geocodingRedis = container.resolve<RedisClient>(SERVICES.GEOCODING_REDIS);
+  const ttlRedis = container.resolve<RedisClient>(SERVICES.TTL_REDIS);
 
-  // returns a race between our timeout and the passed in promise
-  return Promise.race([promise, timeout]);
+  geocodingRedis
+    .ping()
+    .then(() => {
+      return;
+    })
+    .catch((error: Error) => {
+      logger.error({
+        message: `Healthcheck failed for GeocodingRedis.`,
+        error,
+      });
+    });
+
+  ttlRedis
+    .ping()
+    .then(() => {
+      return;
+    })
+    .catch((error: Error) => {
+      logger.error({
+        message: `Healthcheck failed for GeocodingRedis.`,
+        error,
+      });
+    });
 };