From 764815b29921e27d63cee1f96b922bb575dec9e0 Mon Sep 17 00:00:00 2001
From: Shlomi <akunh113@gmail.com>
Date: Wed, 4 Dec 2024 15:10:03 +0200
Subject: [PATCH] feat: Add option for the researcher to decide if filtering
 includes null values

---
 src/app/api/root/route.ts        | 36 +++++++++++++++++++++++---------
 src/app/components/Chat/Chat.tsx |  2 ++
 src/app/general/interfaces.ts    |  6 ++++++
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/src/app/api/root/route.ts b/src/app/api/root/route.ts
index d366873..595865c 100755
--- a/src/app/api/root/route.ts
+++ b/src/app/api/root/route.ts
@@ -1,4 +1,4 @@
-import { Attribute } from "@/app/general/interfaces";
+import { Attribute, BotNullValues } from "@/app/general/interfaces";
 import { getOperator } from "@/app/general/utils";
 import { strOrNum } from "@/app/general/types";
 import { NextRequest, NextResponse } from "next/server";
@@ -10,12 +10,17 @@ export async function POST(request: NextRequest) {
     const req = await request.json();
     const attributes = req.queryParams as Attribute[];
     const fileName = req.filePath as string;
+    const nullValues = req.nullValues as BotNullValues;
     const filePath = path.join(process.cwd(), "src/app/data", fileName);
-    const rows = await filterCSV(filePath, attributes);
+    const rows = await filterCSV(filePath, attributes, nullValues);
     return NextResponse.json(rows);
 }
 
-async function filterCSV(filePath: string, attributes: Attribute[]) {
+async function filterCSV(
+    filePath: string,
+    attributes: Attribute[],
+    nullValues: BotNullValues
+) {
     return new Promise<any[]>((resolve, reject) => {
         const operators = attributes.map(
             (query) => getOperator(query.operator) as any
@@ -24,7 +29,7 @@ async function filterCSV(filePath: string, attributes: Attribute[]) {
         fs.createReadStream(filePath)
             .pipe(csvParser())
             .on("data", (row: any) => {
-                if (isRowRequired(attributes, operators, row)) {
+                if (isRowRequired(attributes, operators, row, nullValues)) {
                     rows.push(row);
                 }
             })
@@ -40,17 +45,22 @@ async function filterCSV(filePath: string, attributes: Attribute[]) {
 function isRowRequired(
     attributes: Attribute[],
     operators: any[],
-    row: any
+    row: any,
+    nullValues: BotNullValues
 ): boolean {
     let isRequired: boolean = true;
     const duplicates = findDuplicates(attributes, "name");
+    const { isFilterIncludesNull, nullValues: nullValuesArray } = nullValues;
 
     if (duplicates.length > 0) {
         const notDuplicatesAttributes = attributes.filter(
             (attribute) => !duplicates.includes(attribute)
         );
         const isRequiredNotDuplicates = notDuplicatesAttributes.every(
-            (query, index) => operators[index](row[query.name], ...query.params)
+            (query, index) =>
+                operators[index](row[query.name], ...query.params) ||
+                (isFilterIncludesNull &&
+                    nullValuesArray.includes(row[query.name]))
         );
         const namesOfDuplicates = duplicates.map((query) => query.name);
         const duplicatesAttributes = [...new Set(namesOfDuplicates)];
@@ -58,8 +68,11 @@ function isRowRequired(
             const attributeOfTheName = attributes.filter(
                 (attribute) => attribute.name === name
             );
-            const isRowRequired = attributeOfTheName.some((query, index) =>
-                operators[index](row[query.name], ...query.params)
+            const isRowRequired = attributeOfTheName.some(
+                (query, index) =>
+                    operators[index](row[query.name], ...query.params) ||
+                    (isFilterIncludesNull &&
+                        nullValuesArray.includes(row[query.name]))
             );
             return isRowRequired;
         });
@@ -69,8 +82,11 @@ function isRowRequired(
 
         isRequired = isRequiredNotDuplicates && isRequiredDuplicates;
     } else {
-        isRequired = attributes.every((query, index) =>
-            operators[index](row[query.name], ...query.params)
+        isRequired = attributes.every(
+            (query, index) =>
+                operators[index](row[query.name], ...query.params) ||
+                (isFilterIncludesNull &&
+                    nullValuesArray.includes(row[query.name]))
         );
     }
 
diff --git a/src/app/components/Chat/Chat.tsx b/src/app/components/Chat/Chat.tsx
index 7c477e1..47e9943 100644
--- a/src/app/components/Chat/Chat.tsx
+++ b/src/app/components/Chat/Chat.tsx
@@ -47,12 +47,14 @@ export default function Chat({ bot }: ChatProps) {
 
                 const pathArray = bot?.filePath.split("/");
                 const path = pathArray.pop();
+                const nullValues = bot?.nullValues;
 
                 const response = await fetch("/api/root", {
                     method: "POST",
                     body: JSON.stringify({
                         queryParams,
                         filePath: path,
+                        nullValues,
                     }),
                 });
                 if (!response.ok) {
diff --git a/src/app/general/interfaces.ts b/src/app/general/interfaces.ts
index 6683391..b40db7a 100644
--- a/src/app/general/interfaces.ts
+++ b/src/app/general/interfaces.ts
@@ -39,6 +39,12 @@ export interface Bot {
     currentOperatorIndex: number;
     operatorsFiles: BorOperatorsFiles;
     colors: Colors;
+    nullValues: BotNullValues;
+}
+
+export interface BotNullValues {
+    nullValues: any[];
+    isFilterIncludesNull: boolean;
 }
 
 export interface Colors {