From b1df5c8c47d2a21f09a37af695f426098e237865 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Apr 2026 09:03:57 -0700 Subject: [PATCH 1/9] Initial commit for benchmarking Signed-off-by: Andrew Carbonetto --- benchmark-tests/answer_eval.py | 70 +++++ benchmark-tests/run_evaluation.py | 290 ++++++++++++++++++ .../graphrag_toolkit_tests/benchmark_build.py | 72 +++++ 3 files changed, 432 insertions(+) create mode 100644 benchmark-tests/answer_eval.py create mode 100644 benchmark-tests/run_evaluation.py create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py diff --git a/benchmark-tests/answer_eval.py b/benchmark-tests/answer_eval.py new file mode 100644 index 00000000..4e59f875 --- /dev/null +++ b/benchmark-tests/answer_eval.py @@ -0,0 +1,70 @@ +from bedrock_generator import BedrockGenerator +import argparse +import json +import multiprocessing as mp + +PROMPT = """You are an evaluator tasked with determining whether a specific statement is semantically entailed in a response. +Given the question: {question}: +Check whether the following statement is entailed/covered in the response. Response Yes/No without any other text. +Statement: {evidence} +Response: {response} +""" + +def run_evaluation(prompt): + response = generator.generate(text=[ + {"role": "user", "content": [{"type": "text", "text": prompt}]} + ]) + print(f"prompt: {prompt}") + print(f"entailment: {response}") + return response + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input-file-path", required=True, type=str) + parser.add_argument("--output-file-path", type=str) + parser.add_argument("--query-file-path", type=str) # only needed for multihop-rag + parser.add_argument("--dataset", type=str, choices = ["multihop-rag", "wikihowqa"]) + args = parser.parse_args() + generator = BedrockGenerator.from_config({"model_id": "us.anthropic.claude-3-sonnet-20240229-v1:0"}) + with open(args.input_file_path) as fin: + data = json.load(fin) + arguments = [] + entries = [] + cnt = 0 + + if args.dataset == "multihop-rag": + with open(args.query_file_path) as fin: + queries = json.load(fin) + for query, response in zip(queries, data): + for evidence in query[0]["evidence_list"]: + prompt = PROMPT.format(evidence=evidence["fact"], response=response["response"]) + arguments.append(prompt) + + elif args.dataset == "wikihowqa": + for entry in data: + evidences = entry["answers"][0].split(".") + for evidence in evidences: + if len(evidence) ==0: + continue + prompt = PROMPT.format(question = entry["raw_question"], evidence=evidence, response=entry["response"]) + arguments.append(prompt) + entries.append({"question": entry["raw_question"], "evidence": evidence, "response": entry["response"]}) + else: + raise NotImplemented + + pool = mp.Pool(16) + responses = pool.map(run_evaluation, arguments) + correct = 0 + total = 0 + for idx, response in enumerate(responses): + if "Yes" in response: + correct += 1 + entries[idx]["judgement"] = "Yes" + else: + entries[idx]["judgement"] = "No" + total += 1 + print(correct, total) + print(correct / total) + + with open(args.output_file_path, 'w') as output_file: + json.dump(entries, output_file, indent=2) diff --git a/benchmark-tests/run_evaluation.py b/benchmark-tests/run_evaluation.py new file mode 100644 index 00000000..ba586896 --- /dev/null +++ b/benchmark-tests/run_evaluation.py @@ -0,0 +1,290 @@ +from boto3 import Session +from botocore.config import Config +import logging +import json +import argparse +from tqdm import tqdm +import os +import time + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def call_bedrock_invoke_model(prompt, bedrock, model_id, is_json_output=True): + while True: + try: + accept = 'application/json' + contentType = 'application/json' + + if 'anthropic.claude-3' in model_id: + payload_body = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "temperature": 0.0, + "top_p": 1, + "top_k": 50, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + } + ] + } + body = json.dumps(payload_body) + else: + body = json.dumps({"prompt": prompt, "max_tokens_to_sample": 2048, "temperature": 0.0, "top_p": 1, "top_k": 50, + "stop_sequences": ["\\n\\nHuman:"]}) + + response = bedrock.invoke_model(body=body, modelId=model_id, accept=accept, contentType=contentType) + + if 'anthropic.claude-3' in model_id: + response = response['body'].read().decode('utf-8') + response = json.loads(response) + response_text = response['content'][0]['text'] + + else: + response_obj = response.get('body').read().decode('utf-8') + print("RESPONSE OBJ: " + str(response_obj)) + json_response_obj = json.loads(response_obj) + response_text = json_response_obj['completion'] + if is_json_output: + try: + start_idx = response_text.find("{") + end_idx = response_text.find("}") + parsed_completion = response_text[start_idx:end_idx + 1] + parsed_json = json.loads(parsed_completion) + parsed_json['llm_response'] = response_text + return parsed_json + except: + logger.error(response_text) + return { + 'grade': "incorrect", + 'justification': "LLM failed grading", + 'llm_response': response_text + } + else: + return response_text + except Exception as e: + logger.error(str(e)) + time.sleep(3) + + + +BKB_CORRECTNESS_GRADING = """ +Human: +You are a teacher grading a quiz. +You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect. + +Example Format: +QUESTION: question here +STUDENT ANSWER: student's answer here +TRUE ANSWER: true answer here +GRADE: Correct or Incorrect here + +Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student fails to answers or claims that the search results do not mention the answer then mark as incorrect. Begin! + +QUESTION: {query} +STUDENT ANSWER: {answer} +TRUE ANSWER: {expected_answer} +GRADE: + +Your response should be in json format as follows: +{{ + "grade": (correct or incorrect), + "justification": (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect. Use one or two sentences maximum. Keep the answer as concise as possible.) +}} + + +Assistant: +""" + +IDK_DETECTION = """You are a teacher grading a quiz. Based on students' response, you are asked to determine if the students think they can not answer the question because some information are missing. +Response: {response} +Please output "Unanswerable" if the students identify that they can not answer the question. Otherwise, output "Answerable". +""" +import os +os.environ["AWS_REGION_NAME"] = "us-west-2" + + +class GenerationEvaluator: + bedrock = Session().client( + service_name='bedrock-runtime', + region_name="us-west-2", + config=Config( + max_pool_connections=50, + retries={"max_attempts": 10, "mode": "standard"}, + connect_timeout=500, + read_timeout=500, + region_name="us-west-2" + )) + + + def __init__(self, model_id): + self.model_id = model_id + +class CorrectnessEvaluator(GenerationEvaluator): + def __init__(self, model_id): + super().__init__(model_id) + + def evaluate(self, question, answer, response): + grading = {} + grading.update(self._llm_evaluate(question, answer, response)) + return grading + + def _llm_evaluate(self, question, answer, response): + prompt = BKB_CORRECTNESS_GRADING.format( + query=question, + answer=response, + expected_answer=answer + ) + completion = call_bedrock_invoke_model(prompt, self.bedrock, model_id=self.model_id) + if answer == "": + completion['grade'] = "incorrect" + completion['justification'] = "No answer was provided" + + if not completion or not completion['grade'] or not completion['justification']: + logger.error("Failed to grade") + logger.error(str(completion)) + return { + 'question': question, + 'llmCorrectnessGrade': "incorrect", + 'llmCorrectnessGradeJustification': "LLM failed grading", + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + + try: + grading = { + 'question': question, + 'llmCorrectnessGrade': completion['grade'].lower(), + 'llmCorrectnessGradeJustification': completion['justification'].replace("\"", "\\\""), + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + return grading + except Exception as e: + logger.info(str(e)) + return { + 'question': question, + 'llmCorrectnessGrade': "incorrect", + 'llmCorrectnessGradeJustification': "LLM failed grading", + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + + +class IDKEvaluator(GenerationEvaluator): + def __init__(self, model_id): + super().__init__(model_id) + + def evaluate(self, question, answer, response): + grading = {} + grading.update(self._llm_evaluate(question, answer, response)) + return grading + + def _llm_evaluate(self, question, answer, response): + prompt = IDK_DETECTION.format( + question=question, + answer=answer, + response=response + ) + completion = call_bedrock_invoke_model(prompt, self.bedrock, model_id=self.model_id, is_json_output=False) + if "Unanswerable" in completion: + return { + "label": "unanswerable" + } + else: + return { + "label": "answerable" + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input-file-path", type=str) + parser.add_argument("--metrics-output-path", type=str) + parser.add_argument("--eval-artifacts", type=str) + parser.add_argument("--metric", type=str, default="correctness", choices=["correctness", "idk", "correctness_on_answerable"]) + args = parser.parse_args() + + if args.metric == "correctness_on_answerable": + eval_correctness_artifact, eval_idk_artifact = args.eval_artifacts.split(",") + with open(eval_correctness_artifact) as fin: + eval_correctness_data = json.load(fin) + with open(eval_idk_artifact) as fin: + eval_idk_data = json.load(fin) + assert len(eval_correctness_data) == len(eval_idk_data) + total, count = 0, 0 + for correctness_eval, idk_eval in zip(eval_correctness_data, eval_idk_data): + if idk_eval["label"] == "answerable": + total += 1 + if correctness_eval["llmCorrectnessGrade"] == "correct": + count += 1 + logger.info("{}: {}".format(args.metric, count / total)) + else: + if args.metric == "correctness": + evaluator = CorrectnessEvaluator(model_id="anthropic.claude-3-sonnet-20240229-v1:0") + elif args.metric == "idk": + evaluator = IDKEvaluator(model_id="anthropic.claude-3-sonnet-20240229-v1:0") + + + data = [] + with open(args.input_file_path) as fin: + for line in fin: + data.append(json.loads(line)) + evaluation_outputs = [] + for example in tqdm(data): + answer = example["raw_example"]["answer"] + response = example["response"] + question = example["raw_example"]["question"] + evaluation_outputs.append(evaluator.evaluate(question, answer, response)) + count, total = 0, 0 + for evaluation in evaluation_outputs: + total += 1 + if args.metric == "correctness": + if evaluation["llmCorrectnessGrade"] == "correct": + count += 1 + if args.metric == "idk": + if evaluation["label"] == "unanswerable": + count += 1 + + logger.info("{}: {}".format(args.metric, count / total)) + os.makedirs(os.path.dirname(args.metrics_output_path), exist_ok=True) + with open(args.metrics_output_path, "w") as fout: + json.dump(evaluation_outputs, fout, indent=4) + with open(os.path.join(os.path.dirname(args.metrics_output_path), "{}.json".format(args.metric)), "w") as fout: + json.dump({ + args.metric: count / total + }, fout, indent=4) + +import multiprocessing as mp +import yaml + +class SafeCounter(): + # constructor + def __init__(self): + # initialize counter + self._counter = mp.Value('i', 0) + # initialize lock + self._lock = mp.Lock() + + # increment the counter + def increment(self): + # get the lock + with self._lock: + self._counter.value += 1 + + # get the counter value + def value(self): + with self._lock: + return self._counter.value + +def read_yaml(file_path): + if file_path: + with open(file_path, 'r') as file: + data = yaml.safe_load(file) + return data + else: + return {} \ No newline at end of file diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py new file mode 100644 index 00000000..488429f1 --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import os +import unittest +from typing import Dict, Any + +from graphrag_toolkit_tests.integration_test_base import IntegrationTestBase +from graphrag_toolkit_tests.integration_test_handler import IntegrationTestHandler + +from graphrag_toolkit.lexical_graph import LexicalGraphIndex +from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory +from graphrag_toolkit.lexical_graph.storage import VectorStoreFactory +from graphrag_toolkit.lexical_graph.storage.graph import NonRedactedGraphQueryLogFormatting +from graphrag_toolkit.lexical_graph.indexing.load import FileBasedDocs + +CUAD_NUM_DOCS = 510 + +class BenchmarkBuild(IntegrationTestBase): + + @property + def description(self): + return 'Build graph and vector stores from CUAD pre-extracted chunks for benchmarking' + + def _run_test(self, handler:IntegrationTestHandler, params:Dict[str, Any]): + + data_dir = os.environ.get('BENCHMARK_DATA_DIR', 'benchmark-tests/data') + dataset = os.environ.get('BENCHMARK_DATASET', 'cuad') + + docs_directory = os.path.join(data_dir, dataset, 'extracted', '2026-02-17') + + docs = FileBasedDocs( + docs_directory=docs_directory, + collection_id=dataset + ) + + with( + GraphStoreFactory.for_graph_store( + os.environ['GRAPH_STORE'], + log_formatting=NonRedactedGraphQueryLogFormatting() + ) as graph_store, + VectorStoreFactory.for_vector_store(os.environ['VECTOR_STORE']) as vector_store + ): + + graph_index = LexicalGraphIndex( + graph_store, + vector_store + ) + + graph_index.build(docs, show_progress=True) + + # Store graph_store in params for downstream tests (query stage) + params['benchmark_graph_store'] = os.environ['GRAPH_STORE'] + params['benchmark_vector_store'] = os.environ['VECTOR_STORE'] + params['benchmark_dataset'] = dataset + params['benchmark_data_dir'] = data_dir + + class BenchmarkBuildAssertions(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls._graph_store = graph_store + cls._expected_num_docs = CUAD_NUM_DOCS + + def test_one_source_node_for_each_doc(self): + """Graph contains one source node per CUAD document""" + + results = self._graph_store.execute_query('MATCH (n:`__Source__`) RETURN count(n) AS count') + source_node_count = results[0]['count'] + + self.assertEqual(source_node_count, self._expected_num_docs) + + handler.run_assertions(BenchmarkBuildAssertions) From 1fd04375258f12b7978cc91baa47ef415d70271c Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 14 Apr 2026 12:04:01 -0700 Subject: [PATCH 2/9] Add benchmarking tasks Signed-off-by: Andrew Carbonetto --- integration-tests/benchmark.cuad.prototype | 3 + .../graphrag_toolkit_tests/benchmark_build.py | 146 ++++++++++------ .../benchmark_evaluate.py | 133 +++++++++++++++ .../graphrag_toolkit_tests/benchmark_query.py | 158 ++++++++++++++++++ 4 files changed, 387 insertions(+), 53 deletions(-) create mode 100644 integration-tests/benchmark.cuad.prototype create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py diff --git a/integration-tests/benchmark.cuad.prototype b/integration-tests/benchmark.cuad.prototype new file mode 100644 index 00000000..f98b4914 --- /dev/null +++ b/integration-tests/benchmark.cuad.prototype @@ -0,0 +1,3 @@ +benchmark_build.CuadBenchmarkBuild +benchmark_query.CuadBenchmarkQuery +benchmark_evaluate.CuadBenchmarkEvaluate diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py index 488429f1..5d97539a 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 import os import unittest -from typing import Dict, Any +from contextlib import nullcontext +from typing import Dict, Any, Optional from graphrag_toolkit_tests.integration_test_base import IntegrationTestBase from graphrag_toolkit_tests.integration_test_handler import IntegrationTestHandler @@ -13,60 +14,99 @@ from graphrag_toolkit.lexical_graph.storage.graph import NonRedactedGraphQueryLogFormatting from graphrag_toolkit.lexical_graph.indexing.load import FileBasedDocs -CUAD_NUM_DOCS = 510 +DATASET_CONFIG = { + 'cuad': { + 'num_docs': 510, + 'extracted_dir': os.path.join('extracted', '2026-02-17'), + }, + 'pga': { + 'num_docs': 507, + }, + 'concurrentqa': { + 'num_docs': 13501, + }, +} + + +def run_benchmark_build(handler: IntegrationTestHandler, dataset: str, data_dir: str, + graph_store_conn: Optional[str] = None, vector_store_conn: Optional[str] = None): + """ + Builds graph and vector stores from pre-extracted document chunks for a benchmark dataset. + + Loads extracted chunks via FileBasedDocs, builds the graph and vector indexes, and + asserts that the expected number of source nodes were created. Either store connection + can be omitted — the build will proceed with whichever stores are provided, and the + source node assertion will be skipped if no graph store is configured. + + Args: + handler: Integration test handler for recording assertions and output. + dataset: Dataset key (e.g. 'cuad', 'pga', 'concurrentqa'). Must have a + corresponding entry in DATASET_CONFIG. + data_dir: Root path to the benchmark data directory containing dataset subdirectories. + graph_store_conn: Optional graph store connection string (e.g. 'neptune-db://' or + 'neptune-graph://'). + vector_store_conn: Optional vector store connection string (e.g. 'aoss://...'). + """ + config = DATASET_CONFIG.get(dataset, {}) + + extracted_subdir = config.get('extracted_dir', 'extracted') + docs_directory = os.path.join(data_dir, dataset, extracted_subdir) + + docs = FileBasedDocs( + docs_directory=docs_directory, + collection_id=dataset + ) + + graph_ctx = GraphStoreFactory.for_graph_store( + graph_store_conn, log_formatting=NonRedactedGraphQueryLogFormatting() + ) if graph_store_conn else nullcontext() + + vector_ctx = VectorStoreFactory.for_vector_store( + vector_store_conn + ) if vector_store_conn else nullcontext() + + with graph_ctx as graph_store, vector_ctx as vector_store: + graph_index = LexicalGraphIndex(graph_store, vector_store) + graph_index.build(docs, show_progress=True) + + expected_num_docs = config.get('num_docs') + + class BenchmarkBuildAssertions(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._graph_store = graph_store + cls._expected_num_docs = expected_num_docs + + def test_one_source_node_for_each_doc(self): + """Graph contains one source node per document""" + if self._graph_store is None: + self.skipTest('No graph store configured') + results = self._graph_store.execute_query('MATCH (n:`__Source__`) RETURN count(n) AS count') + source_node_count = results[0]['count'] + if self._expected_num_docs is not None: + self.assertEqual(source_node_count, self._expected_num_docs) + else: + self.assertGreater(source_node_count, 0) + + handler.run_assertions(BenchmarkBuildAssertions) + + +class CuadBenchmarkBuild(IntegrationTestBase): -class BenchmarkBuild(IntegrationTestBase): - @property def description(self): return 'Build graph and vector stores from CUAD pre-extracted chunks for benchmarking' - - def _run_test(self, handler:IntegrationTestHandler, params:Dict[str, Any]): - + + def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): data_dir = os.environ.get('BENCHMARK_DATA_DIR', 'benchmark-tests/data') - dataset = os.environ.get('BENCHMARK_DATASET', 'cuad') - - docs_directory = os.path.join(data_dir, dataset, 'extracted', '2026-02-17') - - docs = FileBasedDocs( - docs_directory=docs_directory, - collection_id=dataset - ) - - with( - GraphStoreFactory.for_graph_store( - os.environ['GRAPH_STORE'], - log_formatting=NonRedactedGraphQueryLogFormatting() - ) as graph_store, - VectorStoreFactory.for_vector_store(os.environ['VECTOR_STORE']) as vector_store - ): - - graph_index = LexicalGraphIndex( - graph_store, - vector_store - ) - - graph_index.build(docs, show_progress=True) - - # Store graph_store in params for downstream tests (query stage) - params['benchmark_graph_store'] = os.environ['GRAPH_STORE'] - params['benchmark_vector_store'] = os.environ['VECTOR_STORE'] - params['benchmark_dataset'] = dataset - params['benchmark_data_dir'] = data_dir - - class BenchmarkBuildAssertions(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls._graph_store = graph_store - cls._expected_num_docs = CUAD_NUM_DOCS - - def test_one_source_node_for_each_doc(self): - """Graph contains one source node per CUAD document""" - - results = self._graph_store.execute_query('MATCH (n:`__Source__`) RETURN count(n) AS count') - source_node_count = results[0]['count'] - - self.assertEqual(source_node_count, self._expected_num_docs) - - handler.run_assertions(BenchmarkBuildAssertions) + graph_store_conn = os.environ.get('GRAPH_STORE') + vector_store_conn = os.environ.get('VECTOR_STORE') + + run_benchmark_build(handler, 'cuad', data_dir, graph_store_conn, vector_store_conn) + + params['benchmark_dataset'] = 'cuad' + params['benchmark_data_dir'] = data_dir + if graph_store_conn: + params['benchmark_graph_store'] = graph_store_conn + if vector_store_conn: + params['benchmark_vector_store'] = vector_store_conn diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py new file mode 100644 index 00000000..2a6b0595 --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py @@ -0,0 +1,133 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +import os +import sys +import unittest +from typing import Dict, Any, Optional, List + +from graphrag_toolkit_tests.integration_test_base import IntegrationTestBase +from graphrag_toolkit_tests.integration_test_handler import IntegrationTestHandler + +# Add benchmark-tests to path so we can import run_evaluation +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', 'benchmark-tests')) +from run_evaluation import CorrectnessEvaluator, IDKEvaluator + + +def run_benchmark_evaluate(handler: IntegrationTestHandler, params: Dict[str, Any], + dataset: str, responses_path: str, + metrics: Optional[List[str]] = None): + """ + Evaluates benchmark query responses against ground-truth answers using LLM-as-judge. + + Reads the responses JSONL file produced by run_benchmark_query, runs each specified + metric evaluator (correctness, idk, correctness_on_answerable), and writes per-item + grades and aggregate scores to benchmark-results//. + + Output files per metric: + - benchmark-results//_evals.json — per-item grading array + - benchmark-results//.json — aggregate score, e.g. {"correctness": 0.72} + + The 'correctness_on_answerable' metric requires both 'correctness' and 'idk' to have + been run first (it reads their output files). If included in the metrics list, it must + come after both. + + Args: + handler: Integration test handler for recording assertions and output. + params: Shared params dict passed between pipeline stages. + dataset: Dataset key (e.g. 'cuad', 'pga', 'concurrentqa'). + responses_path: Path to the responses JSONL file from the query stage. + metrics: List of metrics to run. Defaults to ['correctness', 'idk']. + """ + if metrics is None: + metrics = ['correctness', 'idk'] + + results_dir = os.path.join('benchmark-results', dataset) + os.makedirs(results_dir, exist_ok=True) + + data = [] + with open(responses_path) as fin: + for line in fin: + data.append(json.loads(line)) + + scores = {} + + for metric in metrics: + if metric == 'correctness_on_answerable': + correctness_path = os.path.join(results_dir, 'correctness_evals.json') + idk_path = os.path.join(results_dir, 'idk_evals.json') + with open(correctness_path) as f: + correctness_data = json.load(f) + with open(idk_path) as f: + idk_data = json.load(f) + total, count = 0, 0 + for c_eval, i_eval in zip(correctness_data, idk_data): + if i_eval['label'] == 'answerable': + total += 1 + if c_eval['llmCorrectnessGrade'] == 'correct': + count += 1 + score = count / total if total > 0 else 0.0 + else: + if metric == 'correctness': + evaluator = CorrectnessEvaluator(model_id='anthropic.claude-3-sonnet-20240229-v1:0') + elif metric == 'idk': + evaluator = IDKEvaluator(model_id='anthropic.claude-3-sonnet-20240229-v1:0') + + evals = [] + for example in data: + question = example['raw_example']['question'] + answer = example['raw_example']['answer'] + response = example['response'] + evals.append(evaluator.evaluate(question, answer, response)) + + evals_path = os.path.join(results_dir, f'{metric}_evals.json') + with open(evals_path, 'w') as f: + json.dump(evals, f, indent=2) + + count, total = 0, len(evals) + for e in evals: + if metric == 'correctness' and e.get('llmCorrectnessGrade') == 'correct': + count += 1 + elif metric == 'idk' and e.get('label') == 'unanswerable': + count += 1 + score = count / total if total > 0 else 0.0 + + scores[metric] = score + score_path = os.path.join(results_dir, f'{metric}.json') + with open(score_path, 'w') as f: + json.dump({metric: score}, f, indent=2) + + handler.add_output(metric, score) + + params['benchmark_scores'] = scores + + class BenchmarkEvaluateAssertions(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._scores = scores + + def test_scores_are_plausible(self): + """All metric scores are between 0 and 1""" + for metric, score in self._scores.items(): + self.assertGreaterEqual(score, 0.0, f'{metric} score is negative') + self.assertLessEqual(score, 1.0, f'{metric} score exceeds 1.0') + + handler.run_assertions(BenchmarkEvaluateAssertions) + + +class CuadBenchmarkEvaluate(IntegrationTestBase): + + @property + def description(self): + return 'Evaluate CUAD benchmark responses using LLM-as-judge correctness and IDK metrics' + + def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): + responses_path = params.get('benchmark_responses_path', + os.path.join('benchmark-results', 'cuad', 'responses.jsonl')) + + run_benchmark_evaluate( + handler, params, + dataset='cuad', + responses_path=responses_path, + metrics=['correctness', 'idk'], + ) diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py new file mode 100644 index 00000000..0948424d --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py @@ -0,0 +1,158 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +import os +import unittest +from contextlib import nullcontext +from typing import Dict, Any, Optional, List + +from graphrag_toolkit_tests.integration_test_base import IntegrationTestBase +from graphrag_toolkit_tests.integration_test_handler import IntegrationTestHandler + +from graphrag_toolkit.lexical_graph import LexicalGraphQueryEngine, GraphRAGConfig +from graphrag_toolkit.lexical_graph.storage import GraphStoreFactory, VectorStoreFactory +from graphrag_toolkit.lexical_graph.storage.graph import NonRedactedGraphQueryLogFormatting + +from llama_index.core.schema import QueryBundle + +QA_FILE_MAP = { + 'cuad': ['qa.json'], + 'pga': ['pga_bio.json', 'pga_stat.json'], + 'concurrentqa': ['qa.json'], +} + + +def load_qa_pairs(data_dir: str, dataset: str, qa_files: List[str], limit: Optional[int] = None): + pairs = [] + for f in qa_files: + path = os.path.join(data_dir, dataset, f) + with open(path) as fh: + pairs.extend(json.load(fh)) + if limit: + pairs = pairs[:limit] + return pairs + + +def run_benchmark_query(handler: IntegrationTestHandler, params: Dict[str, Any], + dataset: str, data_dir: str, + graph_store_conn: Optional[str] = None, + vector_store_conn: Optional[str] = None, + response_llm: str = 'anthropic.claude-sonnet-4-20250514-v1:0', + qa_limit: Optional[int] = None): + """ + Queries a benchmark dataset's QA pairs and writes responses in run_evaluation.py format. + + Loads QA pairs from the dataset's JSON files (mapped via QA_FILE_MAP), queries each + question through a traversal-based LexicalGraphQueryEngine, and writes a JSONL file + with one line per question in the format: + {"raw_example": {"question": "...", "answer": "..."}, "response": "..."} + + Either store connection can be omitted — a nullcontext is used for the missing store. + The assertion verifies that all questions received a non-empty response. + + Results are written to benchmark-results//responses.jsonl and the path is + stored in params['benchmark_responses_path'] for downstream BenchmarkEvaluate. + + Args: + handler: Integration test handler for recording assertions and output. + params: Shared params dict passed between pipeline stages. This function sets + 'benchmark_responses_path' and 'benchmark_num_questions' for downstream use. + dataset: Dataset key (e.g. 'cuad', 'pga', 'concurrentqa'). Must have a + corresponding entry in QA_FILE_MAP. + data_dir: Root path to the benchmark data directory containing dataset subdirectories. + graph_store_conn: Optional graph store connection string (e.g. 'neptune-db://...'). + vector_store_conn: Optional vector store connection string (e.g. 'aoss://...'). + response_llm: Bedrock model ID for generating query responses. + qa_limit: Optional cap on the number of QA pairs to query (for prototype runs). + """ + qa_files = QA_FILE_MAP.get(dataset, ['qa.json']) + qa_pairs = load_qa_pairs(data_dir, dataset, qa_files, qa_limit) + + GraphRAGConfig.response_llm = response_llm + + graph_ctx = GraphStoreFactory.for_graph_store( + graph_store_conn, log_formatting=NonRedactedGraphQueryLogFormatting() + ) if graph_store_conn else nullcontext() + + vector_ctx = VectorStoreFactory.for_vector_store( + vector_store_conn + ) if vector_store_conn else nullcontext() + + with graph_ctx as graph_store, vector_ctx as vector_store: + query_engine = LexicalGraphQueryEngine.for_traversal_based_search( + graph_store, + vector_store + ) + + responses_path = os.path.join('benchmark-results', dataset, 'responses.jsonl') + os.makedirs(os.path.dirname(responses_path), exist_ok=True) + + num_empty = 0 + + with open(responses_path, 'w') as out: + for i, item in enumerate(qa_pairs): + question = item['input'] + answer = item['output'][0]['text'] + + response = query_engine.query(question) + response_text = response.response or '' + + if not response_text.strip(): + num_empty += 1 + + out.write(json.dumps({ + 'raw_example': {'question': question, 'answer': answer}, + 'response': response_text + }) + '\n') + + handler.add_output(f'q{i}', { + 'question': question, + 'response': response_text[:500] + }) + + params['benchmark_responses_path'] = responses_path + params['benchmark_num_questions'] = len(qa_pairs) + + handler.add_output('total_questions', len(qa_pairs)) + handler.add_output('empty_responses', num_empty) + + class BenchmarkQueryAssertions(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._num_questions = len(qa_pairs) + cls._num_empty = num_empty + + def test_all_questions_answered(self): + """All questions received a non-empty response""" + self.assertEqual(self._num_empty, 0, + f'{self._num_empty}/{self._num_questions} questions got empty responses') + + handler.run_assertions(BenchmarkQueryAssertions) + + +class CuadBenchmarkQuery(IntegrationTestBase): + + @property + def description(self): + return 'Query CUAD benchmark QA pairs and write responses JSONL' + + def wait(self) -> bool: + vector_store_conn = os.environ.get('VECTOR_STORE') + if not vector_store_conn: + return False + with VectorStoreFactory.for_vector_store(vector_store_conn) as vector_store: + return len(vector_store.get_index('chunk').top_k(QueryBundle(query_str='contract'), top_k=1)) == 0 + + def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): + data_dir = os.environ.get('BENCHMARK_DATA_DIR', 'benchmark-tests/data') + limit_str = os.environ.get('BENCHMARK_QA_LIMIT') + + run_benchmark_query( + handler, params, + dataset='cuad', + data_dir=data_dir, + graph_store_conn=os.environ.get('GRAPH_STORE'), + vector_store_conn=os.environ.get('VECTOR_STORE'), + response_llm=os.environ.get('TEST_RESPONSE_LLM', 'anthropic.claude-sonnet-4-20250514-v1:0'), + qa_limit=int(limit_str) if limit_str else None, + ) From 9f42287ceb4db6749ecb1234091c3fbcb41420fd Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Apr 2026 14:13:13 -0700 Subject: [PATCH 3/9] Move evaluation tools Signed-off-by: Andrew Carbonetto --- integration-tests/build-tests.sh | 4 + .../benchmark_evaluate.py | 6 +- .../benchmark_utils/__init__.py | 2 + .../benchmark_utils/answer_eval.py | 70 +++++ .../benchmark_utils/run_evaluation.py | 290 ++++++++++++++++++ 5 files changed, 367 insertions(+), 5 deletions(-) create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/__init__.py create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/answer_eval.py create mode 100644 integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index 4496beeb..5bfc200c 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -170,7 +170,11 @@ while [[ "$#" -gt 0 ]]; do done if [[ -z "$SSHCIDR" ]]; then +<<<<<<< HEAD echo "Auto-detecting public IPv4 address..." +======= + echo "Auto-detecting public IP address..." +>>>>>>> f249592b (Move evaluation tools) MY_IP=$(curl -4 -s --max-time 10 ifconfig.co) if [[ -z "$MY_IP" ]]; then echo "ERROR: Failed to auto-detect public IPv4 address. Please specify --ssh-cidr manually." diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py index 2a6b0595..e5f6612a 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py @@ -2,16 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 import json import os -import sys import unittest from typing import Dict, Any, Optional, List from graphrag_toolkit_tests.integration_test_base import IntegrationTestBase from graphrag_toolkit_tests.integration_test_handler import IntegrationTestHandler - -# Add benchmark-tests to path so we can import run_evaluation -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', 'benchmark-tests')) -from run_evaluation import CorrectnessEvaluator, IDKEvaluator +from graphrag_toolkit_tests.benchmark_utils.run_evaluation import CorrectnessEvaluator, IDKEvaluator def run_benchmark_evaluate(handler: IntegrationTestHandler, params: Dict[str, Any], diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/__init__.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/__init__.py new file mode 100644 index 00000000..04f8b7b7 --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/answer_eval.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/answer_eval.py new file mode 100644 index 00000000..4e59f875 --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/answer_eval.py @@ -0,0 +1,70 @@ +from bedrock_generator import BedrockGenerator +import argparse +import json +import multiprocessing as mp + +PROMPT = """You are an evaluator tasked with determining whether a specific statement is semantically entailed in a response. +Given the question: {question}: +Check whether the following statement is entailed/covered in the response. Response Yes/No without any other text. +Statement: {evidence} +Response: {response} +""" + +def run_evaluation(prompt): + response = generator.generate(text=[ + {"role": "user", "content": [{"type": "text", "text": prompt}]} + ]) + print(f"prompt: {prompt}") + print(f"entailment: {response}") + return response + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input-file-path", required=True, type=str) + parser.add_argument("--output-file-path", type=str) + parser.add_argument("--query-file-path", type=str) # only needed for multihop-rag + parser.add_argument("--dataset", type=str, choices = ["multihop-rag", "wikihowqa"]) + args = parser.parse_args() + generator = BedrockGenerator.from_config({"model_id": "us.anthropic.claude-3-sonnet-20240229-v1:0"}) + with open(args.input_file_path) as fin: + data = json.load(fin) + arguments = [] + entries = [] + cnt = 0 + + if args.dataset == "multihop-rag": + with open(args.query_file_path) as fin: + queries = json.load(fin) + for query, response in zip(queries, data): + for evidence in query[0]["evidence_list"]: + prompt = PROMPT.format(evidence=evidence["fact"], response=response["response"]) + arguments.append(prompt) + + elif args.dataset == "wikihowqa": + for entry in data: + evidences = entry["answers"][0].split(".") + for evidence in evidences: + if len(evidence) ==0: + continue + prompt = PROMPT.format(question = entry["raw_question"], evidence=evidence, response=entry["response"]) + arguments.append(prompt) + entries.append({"question": entry["raw_question"], "evidence": evidence, "response": entry["response"]}) + else: + raise NotImplemented + + pool = mp.Pool(16) + responses = pool.map(run_evaluation, arguments) + correct = 0 + total = 0 + for idx, response in enumerate(responses): + if "Yes" in response: + correct += 1 + entries[idx]["judgement"] = "Yes" + else: + entries[idx]["judgement"] = "No" + total += 1 + print(correct, total) + print(correct / total) + + with open(args.output_file_path, 'w') as output_file: + json.dump(entries, output_file, indent=2) diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py new file mode 100644 index 00000000..ba586896 --- /dev/null +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py @@ -0,0 +1,290 @@ +from boto3 import Session +from botocore.config import Config +import logging +import json +import argparse +from tqdm import tqdm +import os +import time + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def call_bedrock_invoke_model(prompt, bedrock, model_id, is_json_output=True): + while True: + try: + accept = 'application/json' + contentType = 'application/json' + + if 'anthropic.claude-3' in model_id: + payload_body = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + "temperature": 0.0, + "top_p": 1, + "top_k": 50, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + } + ] + } + body = json.dumps(payload_body) + else: + body = json.dumps({"prompt": prompt, "max_tokens_to_sample": 2048, "temperature": 0.0, "top_p": 1, "top_k": 50, + "stop_sequences": ["\\n\\nHuman:"]}) + + response = bedrock.invoke_model(body=body, modelId=model_id, accept=accept, contentType=contentType) + + if 'anthropic.claude-3' in model_id: + response = response['body'].read().decode('utf-8') + response = json.loads(response) + response_text = response['content'][0]['text'] + + else: + response_obj = response.get('body').read().decode('utf-8') + print("RESPONSE OBJ: " + str(response_obj)) + json_response_obj = json.loads(response_obj) + response_text = json_response_obj['completion'] + if is_json_output: + try: + start_idx = response_text.find("{") + end_idx = response_text.find("}") + parsed_completion = response_text[start_idx:end_idx + 1] + parsed_json = json.loads(parsed_completion) + parsed_json['llm_response'] = response_text + return parsed_json + except: + logger.error(response_text) + return { + 'grade': "incorrect", + 'justification': "LLM failed grading", + 'llm_response': response_text + } + else: + return response_text + except Exception as e: + logger.error(str(e)) + time.sleep(3) + + + +BKB_CORRECTNESS_GRADING = """ +Human: +You are a teacher grading a quiz. +You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect. + +Example Format: +QUESTION: question here +STUDENT ANSWER: student's answer here +TRUE ANSWER: true answer here +GRADE: Correct or Incorrect here + +Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student fails to answers or claims that the search results do not mention the answer then mark as incorrect. Begin! + +QUESTION: {query} +STUDENT ANSWER: {answer} +TRUE ANSWER: {expected_answer} +GRADE: + +Your response should be in json format as follows: +{{ + "grade": (correct or incorrect), + "justification": (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect. Use one or two sentences maximum. Keep the answer as concise as possible.) +}} + + +Assistant: +""" + +IDK_DETECTION = """You are a teacher grading a quiz. Based on students' response, you are asked to determine if the students think they can not answer the question because some information are missing. +Response: {response} +Please output "Unanswerable" if the students identify that they can not answer the question. Otherwise, output "Answerable". +""" +import os +os.environ["AWS_REGION_NAME"] = "us-west-2" + + +class GenerationEvaluator: + bedrock = Session().client( + service_name='bedrock-runtime', + region_name="us-west-2", + config=Config( + max_pool_connections=50, + retries={"max_attempts": 10, "mode": "standard"}, + connect_timeout=500, + read_timeout=500, + region_name="us-west-2" + )) + + + def __init__(self, model_id): + self.model_id = model_id + +class CorrectnessEvaluator(GenerationEvaluator): + def __init__(self, model_id): + super().__init__(model_id) + + def evaluate(self, question, answer, response): + grading = {} + grading.update(self._llm_evaluate(question, answer, response)) + return grading + + def _llm_evaluate(self, question, answer, response): + prompt = BKB_CORRECTNESS_GRADING.format( + query=question, + answer=response, + expected_answer=answer + ) + completion = call_bedrock_invoke_model(prompt, self.bedrock, model_id=self.model_id) + if answer == "": + completion['grade'] = "incorrect" + completion['justification'] = "No answer was provided" + + if not completion or not completion['grade'] or not completion['justification']: + logger.error("Failed to grade") + logger.error(str(completion)) + return { + 'question': question, + 'llmCorrectnessGrade': "incorrect", + 'llmCorrectnessGradeJustification': "LLM failed grading", + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + + try: + grading = { + 'question': question, + 'llmCorrectnessGrade': completion['grade'].lower(), + 'llmCorrectnessGradeJustification': completion['justification'].replace("\"", "\\\""), + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + return grading + except Exception as e: + logger.info(str(e)) + return { + 'question': question, + 'llmCorrectnessGrade': "incorrect", + 'llmCorrectnessGradeJustification': "LLM failed grading", + 'llm_response': completion.get('llm_response', str(completion)) # Store the raw response + } + + +class IDKEvaluator(GenerationEvaluator): + def __init__(self, model_id): + super().__init__(model_id) + + def evaluate(self, question, answer, response): + grading = {} + grading.update(self._llm_evaluate(question, answer, response)) + return grading + + def _llm_evaluate(self, question, answer, response): + prompt = IDK_DETECTION.format( + question=question, + answer=answer, + response=response + ) + completion = call_bedrock_invoke_model(prompt, self.bedrock, model_id=self.model_id, is_json_output=False) + if "Unanswerable" in completion: + return { + "label": "unanswerable" + } + else: + return { + "label": "answerable" + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input-file-path", type=str) + parser.add_argument("--metrics-output-path", type=str) + parser.add_argument("--eval-artifacts", type=str) + parser.add_argument("--metric", type=str, default="correctness", choices=["correctness", "idk", "correctness_on_answerable"]) + args = parser.parse_args() + + if args.metric == "correctness_on_answerable": + eval_correctness_artifact, eval_idk_artifact = args.eval_artifacts.split(",") + with open(eval_correctness_artifact) as fin: + eval_correctness_data = json.load(fin) + with open(eval_idk_artifact) as fin: + eval_idk_data = json.load(fin) + assert len(eval_correctness_data) == len(eval_idk_data) + total, count = 0, 0 + for correctness_eval, idk_eval in zip(eval_correctness_data, eval_idk_data): + if idk_eval["label"] == "answerable": + total += 1 + if correctness_eval["llmCorrectnessGrade"] == "correct": + count += 1 + logger.info("{}: {}".format(args.metric, count / total)) + else: + if args.metric == "correctness": + evaluator = CorrectnessEvaluator(model_id="anthropic.claude-3-sonnet-20240229-v1:0") + elif args.metric == "idk": + evaluator = IDKEvaluator(model_id="anthropic.claude-3-sonnet-20240229-v1:0") + + + data = [] + with open(args.input_file_path) as fin: + for line in fin: + data.append(json.loads(line)) + evaluation_outputs = [] + for example in tqdm(data): + answer = example["raw_example"]["answer"] + response = example["response"] + question = example["raw_example"]["question"] + evaluation_outputs.append(evaluator.evaluate(question, answer, response)) + count, total = 0, 0 + for evaluation in evaluation_outputs: + total += 1 + if args.metric == "correctness": + if evaluation["llmCorrectnessGrade"] == "correct": + count += 1 + if args.metric == "idk": + if evaluation["label"] == "unanswerable": + count += 1 + + logger.info("{}: {}".format(args.metric, count / total)) + os.makedirs(os.path.dirname(args.metrics_output_path), exist_ok=True) + with open(args.metrics_output_path, "w") as fout: + json.dump(evaluation_outputs, fout, indent=4) + with open(os.path.join(os.path.dirname(args.metrics_output_path), "{}.json".format(args.metric)), "w") as fout: + json.dump({ + args.metric: count / total + }, fout, indent=4) + +import multiprocessing as mp +import yaml + +class SafeCounter(): + # constructor + def __init__(self): + # initialize counter + self._counter = mp.Value('i', 0) + # initialize lock + self._lock = mp.Lock() + + # increment the counter + def increment(self): + # get the lock + with self._lock: + self._counter.value += 1 + + # get the counter value + def value(self): + with self._lock: + return self._counter.value + +def read_yaml(file_path): + if file_path: + with open(file_path, 'r') as file: + data = yaml.safe_load(file) + return data + else: + return {} \ No newline at end of file From e8eaf68527eccb3da28a8e70a77c50d9e1afb994 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Apr 2026 14:17:28 -0700 Subject: [PATCH 4/9] Fix build-tests.sh Signed-off-by: Andrew Carbonetto --- integration-tests/build-tests.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index 5bfc200c..46950a7d 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -170,11 +170,7 @@ while [[ "$#" -gt 0 ]]; do done if [[ -z "$SSHCIDR" ]]; then -<<<<<<< HEAD - echo "Auto-detecting public IPv4 address..." -======= echo "Auto-detecting public IP address..." ->>>>>>> f249592b (Move evaluation tools) MY_IP=$(curl -4 -s --max-time 10 ifconfig.co) if [[ -z "$MY_IP" ]]; then echo "ERROR: Failed to auto-detect public IPv4 address. Please specify --ssh-cidr manually." From 9b002edb4c298684e71e7b7971f8a73c2ce15937 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Apr 2026 09:27:21 -0700 Subject: [PATCH 5/9] Fix benchmarking tests Signed-off-by: Andrew Carbonetto --- integration-tests/build-tests.sh | 62 ++++++++++++++++++- integration-tests/env.template | 5 ++ .../graphrag_toolkit_tests/benchmark_build.py | 20 +++--- .../benchmark_evaluate.py | 7 ++- .../graphrag_toolkit_tests/benchmark_query.py | 7 ++- 5 files changed, 85 insertions(+), 16 deletions(-) diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index 46950a7d..ed3a95d4 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -81,6 +81,10 @@ if [[ "$#" -gt 0 ]]; then echo " --embeddings-model " echo " --embeddings-dimensions " echo " --ssh-cidr " + echo " --benchmark-data-dir " + echo " --benchmark-data-s3-uri " + echo " --benchmark-qa-limit " + echo " --benchmark-prototype" echo " --prev-stack " echo " --delete-on-pass" echo " --fail-fast" @@ -92,8 +96,10 @@ fi source ./.env ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -STACK_SUFFIX=$(date +%s) -GRAPH_NAME="gr-$STACK_SUFFIX" +if [[ -z "$STACK_PREFIX" ]]; then + STACK_PREFIX="gr" +fi +GRAPH_NAME="$STACK_PREFIX-$(date +%s)" TEST_DESCRIPTION="graphrag-toolkit integration test" TESTS="" PREV_STACK_NAME="" @@ -160,6 +166,10 @@ while [[ "$#" -gt 0 ]]; do --embeddings-dimensions) EMBEDDINGS_DIMENSIONS="$2"; shift ;; --toolkit-dir) GRAPHRAG_TOOLKIT_DIR="$2"; shift ;; --ssh-cidr) SSHCIDR="$2"; shift ;; + --benchmark-data-dir) BENCHMARK_DATA_DIR="$2"; shift ;; + --benchmark-data-s3-uri) BENCHMARK_DATA_S3_URI="$2"; shift ;; + --benchmark-qa-limit) BENCHMARK_QA_LIMIT="$2"; shift ;; + --benchmark-prototype) BENCHMARK_IS_PROTOTYPE=true ;; --prev-stack) PREV_STACK_NAME="$2"; shift ;; --delete-on-pass) DELETE_ON_PASS=True ;; --fail-fast) FAIL_FAST=True ;; @@ -234,6 +244,37 @@ cp -r $GRAPHRAG_TOOLKIT_DIR/examples/byokg-rag/* lexical-graph-examples cp -r ./../test-scripts/* lexical-graph-examples cp -r ./../source-data/* lexical-graph-examples +# Include benchmark data if local dir is specified and no S3 URI is provided +# Only copies dataset subdirectories that match the tests being run +if [[ "$BENCHMARK_DATA_DIR" ]] && [[ -z "$BENCHMARK_DATA_S3_URI" ]]; then + mkdir -p lexical-graph-examples/source-data + BENCHMARK_DATASETS="" + if echo "$TESTS" | grep -qi "Cuad"; then + if [[ "$BENCHMARK_IS_PROTOTYPE" == "true" ]]; then + BENCHMARK_DATASETS="$BENCHMARK_DATASETS cuad-prototype" + else + BENCHMARK_DATASETS="$BENCHMARK_DATASETS cuad" + fi + fi + if echo "$TESTS" | grep -qi "Pga"; then + BENCHMARK_DATASETS="$BENCHMARK_DATASETS pga" + fi + if echo "$TESTS" | grep -qi "Concurrentqa"; then + BENCHMARK_DATASETS="$BENCHMARK_DATASETS concurrentqa" + fi + if echo "$TESTS" | grep -qi "Wikihow"; then + BENCHMARK_DATASETS="$BENCHMARK_DATASETS wikihow" + fi + for ds in $BENCHMARK_DATASETS; do + if [[ -d "$BENCHMARK_DATA_DIR/$ds" ]]; then + echo "Including benchmark data for $ds from $BENCHMARK_DATA_DIR/$ds" + cp -r "$BENCHMARK_DATA_DIR/$ds" lexical-graph-examples/source-data/ + else + echo "WARNING: Benchmark data directory not found: $BENCHMARK_DATA_DIR/$ds" + fi + done +fi + echo "__version__ = '$toolkit_version.$current_timestamp'" >> ./graphrag-toolkit/graphrag_toolkit/lexical_graph/_version.py echo "export GRAPHRAG_TOOLKIT_S3_URI=$GRAPHRAG_TOOLKIT_S3_URI" >> lexical-graph-examples/.env.testing @@ -269,6 +310,18 @@ fi if [[ "$EMBEDDINGS_DIMENSIONS" ]]; then echo "export EMBEDDINGS_DIMENSIONS='$EMBEDDINGS_DIMENSIONS'" >> lexical-graph-examples/.env.testing fi +if [[ "$BENCHMARK_DATA_S3_URI" ]]; then + echo "export BENCHMARK_DATA_S3_URI='$BENCHMARK_DATA_S3_URI'" >> lexical-graph-examples/.env.testing +fi +if [[ "$BENCHMARK_DATA_DIR" ]] && [[ -z "$BENCHMARK_DATA_S3_URI" ]]; then + echo "export BENCHMARK_DATA_DIR='data'" >> lexical-graph-examples/.env.testing +fi +if [[ "$BENCHMARK_QA_LIMIT" ]]; then + echo "export BENCHMARK_QA_LIMIT=$BENCHMARK_QA_LIMIT" >> lexical-graph-examples/.env.testing +fi +if [[ "$BENCHMARK_IS_PROTOTYPE" ]]; then + echo "export BENCHMARK_IS_PROTOTYPE=$BENCHMARK_IS_PROTOTYPE" >> lexical-graph-examples/.env.testing +fi zip -r graphrag-toolkit.zip graphrag-toolkit # zip under directory @@ -288,7 +341,7 @@ zip -r graphrag.zip graphrag cp graphrag.zip ../target/graphrag.zip popd -rm -rf temp +# rm -rf temp pushd target unzip graphrag.zip @@ -324,6 +377,9 @@ echo "EMBEDDINGS_MODEL : $EMBEDDINGS_MODEL" echo "EMBEDDINGS_DIMENSIONS : $EMBEDDINGS_DIMENSIONS" echo "SSHCIDR : $SSHCIDR" echo "TESTS : $TESTS" +echo "BENCHMARK_DATA_DIR : $BENCHMARK_DATA_DIR" +echo "BENCHMARK_DATA_S3_URI : $BENCHMARK_DATA_S3_URI" +echo "BENCHMARK_QA_LIMIT : $BENCHMARK_QA_LIMIT" echo "PREV_STACK_NAME" : $PREV_STACK_NAME echo "----------------------------------------------------" echo "" diff --git a/integration-tests/env.template b/integration-tests/env.template index a2316d22..81743638 100644 --- a/integration-tests/env.template +++ b/integration-tests/env.template @@ -16,3 +16,8 @@ export TEST_EXTRACTION_LLM= export EMBEDDINGS_MODEL= export EMBEDDINGS_DIMENSIONS= + +export BENCHMARK_DATA_DIR= +export BENCHMARK_DATA_S3_URI= +export BENCHMARK_QA_LIMIT= +export BENCHMARK_IS_PROTOTYPE= diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py index 5d97539a..e70ed267 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py @@ -15,6 +15,10 @@ from graphrag_toolkit.lexical_graph.indexing.load import FileBasedDocs DATASET_CONFIG = { + 'cuad-prototype': { + 'num_docs': 2, + 'extracted_dir': os.path.join('extracted', '2026-04-16'), + }, 'cuad': { 'num_docs': 510, 'extracted_dir': os.path.join('extracted', '2026-02-17'), @@ -27,6 +31,7 @@ }, } +BENCHMARK_DATA_DIR = 'source-data' def run_benchmark_build(handler: IntegrationTestHandler, dataset: str, data_dir: str, graph_store_conn: Optional[str] = None, vector_store_conn: Optional[str] = None): @@ -98,15 +103,12 @@ def description(self): return 'Build graph and vector stores from CUAD pre-extracted chunks for benchmarking' def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): - data_dir = os.environ.get('BENCHMARK_DATA_DIR', 'benchmark-tests/data') graph_store_conn = os.environ.get('GRAPH_STORE') vector_store_conn = os.environ.get('VECTOR_STORE') + is_prototype = os.environ.get('BENCHMARK_IS_PROTOTYPE') + if is_prototype == 'true': + dataset_name = 'cuad-prototype' + else: + dataset_name = 'cuad' - run_benchmark_build(handler, 'cuad', data_dir, graph_store_conn, vector_store_conn) - - params['benchmark_dataset'] = 'cuad' - params['benchmark_data_dir'] = data_dir - if graph_store_conn: - params['benchmark_graph_store'] = graph_store_conn - if vector_store_conn: - params['benchmark_vector_store'] = vector_store_conn + run_benchmark_build(handler, dataset_name, BENCHMARK_DATA_DIR, graph_store_conn, vector_store_conn) diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py index e5f6612a..72f31d0c 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py @@ -118,12 +118,15 @@ def description(self): return 'Evaluate CUAD benchmark responses using LLM-as-judge correctness and IDK metrics' def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): + is_prototype = os.environ.get('BENCHMARK_IS_PROTOTYPE') + dataset_name = 'cuad-prototype' if is_prototype == 'true' else 'cuad' + responses_path = params.get('benchmark_responses_path', - os.path.join('benchmark-results', 'cuad', 'responses.jsonl')) + os.path.join('benchmark-results', dataset_name, 'responses.jsonl')) run_benchmark_evaluate( handler, params, - dataset='cuad', + dataset=dataset_name, responses_path=responses_path, metrics=['correctness', 'idk'], ) diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py index 0948424d..ccd456d6 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py @@ -17,6 +17,7 @@ QA_FILE_MAP = { 'cuad': ['qa.json'], + 'cuad-prototype': ['qa.json'], 'pga': ['pga_bio.json', 'pga_stat.json'], 'concurrentqa': ['qa.json'], } @@ -144,12 +145,14 @@ def wait(self) -> bool: return len(vector_store.get_index('chunk').top_k(QueryBundle(query_str='contract'), top_k=1)) == 0 def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): - data_dir = os.environ.get('BENCHMARK_DATA_DIR', 'benchmark-tests/data') + data_dir = os.environ.get('BENCHMARK_DATA_DIR') limit_str = os.environ.get('BENCHMARK_QA_LIMIT') + is_prototype = os.environ.get('BENCHMARK_IS_PROTOTYPE') + dataset_name = 'cuad-prototype' if is_prototype == 'true' else 'cuad' run_benchmark_query( handler, params, - dataset='cuad', + dataset=dataset_name, data_dir=data_dir, graph_store_conn=os.environ.get('GRAPH_STORE'), vector_store_conn=os.environ.get('VECTOR_STORE'), From 4abc8cb609e2244aeaad2fb2c7f39723ca0b5503 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 27 Apr 2026 08:57:16 -0700 Subject: [PATCH 6/9] Add benchmark data Signed-off-by: Andrew Carbonetto --- .../cuad-prototype/corpus.jsonl | 2 + ...o-Branding_Agreement__Agency_Agreement.txt | 227 ++++++++++++++++++ ..._06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt | 79 ++++++ .../benchmark-data/cuad-prototype/qa.json | 10 + integration-tests/build-tests.sh | 18 +- .../aws__abd5427c_6614_848bb4a4.json | 0 .../aws__abd5427c_6614_848bb4a4.json | 0 .../{source-data => }/corpus-modified.json | 0 .../{source-data => }/versioning/v1/readme.md | 0 .../{source-data => }/versioning/v2/readme.md | 0 .../{source-data => }/versioning/v3/readme.md | 0 .../{source-data => }/versioning/v4/readme.md | 0 12 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 integration-tests/benchmark-data/cuad-prototype/corpus.jsonl create mode 100644 integration-tests/benchmark-data/cuad-prototype/documents/2ThemartComInc_19990826_10-12G_EX-10.10_6700288_EX-10.10_Co-Branding_Agreement__Agency_Agreement.txt create mode 100644 integration-tests/benchmark-data/cuad-prototype/documents/ABILITYINC_06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt create mode 100644 integration-tests/benchmark-data/cuad-prototype/qa.json rename integration-tests/source-data/{source-data => }/collection-1/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json (100%) rename integration-tests/source-data/{source-data => }/collection-2/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json (100%) rename integration-tests/source-data/{source-data => }/corpus-modified.json (100%) rename integration-tests/source-data/{source-data => }/versioning/v1/readme.md (100%) rename integration-tests/source-data/{source-data => }/versioning/v2/readme.md (100%) rename integration-tests/source-data/{source-data => }/versioning/v3/readme.md (100%) rename integration-tests/source-data/{source-data => }/versioning/v4/readme.md (100%) diff --git a/integration-tests/benchmark-data/cuad-prototype/corpus.jsonl b/integration-tests/benchmark-data/cuad-prototype/corpus.jsonl new file mode 100644 index 00000000..43729ed8 --- /dev/null +++ b/integration-tests/benchmark-data/cuad-prototype/corpus.jsonl @@ -0,0 +1,2 @@ +{"doc_id": "2ThemartComInc_19990826_10-12G_EX-10.10_6700288_EX-10.10_Co-Branding_Agreement__Agency_Agreement.txt", "text": "CO-BRANDING AND ADVERTISING AGREEMENT\n\nTHIS CO-BRANDING AND ADVERTISING AGREEMENT (the \"Agreement\") is made as of June 21, 1999 (the \"Effective Date\") by and between I-ESCROW, INC., with its principal place of business at 1730 S. Amphlett Blvd., Suite 233, San Mateo, California 94402 (\"i-Escrow\"), and 2THEMART.COM, INC. having its principal place of business at 18301 Von Karman Avenue, 7th Floor, Irvine, California 92612 (\"2TheMart\").\n\n1. DEFINITIONS.\n\n(a) \"CONTENT\" means all content or information, in any medium, provided by a party to the other party for use in conjunction with the performance of its obligations hereunder, including without limitation any text, music, sound, photographs, video, graphics, data or software. Content provided by 2TheMart is referred to herein as \"2TheMart Content\" and Content provided by i-Escrow is referred to herein as \"i-Escrow Content.\"\n\n(b) \"CO-BRANDED SITE\" means the web-site accessible through Domain Name, for the Services implemented by i-Escrow. The homepage of this web-site will visibly display both 2TheMart Marks and i-Escrow Marks.\n\n(c) \"CUSTOMERS\" means all users who access Co-Branded Site.\n\n(d) \"DOMAIN NAME\" means www.iescrow.com/2TheMart.\n\n(e) \"ESCROW SERVICES\" means services for auction sellers and high bidders whereby an agent holds a buyer's money in trust until the buyer approves the applicable item that was physically delivered, at which time the agent releases the buyer's money to seller, after subtracting the escrow fees.\n\n(f) \"INFORMATION TRANSFER MECHANISM\" means the mechanism by which 2TheMart transfers to i-Escrow information to populate the applicable i-Escrow transaction and user registration forms.\n\n(g) \"LAUNCH DATE\" means the first date on which the Co-Branded Site is pointed to in all references to i-Escrow from 2TheMart auction site, and the Information Transfer Mechanism is publicly deployed (post-beta).\n\n(h) \"MARKS\" means all domain names, trademarks and logos designated by a party for the other party's use in conjunction with such other party's performance under this Agreement. Marks designated by 2TheMart for i-Escrow's use are referred to herein as \"2TheMart Marks\" and Marks designated by i-Escrow for 2TheMart' use are referred to herein as \"i-Escrow Marks.\"\n\n(i) \"SERVICES\" means i-Escrow's implementation and performance of the Escrow Services as of the Effective Date, as modified over time.\n\n(j) \"SHADOW SITE\" means the site where Co-Branded Site is made available for 2TheMart's testing of the Information Transfer Mechanism prior to being made publicly available.\n\n(k) \"TRANSACTION\" means a transaction utilizing the Services that actually closes and that was initiated by a Transaction Inquiry from a Customer.\n\n(l) \"TRANSACTION INQUIRY\" means a Customer's submission of i-Escrow's standard New Transaction Inquiry form (or its successor) on or through the Co-Branded Pages. Currently this means entry of a description and price of merchandise by a user (buyer or seller) who agrees to abide by the terms and conditions of the Services, together with email address of the other party, regardless of whether or not any Transaction is completed.\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\n2. DEVELOPMENT AND IMPLEMENTATION.\n\n2.1 OVERVIEW. As set forth herein, 2TheMart will promote Services to its auction users (buyers and sellers), and i-Escrow shall develop Co-Branded Site, and develop the Information Transfer Mechanism working with 2TheMart to make Services available seamlessly to Customers. Unless otherwise specified, each party shall be responsible for all development, hosting and other costs associated with the pages resident on their servers and all emails to users they send.\n\n2.2 INITIAL INFORMATION TRANSFER MECHANISM DEVELOPMENT. The parties shall negotiate in good faith to determine the initial operation of the Information Transfer Mechanism and to describe such operation and development fees, in a statement of work (\"SOW\"). Each party shall make available sufficient and qualified engineers to negotiate the SOW. No SOW shall be binding on the parties unless mutually approved by both parties. In the event that the parties are unable to agree to an SOW within 2 months following the Effective Date, either party may, in its sole discretion, terminate this Agreement by providing written notice.\n\nOnce approved, the parties shall use commercially reasonable efforts to diligently implement their respective obligations under the SOW. Upon completion of its duties under the SOW, a party shall notify the other party and provide the other party with the opportunity to test and evaluate its work. i-Escrow shall make available the Shadow Site for such testing in a timely manner. Each party shall reasonably cooperate with the other party in effectuating their respective duties under the SOW. The Information Transfer Mechanism shall not go live until its operation has been approved (\"Approval Date\") by both parties, such approval not to be unreasonably withheld.\n\n2.3 LAUNCH TIMING. Each party shall use good faith and reasonable efforts to expeditiously develop the Co-Branded Pages and the Information Transfer Mechanism. In the event that, after using such efforts, the Launch Date has not occurred within 4 months following the Effective Date, either party may terminate this Agreement by providing written notice. If\n\nonly one party has used good faith and reasonable development efforts, only that party may exercise the foregoing right to terminate.\n\n2.4 RESTRICTIONS ON COMMUNICATIONS. i-Escrow may place banner advertising on the Co-Branded Site upon prior written approval of 2TheMart, which shall be at the discretion of 2TheMart. All advertising revenue arising from the banner ads shall be solely i-Escrow's. i-Escrow shall not run banner advertisements on the Co-Branded Site for any of 2TheMart's competitors. 2TheMart shall provide in writing, a list of companies they would like to exclude, including every time they wish to change this list.\n\n2.5 SERVICE PERFORMANCE OF INFORMATION TRANSFER MECHANISM. The parties each shall in good faith work to provide reasonable service levels with respect to the operation of the portions of the Information Transfer Mechanism in their control.\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\n2.6 PROGRAM REVIEW MEETINGS. The parties shall meet, at least once per month either in person, or by telephone, to coordinate the implementation of this agreement over time.\n\n3. PROMOTION.\n\nAfter Launch Date, 2TheMart will widely promote the Services:\n\n(a) To every seller and high bidder through means including, but not limited to, end of auction emails containing links, such that, it shall be possible for the buyer or seller to initiate a Transaction Inquiry with i-Escrow, without having to re-enter all their personal or transaction related information.\n\n(b) By adding links to Co-Branded Site in FAQ section of 2TheMart auctions.\n\n(c) By adding links to Co-Branded Site on the seller listing pages of 2TheMart auctions.\n\n(d) By displaying a text or graphic link to a page containing information about Services on all auction item pages and bidding pages to educate bidders about i-Escrow. 2TheMart may use the \"Escrow Services Description\" attached in Exhibit A for creating such a page.\n\n5. PAYMENT.\n\n5.1 ADVERTISING FEES. After the Launch Date, i-Escrow shall pay 2TheMart advertising fees based on the number of Transaction Inquiries. This advertising fees shall consist of a per Transaction Inquiry amount calculated by multiplying 0.025% by the amount of the average Transaction from all Customers in the preceding quarter. The formula for arriving at the per Transaction Inquiry amount may be revised from time to time during the term of this Agreement to reflect present market conditions (\"the Adjusted Rate\"), but only by mutual\n\nconsent of the parties after good faith discussions. The Adjusted Rate shall be added as an addendum to this Agreement.\n\n5.2 REPORTING. Within two (2) weeks following the end of each calendar quarter, i-Escrow shall provide to 2TheMart a report, describing for each quarter: the number of new registrations through the Co-Branded Pages; the number of Transaction Inquiries from Customers; the total number of Transactions from such inquiries; the total dollar value of the Transactions.\n\n5.3 AUDIT RIGHTS. i-Escrow shall keep for one (1) year proper records and books of account relating to the computation of advertising payments owed to 2TheMart (including, as appropriate, the computation of the size of average Transaction). Once every twelve (12) months, 2TheMart through a CPA may inspect and audit such records to verify reports. Any such inspection will be conducted in a manner that does not unreasonably interfere with i-Escrow's business activities and with no less than fifteen (15) days notice. i-Escrow shall within two (2) weeks make any overdue payments disclosed by the audit. Such inspection shall be at 2TheMart's expense; however, if the audit reveals overdue payments in excess of ten percent (10%) of the payments owed to date, i-Escrow shall immediately pay all cost of such audit.\n\n6. RIGHTS AND STANDARDS.\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\n6.1 CONTENT. 2TheMart hereby grants to i-Escrow a worldwide, non-exclusive right to use, reproduce, distribute, publicly perform, publicly display and digitally perform the 2TheMart Content soley with respect to and in conjunction with the Co-Branded Site all with the prior written consent of 2TheMart, for the term of this Agreement. i-Escrow hereby grants to 2TheMart a worldwide, non-exclusive right to use, reproduce, distribute, publicly perform, publicly display and digitally perform the i-Escrow Content on or in conjunction with 2TheMart auctions.\n\n6.2 CONTENT OWNERSHIP. Except as otherwise provided in this Agreement, as between 2TheMart and i-Escrow: (a) 2TheMart and its suppliers retain all rights, title and interest in and to all intellectual property rights embodied in or associated with the 2TheMart Content, and b) i-Escrow and its suppliers retain all rights, title and interest in and to all intellectual property rights embodied in or associated with the i-Escrow Content and Co-Branded Site. There are no implied licenses under this Agreement, and any rights not expressly granted are reserved. Neither party shall exceed the scope of the rights granted hereunder.\n\n6.3 TRADEMARKS. Subject to the terms and conditions of this Agreement: (a) i-Escrow hereby grants to 2TheMart a non-exclusive, nontransferable right to use the i-Escrow Marks (including without limitation the Domain Name) in links to and advertisements and promotions for the Co-Branded Pages or the Services; and (b) 2TheMart hereby grants to i-Escrow a non-exclusive, nontransferable right to use 2TheMart Marks (including without limitation the Domain Name) on the Co-Branded Pages, and for the performance of Services.\n\n6.4 TRADEMARK RESTRICTIONS. The Mark owner may terminate the foregoing rights if, in its reasonable discretion, the other party's use of the Marks tarnishes, blurs or dilutes the quality associated with the Marks or the associated goodwill and such problem is not cured within ten (10) days of notice of breach; alternatively, instead of terminating the right in total, the\n\nowner may specify that certain pages of the other party's web-site may not contain the Marks. Title to and ownership of the owner's Marks shall remain with the owner. The receiving party shall use the Marks exactly in the form provided and in conformance with any trademark usage policies. The other party shall not take any action inconsistent with the owner's ownership of the Marks, and any benefits accruing from use of such Marks shall automatically vest in the owner. The other party shall not form any combination marks with the other party's Marks. Notwithstanding the foregoing, to the extent that the Domain Name is deemed a combination mark, neither party shall use the Domain Name for any purpose except as expressly provided herein or attempt to register the Domain Name, and the parties will jointly cooperate on any enforcement action of infringement of the Domain Name.\n\n6.5 LIMITS ON SUBLICENSING. All rights (under any applicable intellectual property right) granted herein are not sublicenseable,\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\ntransferable or assignable. Notwithstanding the foregoing, either party may use a third party web host, but all actions or failures to act of the web host that would be a breach of this Agreement, were the actions or failures to act taken by the applicable party, shall be deemed a breach of this Agreement. In addition, 2TheMart may grant sublicenses to companies that 2TheMart has a business relationship with to the extent that 2TheMart Content is visible from such company's web-site through a link or other means.\n\n6.6 CONTENT STANDARDS. 2TheMart shall not provide any 2TheMart Content, and i-Escrow shall not provide any i-Escrow Content, that: (a) infringes any third party's copyright, patent, trademark, trade secret or other proprietary rights or rights of publicity or privacy; (b) violates any law, statute, ordinance or regulation (including without limitation the laws and regulations governing export control, unfair competition, antidiscrimination or false advertising); (c) is defamatory, trade libelous, unlawfully threatening or unlawfully harassing; (d) is obscene, harmful to minors or child pornographic; (e) contains any viruses, Trojan horses, worms, time bombs, cancelbots or other computer programming routines that are intended to damage, detrimentally interfere with, surreptitiously intercept or expropriate any system, data or personal information; and (f) is materially false, misleading or inaccurate.\n\n6.7 SERVICE STANDARDS. i-Escrow will comply with all laws and regulations and act as an Independent Escrow Agent as per the guidelines of California Escrow Law (California Financial Code Section17000 et seq., or its successor). Should any of the terms, conditions or provisions of this Agreement conflict with the California Escrow Law, its rules or regulations, which govern i-Escrow's business practices, the California Escrow Law shall prevail. Notwithstanding the foregoing, at any time that i-Escrow reasonably believes such a conflict exists, i-Escrow will give 2TheMart written notice of such conflict and the parties will use their best efforts to resolve such conflict.\n\n7. DISCLAIMER OF WARRANTIES. EACH PARTY PROVIDES ALL MATERIALS AND SERVICES TO THE OTHER PARTY \"AS IS.\" EACH PARTY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED OR STATUTORY, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF TITLE, NON-\n\nINFRINGEMENT, MERCHANTABILITY ANDFITNESS FOR A PARTICULAR PURPOSE. Each party acknowledges that it has not entered into this Agreement in reliance upon any warranty or representation except those specifically set forth herein.\n\n8. TERM AND TERMINATION.\n\n8.1 TERM. The term of this Agreement shall continue for one (1) year following the Launch Date, unless earlier terminated as provided herein. This Agreement may be renewed for any number of successive one (1) year terms by mutual written agreement of the parties prior to the conclusion of the term of this Agreement. A party wishing to renew this Agreement shall give the other party notice thereof no less than thirty (30) days before the expiration of the term then in effect. In the event that either party does not give such notice, the term of this Agreement shall be automatically renewed for another one (1) year.\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\n8.2 TERMINATION FOR BREACH. In addition to other remedies that may be available to it, by providing written notice, a party may immediately terminate this Agreement: (a) if the other party materially breaches this Agreement and fails to cure that breach within sixty (60) days after receiving written notice of the breach, or (b) as provided in Sections 2.2 [INITIAL INFORMATION TRANSFER MECHANISM DEVELOPMENT], 2.4 [RESTRICTIONS ON COMMUNICATIONS], or 12.4.\n\n8.3 TERMINATION FOR CHANGE IN COMPANY STRUCTURE. If a majority of the equity securities of either 2TheMart or i-Escrow, Inc. (except that i-Escrow may sell all or a majority of its equity securities or voting interests to i-Escrow.com, and i-Escrow.com may sell all or a majority of its equity securities or voting interests to i-Escrow's existing shareholders, without triggering the foregoing) are acquired by another company during the term of this Agreement either company may terminate this Agreement, without liability, by giving a thirty (30) days written notice to the other party.\n\n8.4 TERMINATION FOR BANKRUPTCY. Either party may terminate or suspend this Agreement effective immediately and without liability upon written notice to the other party if any one of the following events occurs:\n\n(a) the other party files a voluntary petition in bankruptcy or otherwise seeks protection under any law for the protection of debtors;\n\n(b) a proceeding is instituted against the other party under any provision of any bankruptcy laws which is not dismissed within ninety (90) days;\n\n(c) the other party is adjudged bankrupt;\n\n(d) a court assumes jurisdiction of all or a substantial portion of the assets of the other party under a reorganization law;\n\n(e) a trustee or receiver is appointed by a court for all or a substantial portion of the assets of the other party;\n\n(f) the other party becomes insolvent, ceases or suspends all or substantially all of its business; or\n\n(g) the other party makes an assignment of the majority of its assets for the benefit of its creditors.\n\n8.5 EFFECTS OF TERMINATION. Upon expiration or termination of this Agreement for any reason: (a) all rights granted herein shall terminate, (b) i-Escrow shall pay all amounts owed to 2TheMart within six (6) weeks of termination, and (c) each party shall remove the other party's content and Marks from their servers. Notwithstanding the foregoing, unless this Agreement was terminated for a material breach, all provisions of this Agreement shall survive to the extent necessary for i-Escrow to complete any Customer transactions which are pending at the time of expiration or termination. Sections 1, 7, 8.5 [EFFECTS OF TERMINATION], 9, 10, 11 and 12 shall survive expiration or termination of this Agreement.\n\n9. INDEMNITY. Each party (the \"Indemnifying Party\") shall indemnify the other party (the \"Indemnified Party\") against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which the Indemnified Party may incur as a result of claims in any form by third parties arising from the Indemnifying Party's acts, omissions or misrepresentations to the extent that the Indemnified Party is deemed a principal of the Indemnifying Party. In addition, 2TheMart shall indemnify i-Escrow against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which i-Escrow may incur as a result of claims in any form by third parties arising from 2TheMart Content. In addition, i-Escrow shall indemnify 2TheMart against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which 2TheMart may incur as a result of claims in any form by third parties arising from i-Escrow\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\nContent and or the Services provided to Customers. The foregoing obligations are conditioned on the Indemnified Party: (i) giving the Indemnifying Party notice of the relevant claim, (ii) cooperating with the Indemnifying Party, at the Indemnifying Party's expense, in the defense of such claim, and (iii) giving the Indemnifying Party the right to control the defense and settlement of any such claim, except that the Indemnifying Party shall not enter into any settlement that affects the Indemnified Party's rights or interest without the Indemnified Party's prior written approval. The Indemnified Party shall have the right to participate in the defense at its expense.\n\n10. LIMITATION ON LIABILITY. EXCEPT IN THE EVENT OF A BREACH OF SECTION 11, NEITHER PARTY SHALL BE LIABLE FOR SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OR LOST PROFITS (HOWEVER ARISING, INCLUDING NEGLIGENCE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT, EVEN IF THE PARTIES ARE AWARE OF THE POSSIBILITY OF SUCH DAMAGES.\n\n11. CONFIDENTIAL INFORMATION. A party's \"Confidential Information\" is defined as any confidential or proprietary information of a party which is disclosed to the other party in a writing marked confidential or, if disclosed orally, is identified as confidential at the time of disclosure and is subsequently reduced to a writing marked confidential and delivered to the\n\nother party within ten (10) days of disclosure. Each party shall hold the other party's Confidential Information in confidence and shall not disclose such Confidential Information to third parties nor use the other party's Confidential Information for any purpose other than as required to perform under this Agreement. Such restrictions shall not apply to Confidential Information which (a) is already known by the recipient, (b) becomes, through no act or fault of the recipient, publicly known, (c) is received by recipient from a third party without a restriction on disclosure or use, or (d) is independently developed by recipient without reference to the Confidential Information. The restriction on disclosure shall not apply to Confidential Information which is required to be disclosed by a court or government agency. Upon expiration or termination of this Agreement, within fourteen (14) days of the other party's request, each party will return all Confidential Information and other deliverables to the requesting party.\n\n12. GENERAL PROVISIONS.\n\n12.1 GOVERNING LAW. This Agreement will be governed and construed in accordance with the laws of the State of California without giving effect to conflict of laws principles. Both parties submit to personal jurisdiction in California and further agree that any cause of action arising under this Agreement shall be brought in a court in Orange County, California.\n\n12.2 SEVERABILITY; HEADINGS. If any provision herein is held to be invalid or unenforceable for any reason, the remaining provisions will continue in full force without being impaired or invalidated in any way. The parties agree to replace any invalid provision with a valid provision that most closely approximates the intent and economic effect of the invalid provision. Headings are for reference purposes only and in no way define, limit, construe or describe the scope or extent of such section.\n\n12.3 PUBLICITY. Prior to the release of any press releases or other similar promotional materials related to this Agreement, the releasing party shall submit a written request for approval to the other party with a copy of the materials to be released, which\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\nrequest shall be made no less than three (3) business days prior to the requested release date. A party shall not unreasonably withhold or delay the granting of its approval of such materials, and such approval shall be provided to the other party within one (1) business day of receipt\n\n12.4 FORCE MAJEURE. Except as otherwise provided, if performance hereunder (other than payment) is prevented, restricted or interfered with by any act or condition whatsoever beyond the reasonable control of a party (a \"force majeure event\"), the party so affected, upon giving prompt notice to the other party, shall be excused from such performance to the extent of such prevention, restriction or interference. However, if a force majeure event interferes with the operation of this Agreement for sixty (60) days or more, either party can terminate this Agreement, without penalty. Notwithstanding the foregoing, the occurrence of any force majeure event shall not limit either party's obligations under Section 9 with respect to any third party claim as to which the other party seeks indemnification.\n\n12.5 INDEPENDENT CONTRACTORS. The parties are independent contractors, and no agency, partnership, joint venture, employee- employer or franchisor-franchisee relationship is\n\nintended or created by this Agreement. Neither party shall make any warranties or representations on behalf of the other party.\n\n12.6 NOTICE. Any notices hereunder shall be given to the appropriate party at the address specified below or at such other address as the party shall specify in writing. Notice shall be deemed given: upon personal delivery; if sent by fax, upon confirmation of receipt; or if sent by a reputable overnight courier with tracking capabilities, one (1) day after the date of mailing: To i-Escrow: i-Escrow, Inc. 1730 South Amphlett Blvd., #215 San Mateo, CA 94402 Fax no. (650) 638-7890 Attention: President\n\nWith copy to: Fred M. Greguras, Esq. Legal Counsel of i-Escrow Fenwick & West LLP Two Palo Alto Square Palo Alto, CA 94306\n\nTo 2TheMart: Dominic J. Magliarditi President 18301 Von Karman Avenue, 7th Floor Irvine, CA 92612 Fax no. (949) 477-1221\n\n11.7 COUNTERPARTS. This Agreement may be executed in one or more counterparts, each of which shall be deemed an original and all of which shall be taken together and deemed to be one instrument.\n\n12.8 GOOD FAITH. The parties agree to act in good faith with respect to each provision of this Agreement and any dispute that may arise related hereto.\n\n12.9 ADDITIONAL DOCUMENTS/INFORMATION. The parties agree to sign and/or provide such additional documents and/or information as may reasonably be required to carry out the intent of this Agreement and to effectuate its purposes.\n\n12.10 RIGHTS AND REMEDIES CUMULATIVE. The rights and remedies provided herein will be cumulative and not exclusive of any other rights or remedies provided by law or otherwise.\n\n12.11 NONWAIVER. No failure or forbearance by either party to exercise any right or insist upon or enforce performance of any obligation hereunder shall be deemed a waiver or relinquishment to any extent of that or any other right or obligation, in that or any other instance; rather, the\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999\n\n\n\n\n\nsame shall be and shall remain in full force and effect. Any waiver of any right of a party or any obligation of the other party hereunder must be made in a writing signed by the arty waiving such right or obligation.\n\n12.12 ENTIRE AGREEMENT. This Agreement contains the entire understanding of the parties hereto with respect to the transactions and matters contemplated hereby, supersedes all previous Agreements between i-Escrow and 2TheMart concerning the subject matter (except for the Confidential Agreement Dated January 4 1999, which shall survive this Agreement). No amendments or supplements to this Agreement will be effective for any purpose except by a written Agreement signed by the parties. No party hereto has relied on any statement, representation or promise of any party or with any other officer, agent, employee or attorney for the other party in executing this Agreement except as expressly stated herein.\n\n2THEMART.COM, INC.: I-ESCROW, INC.:\n\nBy:/s/Dominic J. Magliarditi By:/s/Sanjay Bajaj Name: Dominic J. Magliarditi Name: Sanjay Bajaj Title: President Title: VP Business Development Date: 6/21/99 Date: 6/11/99\n\n EXHIBIT A\n\nESCROW SERVICES DESCRIPTION\n\nSuccessful completion of a transaction involves exchange of merchandise with payment. The buyer has to be satisfied he/she received what they thought they were getting and the seller has to be sure he/she gets paid. i-Escrow holds payment from the buyer in trust until the seller sends the merchandise to the buyer. Once the buyer accepts the merchandise, i-Escrow forwards the payment to the seller by writing a check. A typical escrow transaction: When an auction ends, your end of auction email contains links to i-Escrow. Once you have signed up with i-Escrow you go through the following steps to complete your transaction. 1. Start a transaction by entering the description and price of the merchandise along with email address of the other party. 2. The other party receives an email from i-Escrow requesting an acknowledgement of the terms of the transaction. 3. Once the transaction is acknowledged by the other party, the buyer pays i-Escrow the agreed upon price, by credit card or other means. 4. i-Escrow informs the seller that payment has been received, requesting them to ship the merchandise directly to the buyer. 5. The seller provides i-Escrow with the tracking number of the shipment. 6. The buyer receives and accepts the merchandise. 7. i-Escrow sends the check to the seller.\n\nFor more information about I-Escrow, visit their web-site at www.iescrow.com\n\nSource: 2THEMART COM INC, 10-12G, 8/26/1999"} +{"doc_id": "ABILITYINC_06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt", "text": "EXHIBIT 4.25 INFORMATION IN THIS EXHIBIT IDENTIFIED BY [ * * * ] IS CONFIDENTIAL AND HAS BEEN EXCLUDED BECAUSE IT IS BOTH (I) NOT MATERIAL AND (II) WOULD LIKELY CAUSE COMPETITIVE HARM TO THE REGISTRANT IF PUBLICLY DISCLOSED. SERVICES AGREEMENT This Services Agreement (this \"Agreement\") is entered into on October 1, 2019 and is made effective as of November 1, 2019 (the \"Effective Date\"), by and between [ * * * ] (the \"Provider\"), and TELCOSTAR PTE, LTD., a company organized and existing under the laws of Singapore and Ability Computer & Software Industries Ltd, a company organized and existing under the laws of the State of Israel (each and both of them \"Recipient\"). Each of the foregoing parties is referred to herein as a \"Party\" and together as the \"Parties\". RECITALS A. Recipient wishes to engage the Provider to provide certain services and resources (the \"Services\") and Provider desires to provide Recipient with the Services all in accordance with the terms and conditions set forth herein. AGREEMENT The Parties hereby agree as follows: 1. Services. 1.1 Provision of Services. (a) Provider agrees to provide the Services set forth on the Exhibit A attached hereto (as such Exhibit may be amended or supplemented pursuant to the terms of this Agreement, the \"Exhibit\") to Recipient for the respective periods and on the other terms and conditions set forth in this Agreement and in the Exhibit. Notwithstanding the contents of the Exhibit, Provider agrees to respond in good faith to any reasonable request by Recipient for access to any additional services and resources that are necessary for the operation of the Recipient and which are not currently contemplated in the Exhibit, at a price to be agreed upon after good faith negotiations between the Parties. Any such additional services and resources so provided by Provider shall constitute Services under this Agreement and be subject in all respect to the provisions of this Agreement as if fully set forth on the Exhibit as of the date hereof. (b) Recipient may freely assign its rights under this Agreement to receive the Services to any of its affiliates. 1.2 Standard of Service. (a) Provider represents, warrants and agrees that the Services shall be provided in good faith, in accordance with applicable law and in a manner generally consistent with the historical provision of the Services and with the same standard of care as historically provided. (b) Provider shall maintain complete and accurate records relating to the provision of the Services under this Agreement, in such form as Recipient shall approve.\n\n\n\n\n\n(c) Provider shall use its best efforts to provide for employees or contractors to perform the Services, each of whose names, positions, and respective levels of experience and relevant licenses shall be set out in Exhibit A attached hereto (collectively, the \"Provider Representatives\"). Provider may not make any change in the Provider Representatives without the prior consent of the Recipient. Provider Representatives shall be dedicated to solely providing the Services to Recipient and shall not provide any such services or resources to Provider or any other customer of Provider. (d) Recipient acknowledges that this Agreement does not create a fiduciary relationship, partnership, joint venture or relationships of trust or agency between the Parties and that all Services are provided by Provider as an independent contractor. (e) Notwithstanding anything to the contrary in this Section 1.2: (a) in the event that Provider uses any subcontractors to perform any Services, Provider is not released from responsibility for its obligations under this Agreement; (b) Provider shall remain fully responsible, financially and otherwise, for the Services provided by each subcontractor to the same extent as if Provider had performed the Services itself (subject to the limitations set forth in this Agreement) and agrees to pay the fees and expenses of any such subcontractor; (c) Provider shall remain ultimately responsible for ensuring that the Services are provided and any such subcontractor performs any such obligations in accordance with the terms of this Agreement, and (d) the obligations with respect to the nature, quality and standards of care set forth in Section 1.2 are satisfied with respect to any Service provided by any subcontractor. (f) Provider shall at all times during the term of this Agreement maintain, or cause to be maintained, the computer software and computer hardware that is used in connection with the Services with substantially the same degree of care, skill and diligence with which Provider maintains, or causes to be maintained, as of the Effective Date, such computer software and computer hardware for itself, consistent with past practices, as of the Effective Date, including without limitation, with respect to type, quality and timeliness of such maintenance. 1.3 Additional Services. Nothing in this Agreement shall be construed to prevent the Recipient from itself performing or from acquiring services from other providers that are similar to or identical to the Services. 1.4 Intellectual Property. (a) Recipient shall own, and Provider hereby irrevocably assigns to the Recipient, all rights, title, and interest in any invention, technique, process, device, discovery, improvement, or know-how, whether patentable or not and all other proprietary rights, industrial rights and any other similar rights, in each case on a worldwide basis, and all copies and tangible embodiments thereof, or any part thereof, in whatever form or medium hereafter made or conceived solely or jointly by Provider while working for or on behalf of the Recipient, which relate to, is suggested by, or results from the Services. (b) At Recipient's request, Provider shall disclose any such invention, technique, process, device, discovery, improvement, or know-how promptly to Recipient. Provider shall, upon request of Recipient, promptly execute a specific assignment of title to Recipient, and do anything else reasonably necessary to enable Recipient to secure for itself, patent, trade secret, or any other proprietary rights.\n\n2\n\n\n\n\n\n(c) All writings or works of authorship, including, without limitation, program codes or documentation, produced or authored by Provider in the course of performing services for the Recipient, together with any associated copyrights, are works made for hire and the exclusive property of the Recipient. To the extent that any writings or works of authorship may not, by operation of law, be works made for hire, this Agreement shall constitute an irrevocable assignment by Provider to the Recipient of the ownership of and all rights of copyright in, such items, and the Recipient shall have the right to obtain and hold in its own name, rights of copyright, copyright registrations, and similar protections which may be available in the works. Provider shall give the Recipient or its designees all assistance reasonably required to perfect such rights. 2. Compensation. 2.1 Responsibility for Wages and Fees. For such time as any employees of Provider are providing the Services to Recipient under this Agreement, (a) such employees will remain employees of Provider and shall not be deemed to be employees of Recipient for any purpose, and (b) Provider shall be solely responsible for the payment and provision of all wages, bonuses and commissions, employee benefits, including severance and worker's compensation, and the withholding and payment of applicable taxes relating to such employment. 2.2 Terms of Payment and Related Matters. (a) As consideration for provision of the Services following the Effective Date, Recipient shall pay Provider an amount equal to Provider's actual cost of providing the Services plus a 10% service fee. In addition to such amount, in the event that Provider incurs reasonable and documented out-of-pocket expenses in the provision of any Service, including, without limitation, license fees and payments to third-party service providers or subcontractors (such included expenses, collectively, \"Out-of-Pocket Costs\"), Recipient shall reimburse Provider for all such Out-of-Pocket Costs. (b) (i) Provider shall provide Recipient with monthly invoices (\"Invoices\"), which shall set forth in reasonable detail, with such supporting documentation as Recipient may reasonably request with respect to Out-of-Pocket Costs, amounts payable under this Agreement, and (ii) payments pursuant to this Agreement shall be made within fifteen (15) days after the date of receipt of an Invoice by Recipient from Provider. (c) Provider shall allow the Recipient to use [ * * * ] at no cost, until December 31, 2021. 2.3 Invoice Disputes. In the event of an Invoice dispute, Recipient shall deliver a written statement to Provider prior to the date payment is due on the disputed Invoice listing all disputed items and providing a reasonably detailed description of each disputed item. Amounts not so disputed shall be deemed accepted and shall be paid, notwithstanding disputes on other items. The Parties shall seek to resolve all such disputes expeditiously and in good faith. Provider shall continue performing the Services in accordance with this Agreement pending resolution of any dispute.\n\n3\n\n\n\n\n\n2.4 No Right of Setoff. Each of the Parties hereby acknowledges that it shall have no right under this Agreement to offset any amounts owed (or to become due and owing) to the other Party, whether under this Agreement, the Purchase Agreement or otherwise, against any other amount owed (or to become due and owing) to it by the other Party. 3. Termination. 3.1 Termination of Agreement. This Agreement be deemed effective as of the Effective Date, Agreement and shall terminate on December 31, 2020, unless terminated earlier in accordance with Section 3.2. 3.2 Each of the Recipient and the Provider may, in their sole discretion, terminate this Agreement in whole or in part, at any time without cause, and without liability except, in the case of the Recipient, for required payment for services rendered and reimbursement for authorized expenses incurred, by providing at least 90 (ninety) days' prior written notice to the other party (such date, the \"Services Termination Date\"). 3.3 Breach. Any Party (the \"Non-Breaching Party\") may terminate this Agreement with respect to any Service, in whole but not in part, at any time upon prior written notice to the other Party (the \"Breaching Party\"), if the Breaching Party has failed (other than pursuant to Section 3.6) to perform any of its material obligations under this Agreement relating to such Service, and such failure shall have continued without cure for a period of 30 days after receipt by the Breaching Party of a written notice of such failure from the Non-Breaching Party seeking to terminate such service. For the avoidance of doubt, non-payment by Recipient for a Service provided by Provider in accordance with this Agreement and not the subject of a good-faith dispute shall be deemed a breach for purposes of this Section 3.3. 3.4 Insolvency. In the event that either Party hereto shall (a) file a petition in bankruptcy, (b) become or be declared insolvent, or become the subject of any proceedings (not dismissed within sixty (60) days) related to its liquidation, insolvency or the appointment of a receiver, (c) make an assignment on behalf of all or substantially all of its creditors, or (d) take any corporate action for its winding up or dissolution, then the other party shall have the right to terminate this Agreement by providing written notice in accordance with Section 6.6. 3.5 Effect of Termination. Upon termination of this Agreement in its entirety pursuant to Section 3.1, all obligations of the Parties hereto shall terminate, except for the provisions of Section 2.2, and the entirety of Sections 4, 5 and 6, which shall survive any termination or expiration of this Agreement. 3.6 Upon expiration or termination of this Agreement for any reason, Provider shall promptly: (a) Deliver to Recipient all documents, work product, and other materials, whether or not complete, prepared by or on behalf of Provider in the course of performing the Services for which Recipient has paid. (b) Return to Recipient all Recipient -owned property, equipment, or materials in its possession or control.\n\n4\n\n\n\n\n\n(c) Remove any Provider-owned property, equipment, or materials located at Recipient's locations. (d) Deliver to Recipient, all documents and tangible materials (and any copies) containing, reflecting, incorporating, or based on Recipient's Confidential Information. (e) On a pro rata basis, repay all fees and expenses paid in advance for any Services which have not been provided. (f) Permanently erase all of Recipient's Confidential Information from its computer systems. (g) Certify in writing to Recipient that it has complied with the requirements of this Section 3.6 3.7 Force Majeure. If Provider is prevented from or delayed in complying, either totally or in part, with any of the terms or provisions of this Agreement by reason of fire, flood, storm, strike, lockout or other labor trouble or shortage, delays by unaffiliated suppliers or carriers, shortages of fuel, power, raw materials or components, any law, order, proclamation, regulation, ordinance, demand, seizure or requirement of any governmental authority, riot, civil commotion, war, rebellion, acts of terrorism, nuclear accident or other causes beyond the reasonable control of Provider, or acts, omissions, or delays in acting by any governmental or military authority or Recipient (each, a \"Force Majeure\"), then upon written notice to Recipient, the Services affected by the Force Majeure (the \"Affected Services\") and/or other requirements of this Agreement will be suspended during the period of such Force Majeure and Provider will have no liability to Recipient or any other party in connection with such Affected Services. If the Force Majeure in question prevails for a continuous period in excess of three months after the date on which the Force Majeure begins, Provider shall be entitled to give notice to Recipient to terminate the Affected Services. The notice to terminate must specify the termination date, which must be not less than ten (10) days after the date on which the notice to terminate is given. Once a notice to terminate has been validly given, the Affected Services will terminate on the termination date set out in the notice. Neither Party shall have any liability to the other in respect of termination of the Affected Services due to Force Majeure, but rights and liabilities which have accrued prior to termination shall subsist.\n\n5\n\n\n\n\n\n4. Confidentiality. 4.1 Confidentiality. During the term of this Agreement and thereafter, the Parties hereto shall, and shall instruct their respective representatives to, maintain in confidence and not disclose the other Party's financial, technical, sales, marketing, development, personnel, and other information, records, or data, including, without limitation, customer lists, supplier lists, trade secrets, designs, product formulations, product specifications or any other proprietary or confidential information, however recorded or preserved, whether written or oral (any such information, \"Confidential Information\"). Each Party hereto shall use the same degree of care, but no less than reasonable care, to protect the other Party's Confidential Information as it uses to protect its own Confidential Information of like nature. Unless otherwise authorized in any other agreement between the Parties, any Party receiving any Confidential Information of the other Party (the \"Receiving Party\") may use Confidential Information only for the purposes of fulfilling its obligations under this Agreement (the \"Permitted Purpose\"). Any Receiving Party may disclose such Confidential Information only to its representatives who have a need to know such information for the Permitted Purpose and who have been advised of the terms of this Section 4.1 and the Receiving Party shall be liable for any breach of these confidentiality provisions by such Persons; provided, however, that any Receiving Party may disclose such Confidential Information to the extent such Confidential Information is required to be disclosed by law, in which case the Receiving Party shall promptly notify, to the extent possible, the disclosing party (the \"Disclosing Party\"), and take reasonable steps to assist in contesting such disclosure requirement or in protecting the Disclosing Party's rights prior to disclosure, and in which case the Receiving Party shall only disclose such Confidential Information that it is advised by its counsel in writing that it is legally bound to disclose. Notwithstanding the foregoing, \"Confidential Information\" shall not include any information that the Receiving Party can demonstrate: (a) was publicly known at the time of disclosure to it, or has become publicly known through no act of the Receiving Party or its representatives in breach of this Section 4.1, (b) was rightfully received from a third party without a duty of confidentiality, or (c) was developed by it independently without any reliance on the Confidential Information. 4.2 Return of Confidential Information. Upon demand by the Disclosing Party at any time, or upon expiration or termination of this Agreement with respect to any Service, the Receiving Party agrees promptly to return or destroy, at the Disclosing Party's option, all Confidential Information received in connection with this Agreement. If such Confidential Information is destroyed, an authorized officer of the Receiving Party shall certify to such destruction in writing. 5. Indemnification. 5.1 Indemnification. Provider shall indemnify, defend, and hold harmless Recipient and its officers, directors, employees, agents, affiliates, successors, and permitted assigns (collectively, \"Indemnified Party\") against any and all losses, damages, liabilities, deficiencies, claims, actions, judgments, settlements, interest, awards, penalties, fines, costs, or expenses of whatever kind, including attorneys' fees, fees and the costs of enforcing any right to indemnification under this Agreement, and the cost of pursuing any insurance providers, incurred by Indemnified Party or awarded against Indemnified Party (collectively, \"Losses\"), relating to/arising out of or resulting from any claim of a third party or Recipient arising out of or occurring in connection with Provider's negligence, willful misconduct, or breach of this Agreement. Provider shall not enter into any settlement without Recipient's or Indemnified Party's prior written consent. 6. Miscellaneous. 6.1 Entire Agreement. This Agreement, the Purchase Agreement and the documents referred to herein and therein constitute the entire agreement among the Parties and supersedes any prior understandings, agreements, or representations by or among the Parties, written or oral, to the extent they relate in any way to the subject matter hereof.\n\n6\n\n\n\n\n\n6.2 Succession and Assignment. This Agreement shall be binding upon and inure to the benefit of the Parties named herein and their respective successors and permitted assigns. Provider may not assign, delegate or otherwise transfer either this Agreement or any of its rights, interests, or obligations hereunder without the prior written approval of Recipient. 6.3 Counterparts. This Agreement may be executed in one or more counterparts, each of which shall be deemed an original but all of which together will constitute one and the same instrument. Counterparts may be delivered via facsimile and electronic mail (including portable document format (PDF) or any electronic signature complying with the U.S. federal ESIGN Act of 2000, e.g., www.docusign.com). 6.4 Titles and Headings. Titles and section headings contained in this Agreement are inserted for convenience only and shall not affect in any way the meaning or interpretation of this Agreement. 7. Notices. All notices, requests, consents, claims, demands, waivers and other communications hereunder shall be in writing and shall be deemed to have been given (a) when delivered by hand (with written confirmation of receipt); (b) when received by the addressee if sent by a nationally recognized overnight courier (receipt requested); (c) on the date sent by facsimile or e-mail of a PDF document (with confirmation of transmission) if sent during normal business hours of the recipient, and on the next Business Day if sent after normal business hours of the recipient or (d) on the third day after the date mailed, by certified or registered mail, return receipt requested, postage prepaid. Such communications must be sent to the respective parties at the following addresses (or at such other address for a party as shall be specified in a notice given in accordance with this Section 7: If to Provider: [ * * * ] With a copy to: N/A If to Recipient: TELCOSTAR PTE. LTD 6 Eu Tong Sen Street Tel Aviv, Israel, 6770007 #10-15 The Central Singapore 059817 Email: avi@ability.co.il Attention: Avi Levin With a copy to: McDermott Will & Emery LLP 340 Madison Avenue New York, NY 10173-1922 Telephone: (212) 547-5541 Facsimile: (212) 547-5444 EMAIL: GEMMANUEL@MWE.COM Attention: Gary Emmanuel\n\n7\n\n\n\n\n\nAny Party may change the address to which notices, requests, demands, claims, and other communications hereunder are to be delivered by giving the other Parties notice in the manner herein set forth. 7.1 Further Assurances. The Parties agree (a) to furnish upon request to each other such further information, (b) to execute and deliver to each other such other documents, and (c) to do such other acts and things, all as the other party may reasonably request for the purpose of carrying out the intent of this Agreement and the documents referred to in this Agreement. 7.2 Governing Law. This Agreement and any claim, controversy or dispute arising out of or related to this Agreement, any of the transactions contemplated hereby and/or the interpretation and enforcement of the rights and duties of the Parties, whether arising in contract, tort, equity or otherwise, shall be governed by and construed in accordance with the domestic laws of the State of Israel (including in respect of the statute of limitations or other limitations period applicable to any such claim, controversy or dispute), without giving effect to any choice or conflict of law provision or rule (whether of the State of Israel or any other jurisdiction) that would cause the application of the laws of any jurisdiction other than the State of Israel. 7.3 Consent to Jurisdiction. The Parties hereby irrevocably submit any disputes under this Agreement to the exclusive jurisdiction of the courts located in Tel-Aviv, Israel, provided however, that Recipient shall be entitled to seek an injunction or other appropriate remedy against Provider in the country in which Provider has acted in breach of the terms hereof. 7.4 Specific Performance. The Parties hereby agree that, in the event of breach of this Agreement, damages would be difficult, if not impossible, to ascertain and that irreparable damage would occur in the event that any of the provisions of this Agreement were not performed in accordance with their specific terms or were otherwise breached. Accordingly, it is hereby agreed that the Parties shall be entitled to seek an injunction or other equitable relief in any court of competent jurisdiction to enjoin any such breach and enforce specifically the terms and provisions hereof, this being in addition to any other remedy or right to which they are entitled at law or in equity, without any necessity of proving damages or any requirement for the posting of a bond or other security. 7.5 Amendments and Waivers. No amendment of any provision of this Agreement shall be valid unless the same shall be in writing and signed by Recipient and the Provider. No waiver by any Party of any provision of this Agreement or any default, misrepresentation, or breach of warranty or covenant hereunder, whether intentional or not, shall be valid unless the same shall be in writing and signed by the Party making such waiver nor shall such waiver be deemed to extend to any prior or subsequent default, misrepresentation, or breach of warranty or covenant hereunder or affect in any way any rights arising by virtue of any prior or subsequent such occurrence.\n\n8\n\n\n\n\n\n7.6 Severability. Any term or provision of this Agreement that is held invalid or unenforceable by a court of competent jurisdiction or other competent governmental authority in any situation in any jurisdiction shall not affect the validity or enforceability of the remaining terms and provisions hereof or the validity or enforceability of the offending term or provision in any other situation or in any other jurisdiction. Upon such a determination, the Parties shall negotiate in good faith to replace invalid or unenforceable provisions with valid provisions, the economic effect of which comes as close as possible to that of the invalid or unenforceable provisions. 7.7 Construction. The Parties have participated jointly in the negotiation and drafting of this Agreement. In the event an ambiguity or question of intent or interpretation arises, this Agreement shall be construed as if drafted jointly by the Parties and no presumption or burden of proof shall arise favoring or disfavoring any Party by virtue of the authorship of any of the provisions of this Agreement. Any reference to any law shall be deemed also to refer to all rules and regulations promulgated thereunder, unless the context requires otherwise. The word \"including\" shall mean including without limitation. 7.8 Incorporation of Exhibits and Disclosure Schedule. The Exhibit identified in this Agreement is incorporated herein by reference and made a part hereof. 7.9 Amendment and Restatement. This Agreement amends and restates in full the Production Contract. [SIGNATURE PAGE FOLLOWS]\n\n9\n\n\n\n\n\nIN WITNESS WHEREOF, the Parties have executed this Services Agreement as of the date first written above. PROVIDER: RECIPIENT: [ * * * ] [ * * * ] By: By: Name: Name: Title: Title:\n\n10\n\n\n\n\n\nEXHIBIT A Services [ * * * ] 11"} diff --git a/integration-tests/benchmark-data/cuad-prototype/documents/2ThemartComInc_19990826_10-12G_EX-10.10_6700288_EX-10.10_Co-Branding_Agreement__Agency_Agreement.txt b/integration-tests/benchmark-data/cuad-prototype/documents/2ThemartComInc_19990826_10-12G_EX-10.10_6700288_EX-10.10_Co-Branding_Agreement__Agency_Agreement.txt new file mode 100644 index 00000000..b8c8c773 --- /dev/null +++ b/integration-tests/benchmark-data/cuad-prototype/documents/2ThemartComInc_19990826_10-12G_EX-10.10_6700288_EX-10.10_Co-Branding_Agreement__Agency_Agreement.txt @@ -0,0 +1,227 @@ +CO-BRANDING AND ADVERTISING AGREEMENT + +THIS CO-BRANDING AND ADVERTISING AGREEMENT (the "Agreement") is made as of June 21, 1999 (the "Effective Date") by and between I-ESCROW, INC., with its principal place of business at 1730 S. Amphlett Blvd., Suite 233, San Mateo, California 94402 ("i-Escrow"), and 2THEMART.COM, INC. having its principal place of business at 18301 Von Karman Avenue, 7th Floor, Irvine, California 92612 ("2TheMart"). + +1. DEFINITIONS. + +(a) "CONTENT" means all content or information, in any medium, provided by a party to the other party for use in conjunction with the performance of its obligations hereunder, including without limitation any text, music, sound, photographs, video, graphics, data or software. Content provided by 2TheMart is referred to herein as "2TheMart Content" and Content provided by i-Escrow is referred to herein as "i-Escrow Content." + +(b) "CO-BRANDED SITE" means the web-site accessible through Domain Name, for the Services implemented by i-Escrow. The homepage of this web-site will visibly display both 2TheMart Marks and i-Escrow Marks. + +(c) "CUSTOMERS" means all users who access Co-Branded Site. + +(d) "DOMAIN NAME" means www.iescrow.com/2TheMart. + +(e) "ESCROW SERVICES" means services for auction sellers and high bidders whereby an agent holds a buyer's money in trust until the buyer approves the applicable item that was physically delivered, at which time the agent releases the buyer's money to seller, after subtracting the escrow fees. + +(f) "INFORMATION TRANSFER MECHANISM" means the mechanism by which 2TheMart transfers to i-Escrow information to populate the applicable i-Escrow transaction and user registration forms. + +(g) "LAUNCH DATE" means the first date on which the Co-Branded Site is pointed to in all references to i-Escrow from 2TheMart auction site, and the Information Transfer Mechanism is publicly deployed (post-beta). + +(h) "MARKS" means all domain names, trademarks and logos designated by a party for the other party's use in conjunction with such other party's performance under this Agreement. Marks designated by 2TheMart for i-Escrow's use are referred to herein as "2TheMart Marks" and Marks designated by i-Escrow for 2TheMart' use are referred to herein as "i-Escrow Marks." + +(i) "SERVICES" means i-Escrow's implementation and performance of the Escrow Services as of the Effective Date, as modified over time. + +(j) "SHADOW SITE" means the site where Co-Branded Site is made available for 2TheMart's testing of the Information Transfer Mechanism prior to being made publicly available. + +(k) "TRANSACTION" means a transaction utilizing the Services that actually closes and that was initiated by a Transaction Inquiry from a Customer. + +(l) "TRANSACTION INQUIRY" means a Customer's submission of i-Escrow's standard New Transaction Inquiry form (or its successor) on or through the Co-Branded Pages. Currently this means entry of a description and price of merchandise by a user (buyer or seller) who agrees to abide by the terms and conditions of the Services, together with email address of the other party, regardless of whether or not any Transaction is completed. + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +2. DEVELOPMENT AND IMPLEMENTATION. + +2.1 OVERVIEW. As set forth herein, 2TheMart will promote Services to its auction users (buyers and sellers), and i-Escrow shall develop Co-Branded Site, and develop the Information Transfer Mechanism working with 2TheMart to make Services available seamlessly to Customers. Unless otherwise specified, each party shall be responsible for all development, hosting and other costs associated with the pages resident on their servers and all emails to users they send. + +2.2 INITIAL INFORMATION TRANSFER MECHANISM DEVELOPMENT. The parties shall negotiate in good faith to determine the initial operation of the Information Transfer Mechanism and to describe such operation and development fees, in a statement of work ("SOW"). Each party shall make available sufficient and qualified engineers to negotiate the SOW. No SOW shall be binding on the parties unless mutually approved by both parties. In the event that the parties are unable to agree to an SOW within 2 months following the Effective Date, either party may, in its sole discretion, terminate this Agreement by providing written notice. + +Once approved, the parties shall use commercially reasonable efforts to diligently implement their respective obligations under the SOW. Upon completion of its duties under the SOW, a party shall notify the other party and provide the other party with the opportunity to test and evaluate its work. i-Escrow shall make available the Shadow Site for such testing in a timely manner. Each party shall reasonably cooperate with the other party in effectuating their respective duties under the SOW. The Information Transfer Mechanism shall not go live until its operation has been approved ("Approval Date") by both parties, such approval not to be unreasonably withheld. + +2.3 LAUNCH TIMING. Each party shall use good faith and reasonable efforts to expeditiously develop the Co-Branded Pages and the Information Transfer Mechanism. In the event that, after using such efforts, the Launch Date has not occurred within 4 months following the Effective Date, either party may terminate this Agreement by providing written notice. If + +only one party has used good faith and reasonable development efforts, only that party may exercise the foregoing right to terminate. + +2.4 RESTRICTIONS ON COMMUNICATIONS. i-Escrow may place banner advertising on the Co-Branded Site upon prior written approval of 2TheMart, which shall be at the discretion of 2TheMart. All advertising revenue arising from the banner ads shall be solely i-Escrow's. i-Escrow shall not run banner advertisements on the Co-Branded Site for any of 2TheMart's competitors. 2TheMart shall provide in writing, a list of companies they would like to exclude, including every time they wish to change this list. + +2.5 SERVICE PERFORMANCE OF INFORMATION TRANSFER MECHANISM. The parties each shall in good faith work to provide reasonable service levels with respect to the operation of the portions of the Information Transfer Mechanism in their control. + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +2.6 PROGRAM REVIEW MEETINGS. The parties shall meet, at least once per month either in person, or by telephone, to coordinate the implementation of this agreement over time. + +3. PROMOTION. + +After Launch Date, 2TheMart will widely promote the Services: + +(a) To every seller and high bidder through means including, but not limited to, end of auction emails containing links, such that, it shall be possible for the buyer or seller to initiate a Transaction Inquiry with i-Escrow, without having to re-enter all their personal or transaction related information. + +(b) By adding links to Co-Branded Site in FAQ section of 2TheMart auctions. + +(c) By adding links to Co-Branded Site on the seller listing pages of 2TheMart auctions. + +(d) By displaying a text or graphic link to a page containing information about Services on all auction item pages and bidding pages to educate bidders about i-Escrow. 2TheMart may use the "Escrow Services Description" attached in Exhibit A for creating such a page. + +5. PAYMENT. + +5.1 ADVERTISING FEES. After the Launch Date, i-Escrow shall pay 2TheMart advertising fees based on the number of Transaction Inquiries. This advertising fees shall consist of a per Transaction Inquiry amount calculated by multiplying 0.025% by the amount of the average Transaction from all Customers in the preceding quarter. The formula for arriving at the per Transaction Inquiry amount may be revised from time to time during the term of this Agreement to reflect present market conditions ("the Adjusted Rate"), but only by mutual + +consent of the parties after good faith discussions. The Adjusted Rate shall be added as an addendum to this Agreement. + +5.2 REPORTING. Within two (2) weeks following the end of each calendar quarter, i-Escrow shall provide to 2TheMart a report, describing for each quarter: the number of new registrations through the Co-Branded Pages; the number of Transaction Inquiries from Customers; the total number of Transactions from such inquiries; the total dollar value of the Transactions. + +5.3 AUDIT RIGHTS. i-Escrow shall keep for one (1) year proper records and books of account relating to the computation of advertising payments owed to 2TheMart (including, as appropriate, the computation of the size of average Transaction). Once every twelve (12) months, 2TheMart through a CPA may inspect and audit such records to verify reports. Any such inspection will be conducted in a manner that does not unreasonably interfere with i-Escrow's business activities and with no less than fifteen (15) days notice. i-Escrow shall within two (2) weeks make any overdue payments disclosed by the audit. Such inspection shall be at 2TheMart's expense; however, if the audit reveals overdue payments in excess of ten percent (10%) of the payments owed to date, i-Escrow shall immediately pay all cost of such audit. + +6. RIGHTS AND STANDARDS. + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +6.1 CONTENT. 2TheMart hereby grants to i-Escrow a worldwide, non-exclusive right to use, reproduce, distribute, publicly perform, publicly display and digitally perform the 2TheMart Content soley with respect to and in conjunction with the Co-Branded Site all with the prior written consent of 2TheMart, for the term of this Agreement. i-Escrow hereby grants to 2TheMart a worldwide, non-exclusive right to use, reproduce, distribute, publicly perform, publicly display and digitally perform the i-Escrow Content on or in conjunction with 2TheMart auctions. + +6.2 CONTENT OWNERSHIP. Except as otherwise provided in this Agreement, as between 2TheMart and i-Escrow: (a) 2TheMart and its suppliers retain all rights, title and interest in and to all intellectual property rights embodied in or associated with the 2TheMart Content, and b) i-Escrow and its suppliers retain all rights, title and interest in and to all intellectual property rights embodied in or associated with the i-Escrow Content and Co-Branded Site. There are no implied licenses under this Agreement, and any rights not expressly granted are reserved. Neither party shall exceed the scope of the rights granted hereunder. + +6.3 TRADEMARKS. Subject to the terms and conditions of this Agreement: (a) i-Escrow hereby grants to 2TheMart a non-exclusive, nontransferable right to use the i-Escrow Marks (including without limitation the Domain Name) in links to and advertisements and promotions for the Co-Branded Pages or the Services; and (b) 2TheMart hereby grants to i-Escrow a non-exclusive, nontransferable right to use 2TheMart Marks (including without limitation the Domain Name) on the Co-Branded Pages, and for the performance of Services. + +6.4 TRADEMARK RESTRICTIONS. The Mark owner may terminate the foregoing rights if, in its reasonable discretion, the other party's use of the Marks tarnishes, blurs or dilutes the quality associated with the Marks or the associated goodwill and such problem is not cured within ten (10) days of notice of breach; alternatively, instead of terminating the right in total, the + +owner may specify that certain pages of the other party's web-site may not contain the Marks. Title to and ownership of the owner's Marks shall remain with the owner. The receiving party shall use the Marks exactly in the form provided and in conformance with any trademark usage policies. The other party shall not take any action inconsistent with the owner's ownership of the Marks, and any benefits accruing from use of such Marks shall automatically vest in the owner. The other party shall not form any combination marks with the other party's Marks. Notwithstanding the foregoing, to the extent that the Domain Name is deemed a combination mark, neither party shall use the Domain Name for any purpose except as expressly provided herein or attempt to register the Domain Name, and the parties will jointly cooperate on any enforcement action of infringement of the Domain Name. + +6.5 LIMITS ON SUBLICENSING. All rights (under any applicable intellectual property right) granted herein are not sublicenseable, + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +transferable or assignable. Notwithstanding the foregoing, either party may use a third party web host, but all actions or failures to act of the web host that would be a breach of this Agreement, were the actions or failures to act taken by the applicable party, shall be deemed a breach of this Agreement. In addition, 2TheMart may grant sublicenses to companies that 2TheMart has a business relationship with to the extent that 2TheMart Content is visible from such company's web-site through a link or other means. + +6.6 CONTENT STANDARDS. 2TheMart shall not provide any 2TheMart Content, and i-Escrow shall not provide any i-Escrow Content, that: (a) infringes any third party's copyright, patent, trademark, trade secret or other proprietary rights or rights of publicity or privacy; (b) violates any law, statute, ordinance or regulation (including without limitation the laws and regulations governing export control, unfair competition, antidiscrimination or false advertising); (c) is defamatory, trade libelous, unlawfully threatening or unlawfully harassing; (d) is obscene, harmful to minors or child pornographic; (e) contains any viruses, Trojan horses, worms, time bombs, cancelbots or other computer programming routines that are intended to damage, detrimentally interfere with, surreptitiously intercept or expropriate any system, data or personal information; and (f) is materially false, misleading or inaccurate. + +6.7 SERVICE STANDARDS. i-Escrow will comply with all laws and regulations and act as an Independent Escrow Agent as per the guidelines of California Escrow Law (California Financial Code Section17000 et seq., or its successor). Should any of the terms, conditions or provisions of this Agreement conflict with the California Escrow Law, its rules or regulations, which govern i-Escrow's business practices, the California Escrow Law shall prevail. Notwithstanding the foregoing, at any time that i-Escrow reasonably believes such a conflict exists, i-Escrow will give 2TheMart written notice of such conflict and the parties will use their best efforts to resolve such conflict. + +7. DISCLAIMER OF WARRANTIES. EACH PARTY PROVIDES ALL MATERIALS AND SERVICES TO THE OTHER PARTY "AS IS." EACH PARTY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED OR STATUTORY, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF TITLE, NON- + +INFRINGEMENT, MERCHANTABILITY ANDFITNESS FOR A PARTICULAR PURPOSE. Each party acknowledges that it has not entered into this Agreement in reliance upon any warranty or representation except those specifically set forth herein. + +8. TERM AND TERMINATION. + +8.1 TERM. The term of this Agreement shall continue for one (1) year following the Launch Date, unless earlier terminated as provided herein. This Agreement may be renewed for any number of successive one (1) year terms by mutual written agreement of the parties prior to the conclusion of the term of this Agreement. A party wishing to renew this Agreement shall give the other party notice thereof no less than thirty (30) days before the expiration of the term then in effect. In the event that either party does not give such notice, the term of this Agreement shall be automatically renewed for another one (1) year. + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +8.2 TERMINATION FOR BREACH. In addition to other remedies that may be available to it, by providing written notice, a party may immediately terminate this Agreement: (a) if the other party materially breaches this Agreement and fails to cure that breach within sixty (60) days after receiving written notice of the breach, or (b) as provided in Sections 2.2 [INITIAL INFORMATION TRANSFER MECHANISM DEVELOPMENT], 2.4 [RESTRICTIONS ON COMMUNICATIONS], or 12.4. + +8.3 TERMINATION FOR CHANGE IN COMPANY STRUCTURE. If a majority of the equity securities of either 2TheMart or i-Escrow, Inc. (except that i-Escrow may sell all or a majority of its equity securities or voting interests to i-Escrow.com, and i-Escrow.com may sell all or a majority of its equity securities or voting interests to i-Escrow's existing shareholders, without triggering the foregoing) are acquired by another company during the term of this Agreement either company may terminate this Agreement, without liability, by giving a thirty (30) days written notice to the other party. + +8.4 TERMINATION FOR BANKRUPTCY. Either party may terminate or suspend this Agreement effective immediately and without liability upon written notice to the other party if any one of the following events occurs: + +(a) the other party files a voluntary petition in bankruptcy or otherwise seeks protection under any law for the protection of debtors; + +(b) a proceeding is instituted against the other party under any provision of any bankruptcy laws which is not dismissed within ninety (90) days; + +(c) the other party is adjudged bankrupt; + +(d) a court assumes jurisdiction of all or a substantial portion of the assets of the other party under a reorganization law; + +(e) a trustee or receiver is appointed by a court for all or a substantial portion of the assets of the other party; + +(f) the other party becomes insolvent, ceases or suspends all or substantially all of its business; or + +(g) the other party makes an assignment of the majority of its assets for the benefit of its creditors. + +8.5 EFFECTS OF TERMINATION. Upon expiration or termination of this Agreement for any reason: (a) all rights granted herein shall terminate, (b) i-Escrow shall pay all amounts owed to 2TheMart within six (6) weeks of termination, and (c) each party shall remove the other party's content and Marks from their servers. Notwithstanding the foregoing, unless this Agreement was terminated for a material breach, all provisions of this Agreement shall survive to the extent necessary for i-Escrow to complete any Customer transactions which are pending at the time of expiration or termination. Sections 1, 7, 8.5 [EFFECTS OF TERMINATION], 9, 10, 11 and 12 shall survive expiration or termination of this Agreement. + +9. INDEMNITY. Each party (the "Indemnifying Party") shall indemnify the other party (the "Indemnified Party") against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which the Indemnified Party may incur as a result of claims in any form by third parties arising from the Indemnifying Party's acts, omissions or misrepresentations to the extent that the Indemnified Party is deemed a principal of the Indemnifying Party. In addition, 2TheMart shall indemnify i-Escrow against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which i-Escrow may incur as a result of claims in any form by third parties arising from 2TheMart Content. In addition, i-Escrow shall indemnify 2TheMart against any and all claims, losses, costs and expenses, including reasonable attorneys' fees, which 2TheMart may incur as a result of claims in any form by third parties arising from i-Escrow + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +Content and or the Services provided to Customers. The foregoing obligations are conditioned on the Indemnified Party: (i) giving the Indemnifying Party notice of the relevant claim, (ii) cooperating with the Indemnifying Party, at the Indemnifying Party's expense, in the defense of such claim, and (iii) giving the Indemnifying Party the right to control the defense and settlement of any such claim, except that the Indemnifying Party shall not enter into any settlement that affects the Indemnified Party's rights or interest without the Indemnified Party's prior written approval. The Indemnified Party shall have the right to participate in the defense at its expense. + +10. LIMITATION ON LIABILITY. EXCEPT IN THE EVENT OF A BREACH OF SECTION 11, NEITHER PARTY SHALL BE LIABLE FOR SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OR LOST PROFITS (HOWEVER ARISING, INCLUDING NEGLIGENCE) ARISING OUT OF OR IN CONNECTION WITH THIS AGREEMENT, EVEN IF THE PARTIES ARE AWARE OF THE POSSIBILITY OF SUCH DAMAGES. + +11. CONFIDENTIAL INFORMATION. A party's "Confidential Information" is defined as any confidential or proprietary information of a party which is disclosed to the other party in a writing marked confidential or, if disclosed orally, is identified as confidential at the time of disclosure and is subsequently reduced to a writing marked confidential and delivered to the + +other party within ten (10) days of disclosure. Each party shall hold the other party's Confidential Information in confidence and shall not disclose such Confidential Information to third parties nor use the other party's Confidential Information for any purpose other than as required to perform under this Agreement. Such restrictions shall not apply to Confidential Information which (a) is already known by the recipient, (b) becomes, through no act or fault of the recipient, publicly known, (c) is received by recipient from a third party without a restriction on disclosure or use, or (d) is independently developed by recipient without reference to the Confidential Information. The restriction on disclosure shall not apply to Confidential Information which is required to be disclosed by a court or government agency. Upon expiration or termination of this Agreement, within fourteen (14) days of the other party's request, each party will return all Confidential Information and other deliverables to the requesting party. + +12. GENERAL PROVISIONS. + +12.1 GOVERNING LAW. This Agreement will be governed and construed in accordance with the laws of the State of California without giving effect to conflict of laws principles. Both parties submit to personal jurisdiction in California and further agree that any cause of action arising under this Agreement shall be brought in a court in Orange County, California. + +12.2 SEVERABILITY; HEADINGS. If any provision herein is held to be invalid or unenforceable for any reason, the remaining provisions will continue in full force without being impaired or invalidated in any way. The parties agree to replace any invalid provision with a valid provision that most closely approximates the intent and economic effect of the invalid provision. Headings are for reference purposes only and in no way define, limit, construe or describe the scope or extent of such section. + +12.3 PUBLICITY. Prior to the release of any press releases or other similar promotional materials related to this Agreement, the releasing party shall submit a written request for approval to the other party with a copy of the materials to be released, which + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +request shall be made no less than three (3) business days prior to the requested release date. A party shall not unreasonably withhold or delay the granting of its approval of such materials, and such approval shall be provided to the other party within one (1) business day of receipt + +12.4 FORCE MAJEURE. Except as otherwise provided, if performance hereunder (other than payment) is prevented, restricted or interfered with by any act or condition whatsoever beyond the reasonable control of a party (a "force majeure event"), the party so affected, upon giving prompt notice to the other party, shall be excused from such performance to the extent of such prevention, restriction or interference. However, if a force majeure event interferes with the operation of this Agreement for sixty (60) days or more, either party can terminate this Agreement, without penalty. Notwithstanding the foregoing, the occurrence of any force majeure event shall not limit either party's obligations under Section 9 with respect to any third party claim as to which the other party seeks indemnification. + +12.5 INDEPENDENT CONTRACTORS. The parties are independent contractors, and no agency, partnership, joint venture, employee- employer or franchisor-franchisee relationship is + +intended or created by this Agreement. Neither party shall make any warranties or representations on behalf of the other party. + +12.6 NOTICE. Any notices hereunder shall be given to the appropriate party at the address specified below or at such other address as the party shall specify in writing. Notice shall be deemed given: upon personal delivery; if sent by fax, upon confirmation of receipt; or if sent by a reputable overnight courier with tracking capabilities, one (1) day after the date of mailing: To i-Escrow: i-Escrow, Inc. 1730 South Amphlett Blvd., #215 San Mateo, CA 94402 Fax no. (650) 638-7890 Attention: President + +With copy to: Fred M. Greguras, Esq. Legal Counsel of i-Escrow Fenwick & West LLP Two Palo Alto Square Palo Alto, CA 94306 + +To 2TheMart: Dominic J. Magliarditi President 18301 Von Karman Avenue, 7th Floor Irvine, CA 92612 Fax no. (949) 477-1221 + +11.7 COUNTERPARTS. This Agreement may be executed in one or more counterparts, each of which shall be deemed an original and all of which shall be taken together and deemed to be one instrument. + +12.8 GOOD FAITH. The parties agree to act in good faith with respect to each provision of this Agreement and any dispute that may arise related hereto. + +12.9 ADDITIONAL DOCUMENTS/INFORMATION. The parties agree to sign and/or provide such additional documents and/or information as may reasonably be required to carry out the intent of this Agreement and to effectuate its purposes. + +12.10 RIGHTS AND REMEDIES CUMULATIVE. The rights and remedies provided herein will be cumulative and not exclusive of any other rights or remedies provided by law or otherwise. + +12.11 NONWAIVER. No failure or forbearance by either party to exercise any right or insist upon or enforce performance of any obligation hereunder shall be deemed a waiver or relinquishment to any extent of that or any other right or obligation, in that or any other instance; rather, the + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 + + + + + +same shall be and shall remain in full force and effect. Any waiver of any right of a party or any obligation of the other party hereunder must be made in a writing signed by the arty waiving such right or obligation. + +12.12 ENTIRE AGREEMENT. This Agreement contains the entire understanding of the parties hereto with respect to the transactions and matters contemplated hereby, supersedes all previous Agreements between i-Escrow and 2TheMart concerning the subject matter (except for the Confidential Agreement Dated January 4 1999, which shall survive this Agreement). No amendments or supplements to this Agreement will be effective for any purpose except by a written Agreement signed by the parties. No party hereto has relied on any statement, representation or promise of any party or with any other officer, agent, employee or attorney for the other party in executing this Agreement except as expressly stated herein. + +2THEMART.COM, INC.: I-ESCROW, INC.: + +By:/s/Dominic J. Magliarditi By:/s/Sanjay Bajaj Name: Dominic J. Magliarditi Name: Sanjay Bajaj Title: President Title: VP Business Development Date: 6/21/99 Date: 6/11/99 + + EXHIBIT A + +ESCROW SERVICES DESCRIPTION + +Successful completion of a transaction involves exchange of merchandise with payment. The buyer has to be satisfied he/she received what they thought they were getting and the seller has to be sure he/she gets paid. i-Escrow holds payment from the buyer in trust until the seller sends the merchandise to the buyer. Once the buyer accepts the merchandise, i-Escrow forwards the payment to the seller by writing a check. A typical escrow transaction: When an auction ends, your end of auction email contains links to i-Escrow. Once you have signed up with i-Escrow you go through the following steps to complete your transaction. 1. Start a transaction by entering the description and price of the merchandise along with email address of the other party. 2. The other party receives an email from i-Escrow requesting an acknowledgement of the terms of the transaction. 3. Once the transaction is acknowledged by the other party, the buyer pays i-Escrow the agreed upon price, by credit card or other means. 4. i-Escrow informs the seller that payment has been received, requesting them to ship the merchandise directly to the buyer. 5. The seller provides i-Escrow with the tracking number of the shipment. 6. The buyer receives and accepts the merchandise. 7. i-Escrow sends the check to the seller. + +For more information about I-Escrow, visit their web-site at www.iescrow.com + +Source: 2THEMART COM INC, 10-12G, 8/26/1999 \ No newline at end of file diff --git a/integration-tests/benchmark-data/cuad-prototype/documents/ABILITYINC_06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt b/integration-tests/benchmark-data/cuad-prototype/documents/ABILITYINC_06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt new file mode 100644 index 00000000..6078257e --- /dev/null +++ b/integration-tests/benchmark-data/cuad-prototype/documents/ABILITYINC_06_15_2020-EX-4.25-SERVICES_AGREEMENT.txt @@ -0,0 +1,79 @@ +EXHIBIT 4.25 INFORMATION IN THIS EXHIBIT IDENTIFIED BY [ * * * ] IS CONFIDENTIAL AND HAS BEEN EXCLUDED BECAUSE IT IS BOTH (I) NOT MATERIAL AND (II) WOULD LIKELY CAUSE COMPETITIVE HARM TO THE REGISTRANT IF PUBLICLY DISCLOSED. SERVICES AGREEMENT This Services Agreement (this "Agreement") is entered into on October 1, 2019 and is made effective as of November 1, 2019 (the "Effective Date"), by and between [ * * * ] (the "Provider"), and TELCOSTAR PTE, LTD., a company organized and existing under the laws of Singapore and Ability Computer & Software Industries Ltd, a company organized and existing under the laws of the State of Israel (each and both of them "Recipient"). Each of the foregoing parties is referred to herein as a "Party" and together as the "Parties". RECITALS A. Recipient wishes to engage the Provider to provide certain services and resources (the "Services") and Provider desires to provide Recipient with the Services all in accordance with the terms and conditions set forth herein. AGREEMENT The Parties hereby agree as follows: 1. Services. 1.1 Provision of Services. (a) Provider agrees to provide the Services set forth on the Exhibit A attached hereto (as such Exhibit may be amended or supplemented pursuant to the terms of this Agreement, the "Exhibit") to Recipient for the respective periods and on the other terms and conditions set forth in this Agreement and in the Exhibit. Notwithstanding the contents of the Exhibit, Provider agrees to respond in good faith to any reasonable request by Recipient for access to any additional services and resources that are necessary for the operation of the Recipient and which are not currently contemplated in the Exhibit, at a price to be agreed upon after good faith negotiations between the Parties. Any such additional services and resources so provided by Provider shall constitute Services under this Agreement and be subject in all respect to the provisions of this Agreement as if fully set forth on the Exhibit as of the date hereof. (b) Recipient may freely assign its rights under this Agreement to receive the Services to any of its affiliates. 1.2 Standard of Service. (a) Provider represents, warrants and agrees that the Services shall be provided in good faith, in accordance with applicable law and in a manner generally consistent with the historical provision of the Services and with the same standard of care as historically provided. (b) Provider shall maintain complete and accurate records relating to the provision of the Services under this Agreement, in such form as Recipient shall approve. + + + + + +(c) Provider shall use its best efforts to provide for employees or contractors to perform the Services, each of whose names, positions, and respective levels of experience and relevant licenses shall be set out in Exhibit A attached hereto (collectively, the "Provider Representatives"). Provider may not make any change in the Provider Representatives without the prior consent of the Recipient. Provider Representatives shall be dedicated to solely providing the Services to Recipient and shall not provide any such services or resources to Provider or any other customer of Provider. (d) Recipient acknowledges that this Agreement does not create a fiduciary relationship, partnership, joint venture or relationships of trust or agency between the Parties and that all Services are provided by Provider as an independent contractor. (e) Notwithstanding anything to the contrary in this Section 1.2: (a) in the event that Provider uses any subcontractors to perform any Services, Provider is not released from responsibility for its obligations under this Agreement; (b) Provider shall remain fully responsible, financially and otherwise, for the Services provided by each subcontractor to the same extent as if Provider had performed the Services itself (subject to the limitations set forth in this Agreement) and agrees to pay the fees and expenses of any such subcontractor; (c) Provider shall remain ultimately responsible for ensuring that the Services are provided and any such subcontractor performs any such obligations in accordance with the terms of this Agreement, and (d) the obligations with respect to the nature, quality and standards of care set forth in Section 1.2 are satisfied with respect to any Service provided by any subcontractor. (f) Provider shall at all times during the term of this Agreement maintain, or cause to be maintained, the computer software and computer hardware that is used in connection with the Services with substantially the same degree of care, skill and diligence with which Provider maintains, or causes to be maintained, as of the Effective Date, such computer software and computer hardware for itself, consistent with past practices, as of the Effective Date, including without limitation, with respect to type, quality and timeliness of such maintenance. 1.3 Additional Services. Nothing in this Agreement shall be construed to prevent the Recipient from itself performing or from acquiring services from other providers that are similar to or identical to the Services. 1.4 Intellectual Property. (a) Recipient shall own, and Provider hereby irrevocably assigns to the Recipient, all rights, title, and interest in any invention, technique, process, device, discovery, improvement, or know-how, whether patentable or not and all other proprietary rights, industrial rights and any other similar rights, in each case on a worldwide basis, and all copies and tangible embodiments thereof, or any part thereof, in whatever form or medium hereafter made or conceived solely or jointly by Provider while working for or on behalf of the Recipient, which relate to, is suggested by, or results from the Services. (b) At Recipient's request, Provider shall disclose any such invention, technique, process, device, discovery, improvement, or know-how promptly to Recipient. Provider shall, upon request of Recipient, promptly execute a specific assignment of title to Recipient, and do anything else reasonably necessary to enable Recipient to secure for itself, patent, trade secret, or any other proprietary rights. + +2 + + + + + +(c) All writings or works of authorship, including, without limitation, program codes or documentation, produced or authored by Provider in the course of performing services for the Recipient, together with any associated copyrights, are works made for hire and the exclusive property of the Recipient. To the extent that any writings or works of authorship may not, by operation of law, be works made for hire, this Agreement shall constitute an irrevocable assignment by Provider to the Recipient of the ownership of and all rights of copyright in, such items, and the Recipient shall have the right to obtain and hold in its own name, rights of copyright, copyright registrations, and similar protections which may be available in the works. Provider shall give the Recipient or its designees all assistance reasonably required to perfect such rights. 2. Compensation. 2.1 Responsibility for Wages and Fees. For such time as any employees of Provider are providing the Services to Recipient under this Agreement, (a) such employees will remain employees of Provider and shall not be deemed to be employees of Recipient for any purpose, and (b) Provider shall be solely responsible for the payment and provision of all wages, bonuses and commissions, employee benefits, including severance and worker's compensation, and the withholding and payment of applicable taxes relating to such employment. 2.2 Terms of Payment and Related Matters. (a) As consideration for provision of the Services following the Effective Date, Recipient shall pay Provider an amount equal to Provider's actual cost of providing the Services plus a 10% service fee. In addition to such amount, in the event that Provider incurs reasonable and documented out-of-pocket expenses in the provision of any Service, including, without limitation, license fees and payments to third-party service providers or subcontractors (such included expenses, collectively, "Out-of-Pocket Costs"), Recipient shall reimburse Provider for all such Out-of-Pocket Costs. (b) (i) Provider shall provide Recipient with monthly invoices ("Invoices"), which shall set forth in reasonable detail, with such supporting documentation as Recipient may reasonably request with respect to Out-of-Pocket Costs, amounts payable under this Agreement, and (ii) payments pursuant to this Agreement shall be made within fifteen (15) days after the date of receipt of an Invoice by Recipient from Provider. (c) Provider shall allow the Recipient to use [ * * * ] at no cost, until December 31, 2021. 2.3 Invoice Disputes. In the event of an Invoice dispute, Recipient shall deliver a written statement to Provider prior to the date payment is due on the disputed Invoice listing all disputed items and providing a reasonably detailed description of each disputed item. Amounts not so disputed shall be deemed accepted and shall be paid, notwithstanding disputes on other items. The Parties shall seek to resolve all such disputes expeditiously and in good faith. Provider shall continue performing the Services in accordance with this Agreement pending resolution of any dispute. + +3 + + + + + +2.4 No Right of Setoff. Each of the Parties hereby acknowledges that it shall have no right under this Agreement to offset any amounts owed (or to become due and owing) to the other Party, whether under this Agreement, the Purchase Agreement or otherwise, against any other amount owed (or to become due and owing) to it by the other Party. 3. Termination. 3.1 Termination of Agreement. This Agreement be deemed effective as of the Effective Date, Agreement and shall terminate on December 31, 2020, unless terminated earlier in accordance with Section 3.2. 3.2 Each of the Recipient and the Provider may, in their sole discretion, terminate this Agreement in whole or in part, at any time without cause, and without liability except, in the case of the Recipient, for required payment for services rendered and reimbursement for authorized expenses incurred, by providing at least 90 (ninety) days' prior written notice to the other party (such date, the "Services Termination Date"). 3.3 Breach. Any Party (the "Non-Breaching Party") may terminate this Agreement with respect to any Service, in whole but not in part, at any time upon prior written notice to the other Party (the "Breaching Party"), if the Breaching Party has failed (other than pursuant to Section 3.6) to perform any of its material obligations under this Agreement relating to such Service, and such failure shall have continued without cure for a period of 30 days after receipt by the Breaching Party of a written notice of such failure from the Non-Breaching Party seeking to terminate such service. For the avoidance of doubt, non-payment by Recipient for a Service provided by Provider in accordance with this Agreement and not the subject of a good-faith dispute shall be deemed a breach for purposes of this Section 3.3. 3.4 Insolvency. In the event that either Party hereto shall (a) file a petition in bankruptcy, (b) become or be declared insolvent, or become the subject of any proceedings (not dismissed within sixty (60) days) related to its liquidation, insolvency or the appointment of a receiver, (c) make an assignment on behalf of all or substantially all of its creditors, or (d) take any corporate action for its winding up or dissolution, then the other party shall have the right to terminate this Agreement by providing written notice in accordance with Section 6.6. 3.5 Effect of Termination. Upon termination of this Agreement in its entirety pursuant to Section 3.1, all obligations of the Parties hereto shall terminate, except for the provisions of Section 2.2, and the entirety of Sections 4, 5 and 6, which shall survive any termination or expiration of this Agreement. 3.6 Upon expiration or termination of this Agreement for any reason, Provider shall promptly: (a) Deliver to Recipient all documents, work product, and other materials, whether or not complete, prepared by or on behalf of Provider in the course of performing the Services for which Recipient has paid. (b) Return to Recipient all Recipient -owned property, equipment, or materials in its possession or control. + +4 + + + + + +(c) Remove any Provider-owned property, equipment, or materials located at Recipient's locations. (d) Deliver to Recipient, all documents and tangible materials (and any copies) containing, reflecting, incorporating, or based on Recipient's Confidential Information. (e) On a pro rata basis, repay all fees and expenses paid in advance for any Services which have not been provided. (f) Permanently erase all of Recipient's Confidential Information from its computer systems. (g) Certify in writing to Recipient that it has complied with the requirements of this Section 3.6 3.7 Force Majeure. If Provider is prevented from or delayed in complying, either totally or in part, with any of the terms or provisions of this Agreement by reason of fire, flood, storm, strike, lockout or other labor trouble or shortage, delays by unaffiliated suppliers or carriers, shortages of fuel, power, raw materials or components, any law, order, proclamation, regulation, ordinance, demand, seizure or requirement of any governmental authority, riot, civil commotion, war, rebellion, acts of terrorism, nuclear accident or other causes beyond the reasonable control of Provider, or acts, omissions, or delays in acting by any governmental or military authority or Recipient (each, a "Force Majeure"), then upon written notice to Recipient, the Services affected by the Force Majeure (the "Affected Services") and/or other requirements of this Agreement will be suspended during the period of such Force Majeure and Provider will have no liability to Recipient or any other party in connection with such Affected Services. If the Force Majeure in question prevails for a continuous period in excess of three months after the date on which the Force Majeure begins, Provider shall be entitled to give notice to Recipient to terminate the Affected Services. The notice to terminate must specify the termination date, which must be not less than ten (10) days after the date on which the notice to terminate is given. Once a notice to terminate has been validly given, the Affected Services will terminate on the termination date set out in the notice. Neither Party shall have any liability to the other in respect of termination of the Affected Services due to Force Majeure, but rights and liabilities which have accrued prior to termination shall subsist. + +5 + + + + + +4. Confidentiality. 4.1 Confidentiality. During the term of this Agreement and thereafter, the Parties hereto shall, and shall instruct their respective representatives to, maintain in confidence and not disclose the other Party's financial, technical, sales, marketing, development, personnel, and other information, records, or data, including, without limitation, customer lists, supplier lists, trade secrets, designs, product formulations, product specifications or any other proprietary or confidential information, however recorded or preserved, whether written or oral (any such information, "Confidential Information"). Each Party hereto shall use the same degree of care, but no less than reasonable care, to protect the other Party's Confidential Information as it uses to protect its own Confidential Information of like nature. Unless otherwise authorized in any other agreement between the Parties, any Party receiving any Confidential Information of the other Party (the "Receiving Party") may use Confidential Information only for the purposes of fulfilling its obligations under this Agreement (the "Permitted Purpose"). Any Receiving Party may disclose such Confidential Information only to its representatives who have a need to know such information for the Permitted Purpose and who have been advised of the terms of this Section 4.1 and the Receiving Party shall be liable for any breach of these confidentiality provisions by such Persons; provided, however, that any Receiving Party may disclose such Confidential Information to the extent such Confidential Information is required to be disclosed by law, in which case the Receiving Party shall promptly notify, to the extent possible, the disclosing party (the "Disclosing Party"), and take reasonable steps to assist in contesting such disclosure requirement or in protecting the Disclosing Party's rights prior to disclosure, and in which case the Receiving Party shall only disclose such Confidential Information that it is advised by its counsel in writing that it is legally bound to disclose. Notwithstanding the foregoing, "Confidential Information" shall not include any information that the Receiving Party can demonstrate: (a) was publicly known at the time of disclosure to it, or has become publicly known through no act of the Receiving Party or its representatives in breach of this Section 4.1, (b) was rightfully received from a third party without a duty of confidentiality, or (c) was developed by it independently without any reliance on the Confidential Information. 4.2 Return of Confidential Information. Upon demand by the Disclosing Party at any time, or upon expiration or termination of this Agreement with respect to any Service, the Receiving Party agrees promptly to return or destroy, at the Disclosing Party's option, all Confidential Information received in connection with this Agreement. If such Confidential Information is destroyed, an authorized officer of the Receiving Party shall certify to such destruction in writing. 5. Indemnification. 5.1 Indemnification. Provider shall indemnify, defend, and hold harmless Recipient and its officers, directors, employees, agents, affiliates, successors, and permitted assigns (collectively, "Indemnified Party") against any and all losses, damages, liabilities, deficiencies, claims, actions, judgments, settlements, interest, awards, penalties, fines, costs, or expenses of whatever kind, including attorneys' fees, fees and the costs of enforcing any right to indemnification under this Agreement, and the cost of pursuing any insurance providers, incurred by Indemnified Party or awarded against Indemnified Party (collectively, "Losses"), relating to/arising out of or resulting from any claim of a third party or Recipient arising out of or occurring in connection with Provider's negligence, willful misconduct, or breach of this Agreement. Provider shall not enter into any settlement without Recipient's or Indemnified Party's prior written consent. 6. Miscellaneous. 6.1 Entire Agreement. This Agreement, the Purchase Agreement and the documents referred to herein and therein constitute the entire agreement among the Parties and supersedes any prior understandings, agreements, or representations by or among the Parties, written or oral, to the extent they relate in any way to the subject matter hereof. + +6 + + + + + +6.2 Succession and Assignment. This Agreement shall be binding upon and inure to the benefit of the Parties named herein and their respective successors and permitted assigns. Provider may not assign, delegate or otherwise transfer either this Agreement or any of its rights, interests, or obligations hereunder without the prior written approval of Recipient. 6.3 Counterparts. This Agreement may be executed in one or more counterparts, each of which shall be deemed an original but all of which together will constitute one and the same instrument. Counterparts may be delivered via facsimile and electronic mail (including portable document format (PDF) or any electronic signature complying with the U.S. federal ESIGN Act of 2000, e.g., www.docusign.com). 6.4 Titles and Headings. Titles and section headings contained in this Agreement are inserted for convenience only and shall not affect in any way the meaning or interpretation of this Agreement. 7. Notices. All notices, requests, consents, claims, demands, waivers and other communications hereunder shall be in writing and shall be deemed to have been given (a) when delivered by hand (with written confirmation of receipt); (b) when received by the addressee if sent by a nationally recognized overnight courier (receipt requested); (c) on the date sent by facsimile or e-mail of a PDF document (with confirmation of transmission) if sent during normal business hours of the recipient, and on the next Business Day if sent after normal business hours of the recipient or (d) on the third day after the date mailed, by certified or registered mail, return receipt requested, postage prepaid. Such communications must be sent to the respective parties at the following addresses (or at such other address for a party as shall be specified in a notice given in accordance with this Section 7: If to Provider: [ * * * ] With a copy to: N/A If to Recipient: TELCOSTAR PTE. LTD 6 Eu Tong Sen Street Tel Aviv, Israel, 6770007 #10-15 The Central Singapore 059817 Email: avi@ability.co.il Attention: Avi Levin With a copy to: McDermott Will & Emery LLP 340 Madison Avenue New York, NY 10173-1922 Telephone: (212) 547-5541 Facsimile: (212) 547-5444 EMAIL: GEMMANUEL@MWE.COM Attention: Gary Emmanuel + +7 + + + + + +Any Party may change the address to which notices, requests, demands, claims, and other communications hereunder are to be delivered by giving the other Parties notice in the manner herein set forth. 7.1 Further Assurances. The Parties agree (a) to furnish upon request to each other such further information, (b) to execute and deliver to each other such other documents, and (c) to do such other acts and things, all as the other party may reasonably request for the purpose of carrying out the intent of this Agreement and the documents referred to in this Agreement. 7.2 Governing Law. This Agreement and any claim, controversy or dispute arising out of or related to this Agreement, any of the transactions contemplated hereby and/or the interpretation and enforcement of the rights and duties of the Parties, whether arising in contract, tort, equity or otherwise, shall be governed by and construed in accordance with the domestic laws of the State of Israel (including in respect of the statute of limitations or other limitations period applicable to any such claim, controversy or dispute), without giving effect to any choice or conflict of law provision or rule (whether of the State of Israel or any other jurisdiction) that would cause the application of the laws of any jurisdiction other than the State of Israel. 7.3 Consent to Jurisdiction. The Parties hereby irrevocably submit any disputes under this Agreement to the exclusive jurisdiction of the courts located in Tel-Aviv, Israel, provided however, that Recipient shall be entitled to seek an injunction or other appropriate remedy against Provider in the country in which Provider has acted in breach of the terms hereof. 7.4 Specific Performance. The Parties hereby agree that, in the event of breach of this Agreement, damages would be difficult, if not impossible, to ascertain and that irreparable damage would occur in the event that any of the provisions of this Agreement were not performed in accordance with their specific terms or were otherwise breached. Accordingly, it is hereby agreed that the Parties shall be entitled to seek an injunction or other equitable relief in any court of competent jurisdiction to enjoin any such breach and enforce specifically the terms and provisions hereof, this being in addition to any other remedy or right to which they are entitled at law or in equity, without any necessity of proving damages or any requirement for the posting of a bond or other security. 7.5 Amendments and Waivers. No amendment of any provision of this Agreement shall be valid unless the same shall be in writing and signed by Recipient and the Provider. No waiver by any Party of any provision of this Agreement or any default, misrepresentation, or breach of warranty or covenant hereunder, whether intentional or not, shall be valid unless the same shall be in writing and signed by the Party making such waiver nor shall such waiver be deemed to extend to any prior or subsequent default, misrepresentation, or breach of warranty or covenant hereunder or affect in any way any rights arising by virtue of any prior or subsequent such occurrence. + +8 + + + + + +7.6 Severability. Any term or provision of this Agreement that is held invalid or unenforceable by a court of competent jurisdiction or other competent governmental authority in any situation in any jurisdiction shall not affect the validity or enforceability of the remaining terms and provisions hereof or the validity or enforceability of the offending term or provision in any other situation or in any other jurisdiction. Upon such a determination, the Parties shall negotiate in good faith to replace invalid or unenforceable provisions with valid provisions, the economic effect of which comes as close as possible to that of the invalid or unenforceable provisions. 7.7 Construction. The Parties have participated jointly in the negotiation and drafting of this Agreement. In the event an ambiguity or question of intent or interpretation arises, this Agreement shall be construed as if drafted jointly by the Parties and no presumption or burden of proof shall arise favoring or disfavoring any Party by virtue of the authorship of any of the provisions of this Agreement. Any reference to any law shall be deemed also to refer to all rules and regulations promulgated thereunder, unless the context requires otherwise. The word "including" shall mean including without limitation. 7.8 Incorporation of Exhibits and Disclosure Schedule. The Exhibit identified in this Agreement is incorporated herein by reference and made a part hereof. 7.9 Amendment and Restatement. This Agreement amends and restates in full the Production Contract. [SIGNATURE PAGE FOLLOWS] + +9 + + + + + +IN WITNESS WHEREOF, the Parties have executed this Services Agreement as of the date first written above. PROVIDER: RECIPIENT: [ * * * ] [ * * * ] By: By: Name: Name: Title: Title: + +10 + + + + + +EXHIBIT A Services [ * * * ] 11 \ No newline at end of file diff --git a/integration-tests/benchmark-data/cuad-prototype/qa.json b/integration-tests/benchmark-data/cuad-prototype/qa.json new file mode 100644 index 00000000..3a8d7a3b --- /dev/null +++ b/integration-tests/benchmark-data/cuad-prototype/qa.json @@ -0,0 +1,10 @@ +[ + { + "input": "In CO-BRANDING AGREEMENT signed by ebix.com, Inc. and About.com, Inc., is there any clause providing for joint or shared ownership of intellectual property between the parties to the contract?", + "output": [ + { + "text": "Upon request by About, ebix shall provide About with About Customer Data in the aggregated form, which aggregated form shall be jointly owned by ebix and About." + } + ] + } +] \ No newline at end of file diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index ed3a95d4..1ed47877 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -242,28 +242,28 @@ cp -r $GRAPHRAG_TOOLKIT_DIR/byokg-rag/src/* graphrag-toolkit cp -r $GRAPHRAG_TOOLKIT_DIR/examples/lexical-graph/notebooks/* lexical-graph-examples cp -r $GRAPHRAG_TOOLKIT_DIR/examples/byokg-rag/* lexical-graph-examples cp -r ./../test-scripts/* lexical-graph-examples -cp -r ./../source-data/* lexical-graph-examples +cp -r ./../source-data/* lexical-graph-examples/source-data # Include benchmark data if local dir is specified and no S3 URI is provided # Only copies dataset subdirectories that match the tests being run if [[ "$BENCHMARK_DATA_DIR" ]] && [[ -z "$BENCHMARK_DATA_S3_URI" ]]; then mkdir -p lexical-graph-examples/source-data BENCHMARK_DATASETS="" + protype_string="" + if [[ "$BENCHMARK_IS_PROTOTYPE" == "true" ]]; then + protype_string="-prototype" + fi if echo "$TESTS" | grep -qi "Cuad"; then - if [[ "$BENCHMARK_IS_PROTOTYPE" == "true" ]]; then - BENCHMARK_DATASETS="$BENCHMARK_DATASETS cuad-prototype" - else - BENCHMARK_DATASETS="$BENCHMARK_DATASETS cuad" - fi + BENCHMARK_DATASETS="$BENCHMARK_DATASETS cuad$protype_string" fi if echo "$TESTS" | grep -qi "Pga"; then - BENCHMARK_DATASETS="$BENCHMARK_DATASETS pga" + BENCHMARK_DATASETS="$BENCHMARK_DATASETS pga$protype_string" fi if echo "$TESTS" | grep -qi "Concurrentqa"; then - BENCHMARK_DATASETS="$BENCHMARK_DATASETS concurrentqa" + BENCHMARK_DATASETS="$BENCHMARK_DATASETS concurrentqa$protype_string" fi if echo "$TESTS" | grep -qi "Wikihow"; then - BENCHMARK_DATASETS="$BENCHMARK_DATASETS wikihow" + BENCHMARK_DATASETS="$BENCHMARK_DATASETS wikihow$protype_string" fi for ds in $BENCHMARK_DATASETS; do if [[ -d "$BENCHMARK_DATA_DIR/$ds" ]]; then diff --git a/integration-tests/source-data/source-data/collection-1/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json b/integration-tests/source-data/collection-1/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json similarity index 100% rename from integration-tests/source-data/source-data/collection-1/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json rename to integration-tests/source-data/collection-1/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json diff --git a/integration-tests/source-data/source-data/collection-2/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json b/integration-tests/source-data/collection-2/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json similarity index 100% rename from integration-tests/source-data/source-data/collection-2/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json rename to integration-tests/source-data/collection-2/aws__abd5427c_6614/aws__abd5427c_6614_848bb4a4.json diff --git a/integration-tests/source-data/source-data/corpus-modified.json b/integration-tests/source-data/corpus-modified.json similarity index 100% rename from integration-tests/source-data/source-data/corpus-modified.json rename to integration-tests/source-data/corpus-modified.json diff --git a/integration-tests/source-data/source-data/versioning/v1/readme.md b/integration-tests/source-data/versioning/v1/readme.md similarity index 100% rename from integration-tests/source-data/source-data/versioning/v1/readme.md rename to integration-tests/source-data/versioning/v1/readme.md diff --git a/integration-tests/source-data/source-data/versioning/v2/readme.md b/integration-tests/source-data/versioning/v2/readme.md similarity index 100% rename from integration-tests/source-data/source-data/versioning/v2/readme.md rename to integration-tests/source-data/versioning/v2/readme.md diff --git a/integration-tests/source-data/source-data/versioning/v3/readme.md b/integration-tests/source-data/versioning/v3/readme.md similarity index 100% rename from integration-tests/source-data/source-data/versioning/v3/readme.md rename to integration-tests/source-data/versioning/v3/readme.md diff --git a/integration-tests/source-data/source-data/versioning/v4/readme.md b/integration-tests/source-data/versioning/v4/readme.md similarity index 100% rename from integration-tests/source-data/source-data/versioning/v4/readme.md rename to integration-tests/source-data/versioning/v4/readme.md From 0e953912663032b495318471331e8e48b9e34ae1 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 27 Apr 2026 20:47:46 -0700 Subject: [PATCH 7/9] Update profiles to include haiku 4.5 Signed-off-by: Andrew Carbonetto --- ...kit-neptune-analytics-aurora-postgres.json | 32 +++++++++++++++++++ ...ptune-analytics-opensearch-serverless.json | 32 +++++++++++++++++++ ...-toolkit-neptune-analytics-s3-vectors.json | 32 +++++++++++++++++++ .../graphrag-toolkit-neptune-analytics.json | 32 +++++++++++++++++++ ...ptune-db-aurora-postgres-existing-vpc.json | 32 +++++++++++++++++++ ...ag-toolkit-neptune-db-aurora-postgres.json | 32 +++++++++++++++++++ ...lkit-neptune-db-opensearch-serverless.json | 32 +++++++++++++++++++ ...raphrag-toolkit-neptune-db-s3-vectors.json | 32 +++++++++++++++++++ integration-tests/build-tests.sh | 2 +- ...g-toolkit-neo4j-opensearch-serverless.json | 32 +++++++++++++++++++ .../graphrag-toolkit-tests.json | 32 +++++++++++++++++++ .../graphrag_toolkit_tests/benchmark_build.py | 4 ++- .../graphrag_toolkit_tests/benchmark_query.py | 13 +++++--- 13 files changed, 332 insertions(+), 7 deletions(-) diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-aurora-postgres.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-aurora-postgres.json index 23ba68aa..19158a95 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-aurora-postgres.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-aurora-postgres.json @@ -823,6 +823,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-opensearch-serverless.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-opensearch-serverless.json index 56ed7367..c47a9b62 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-opensearch-serverless.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-opensearch-serverless.json @@ -384,6 +384,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-s3-vectors.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-s3-vectors.json index 19a402d1..c963a6d9 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-s3-vectors.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics-s3-vectors.json @@ -440,6 +440,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics.json index f5a3ee6e..c10b7433 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-analytics.json @@ -336,6 +336,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres-existing-vpc.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres-existing-vpc.json index c4d1b89c..4b04733f 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres-existing-vpc.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres-existing-vpc.json @@ -763,6 +763,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres.json index 328fb4e8..e4c75bbd 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-aurora-postgres.json @@ -714,6 +714,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-opensearch-serverless.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-opensearch-serverless.json index 0ca41004..d621a3e6 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-opensearch-serverless.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-opensearch-serverless.json @@ -587,6 +587,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-s3-vectors.json b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-s3-vectors.json index 667295d3..e60293ad 100644 --- a/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-s3-vectors.json +++ b/examples/lexical-graph/cloudformation-templates/graphrag-toolkit-neptune-db-s3-vectors.json @@ -645,6 +645,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index 1ed47877..3105bf74 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -242,7 +242,7 @@ cp -r $GRAPHRAG_TOOLKIT_DIR/byokg-rag/src/* graphrag-toolkit cp -r $GRAPHRAG_TOOLKIT_DIR/examples/lexical-graph/notebooks/* lexical-graph-examples cp -r $GRAPHRAG_TOOLKIT_DIR/examples/byokg-rag/* lexical-graph-examples cp -r ./../test-scripts/* lexical-graph-examples -cp -r ./../source-data/* lexical-graph-examples/source-data +cp -r ./../source-data lexical-graph-examples/source-data # Include benchmark data if local dir is specified and no S3 URI is provided # Only copies dataset subdirectories that match the tests being run diff --git a/integration-tests/cloudformation-templates/graphrag-toolkit-neo4j-opensearch-serverless.json b/integration-tests/cloudformation-templates/graphrag-toolkit-neo4j-opensearch-serverless.json index f21cd376..4763f6a9 100644 --- a/integration-tests/cloudformation-templates/graphrag-toolkit-neo4j-opensearch-serverless.json +++ b/integration-tests/cloudformation-templates/graphrag-toolkit-neo4j-opensearch-serverless.json @@ -570,6 +570,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0" }, diff --git a/integration-tests/cloudformation-templates/graphrag-toolkit-tests.json b/integration-tests/cloudformation-templates/graphrag-toolkit-tests.json index d69f53ed..73d8f694 100644 --- a/integration-tests/cloudformation-templates/graphrag-toolkit-tests.json +++ b/integration-tests/cloudformation-templates/graphrag-toolkit-tests.json @@ -396,6 +396,38 @@ } ] }, + { + "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-haiku-4-5-20251001-v1:0" + }, + { + "Fn::Sub": [ + "arn:${partition}:bedrock:${region}:${accountId}:inference-profile/${regionPrefix}.anthropic.claude-haiku-4-5-20251001-v1:0", + { + "partition": { + "Ref": "AWS::Partition" + }, + "region": { + "Ref": "AWS::Region" + }, + "accountId": { + "Ref": "AWS::AccountId" + }, + "regionPrefix": { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "-", + { + "Ref": "AWS::Region" + } + ] + } + ] + } + } + ] + }, { "Fn::Sub": "arn:${AWS::Partition}:bedrock:*::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0" }, diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py index e70ed267..9540a534 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_build.py @@ -33,7 +33,9 @@ BENCHMARK_DATA_DIR = 'source-data' -def run_benchmark_build(handler: IntegrationTestHandler, dataset: str, data_dir: str, +def run_benchmark_build(handler: IntegrationTestHandler, + dataset: str, + data_dir: str, graph_store_conn: Optional[str] = None, vector_store_conn: Optional[str] = None): """ Builds graph and vector stores from pre-extracted document chunks for a benchmark dataset. diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py index ccd456d6..216eedfa 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_query.py @@ -22,6 +22,7 @@ 'concurrentqa': ['qa.json'], } +BENCHMARK_DATA_DIR = 'source-data' def load_qa_pairs(data_dir: str, dataset: str, qa_files: List[str], limit: Optional[int] = None): pairs = [] @@ -34,8 +35,10 @@ def load_qa_pairs(data_dir: str, dataset: str, qa_files: List[str], limit: Optio return pairs -def run_benchmark_query(handler: IntegrationTestHandler, params: Dict[str, Any], - dataset: str, data_dir: str, +def run_benchmark_query(handler: IntegrationTestHandler, + params: Dict[str, Any], + dataset: str, + data_dir: str, graph_store_conn: Optional[str] = None, vector_store_conn: Optional[str] = None, response_llm: str = 'anthropic.claude-sonnet-4-20250514-v1:0', @@ -145,15 +148,15 @@ def wait(self) -> bool: return len(vector_store.get_index('chunk').top_k(QueryBundle(query_str='contract'), top_k=1)) == 0 def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): - data_dir = os.environ.get('BENCHMARK_DATA_DIR') limit_str = os.environ.get('BENCHMARK_QA_LIMIT') is_prototype = os.environ.get('BENCHMARK_IS_PROTOTYPE') dataset_name = 'cuad-prototype' if is_prototype == 'true' else 'cuad' run_benchmark_query( - handler, params, + handler, + params, dataset=dataset_name, - data_dir=data_dir, + data_dir=BENCHMARK_DATA_DIR, graph_store_conn=os.environ.get('GRAPH_STORE'), vector_store_conn=os.environ.get('VECTOR_STORE'), response_llm=os.environ.get('TEST_RESPONSE_LLM', 'anthropic.claude-sonnet-4-20250514-v1:0'), From 23ac9895ef63357ea3377948d508b77fcc7797b5 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 27 Apr 2026 20:59:48 -0700 Subject: [PATCH 8/9] Rm temp directory Signed-off-by: Andrew Carbonetto --- integration-tests/build-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/build-tests.sh b/integration-tests/build-tests.sh index 3105bf74..4bba2624 100644 --- a/integration-tests/build-tests.sh +++ b/integration-tests/build-tests.sh @@ -341,7 +341,7 @@ zip -r graphrag.zip graphrag cp graphrag.zip ../target/graphrag.zip popd -# rm -rf temp +rm -rf temp pushd target unzip graphrag.zip From da81accccca6832679f6ceecd9fa5d95e6699c7f Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 29 Apr 2026 15:30:41 -0700 Subject: [PATCH 9/9] Fix evaluate tests Signed-off-by: Andrew Carbonetto --- .../benchmark_evaluate.py | 14 +++-- .../benchmark_utils/run_evaluation.py | 56 ++++++++----------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py index 72f31d0c..32e60c44 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_evaluate.py @@ -46,6 +46,9 @@ def run_benchmark_evaluate(handler: IntegrationTestHandler, params: Dict[str, An for line in fin: data.append(json.loads(line)) + model_id = os.environ.get('TEST_RESPONSE_LLM') + assert model_id, 'TEST_RESPONSE_LLM environment variable must be set' + scores = {} for metric in metrics: @@ -65,9 +68,9 @@ def run_benchmark_evaluate(handler: IntegrationTestHandler, params: Dict[str, An score = count / total if total > 0 else 0.0 else: if metric == 'correctness': - evaluator = CorrectnessEvaluator(model_id='anthropic.claude-3-sonnet-20240229-v1:0') + evaluator = CorrectnessEvaluator(model_id=model_id) elif metric == 'idk': - evaluator = IDKEvaluator(model_id='anthropic.claude-3-sonnet-20240229-v1:0') + evaluator = IDKEvaluator(model_id=model_id) evals = [] for example in data: @@ -122,10 +125,13 @@ def _run_test(self, handler: IntegrationTestHandler, params: Dict[str, Any]): dataset_name = 'cuad-prototype' if is_prototype == 'true' else 'cuad' responses_path = params.get('benchmark_responses_path', - os.path.join('benchmark-results', dataset_name, 'responses.jsonl')) + os.path.join('benchmark-results', + dataset_name, + 'responses.jsonl')) run_benchmark_evaluate( - handler, params, + handler, + params, dataset=dataset_name, responses_path=responses_path, metrics=['correctness', 'idk'], diff --git a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py index ba586896..94184635 100644 --- a/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py +++ b/integration-tests/test-scripts/graphrag_toolkit_tests/benchmark_utils/run_evaluation.py @@ -16,42 +16,32 @@ def call_bedrock_invoke_model(prompt, bedrock, model_id, is_json_output=True): accept = 'application/json' contentType = 'application/json' - if 'anthropic.claude-3' in model_id: - payload_body = { - "anthropic_version": "bedrock-2023-05-31", - "max_tokens": 2048, - "temperature": 0.0, - "top_p": 1, - "top_k": 50, - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": prompt - } - ] - } - ] - } - body = json.dumps(payload_body) - else: - body = json.dumps({"prompt": prompt, "max_tokens_to_sample": 2048, "temperature": 0.0, "top_p": 1, "top_k": 50, - "stop_sequences": ["\\n\\nHuman:"]}) + payload_body = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 2048, + # Error: `temperature` and `top_p` cannot both be specified for this model. Please use only one. + # "temperature": 0.0, + "top_p": 1, + "top_k": 50, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt + } + ] + } + ] + } + body = json.dumps(payload_body) response = bedrock.invoke_model(body=body, modelId=model_id, accept=accept, contentType=contentType) - if 'anthropic.claude-3' in model_id: - response = response['body'].read().decode('utf-8') - response = json.loads(response) - response_text = response['content'][0]['text'] - - else: - response_obj = response.get('body').read().decode('utf-8') - print("RESPONSE OBJ: " + str(response_obj)) - json_response_obj = json.loads(response_obj) - response_text = json_response_obj['completion'] + response = response['body'].read().decode('utf-8') + response = json.loads(response) + response_text = response['content'][0]['text'] if is_json_output: try: start_idx = response_text.find("{")