From 71f26973c3853efc8de2ad772c9f435909150d95 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 13 May 2024 20:53:39 -0300 Subject: [PATCH 01/69] adicionado nova tool para gerar testes --- nimrod/__main__.py | 5 +- nimrod/setup_tools/tools.py | 3 +- .../codellama_test_suite_generator.py | 149 ++++++++++++++++++ nimrod/tools/codellama.py | 12 ++ 4 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 nimrod/test_suite_generation/generators/codellama_test_suite_generator.py create mode 100644 nimrod/tools/codellama.py diff --git a/nimrod/__main__.py b/nimrod/__main__.py index 81df5624..1d95e4d4 100644 --- a/nimrod/__main__.py +++ b/nimrod/__main__.py @@ -16,6 +16,7 @@ from nimrod.test_suite_generation.generators.randoop_test_suite_generator import RandoopTestSuiteGenerator from nimrod.test_suite_generation.generators.evosuite_differential_test_suite_generator import EvosuiteDifferentialTestSuiteGenerator from nimrod.test_suite_generation.generators.evosuite_test_suite_generator import EvosuiteTestSuiteGenerator +from nimrod.test_suite_generation.generators.codellama_test_suite_generator import CodellamaTestSuiteGenerator from nimrod.test_suite_generation.generators.project_test_suite_generator import ProjectTestSuiteGenerator from nimrod.test_suites_execution.main import TestSuitesExecution, TestSuiteExecutor from nimrod.tools.bin import MOD_RANDOOP, RANDOOP @@ -26,7 +27,7 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]: config_generators = config.get( - 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'project']) + 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'codellama', 'project']) generators: List[TestSuiteGenerator] = list() if 'randoop' in config_generators: @@ -38,6 +39,8 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator generators.append(EvosuiteTestSuiteGenerator(Java())) if 'evosuite-differential' in config_generators: generators.append(EvosuiteDifferentialTestSuiteGenerator(Java())) + if 'codellama' in config_generators: + generators.append(CodellamaTestSuiteGenerator(Java())) if 'project' in config_generators: generators.append(ProjectTestSuiteGenerator(Java())) diff --git a/nimrod/setup_tools/tools.py b/nimrod/setup_tools/tools.py index e517dbf3..c0985757 100644 --- a/nimrod/setup_tools/tools.py +++ b/nimrod/setup_tools/tools.py @@ -4,4 +4,5 @@ class Tools(Enum): RANDOOP='RANDOOP' RANDOOP_MOD='RANDOOP-MODIFIED' EVOSUITE='EVOSUITE' - DIFF_EVOSUITE='DIFFERENTIAL-EVOSUITE' \ No newline at end of file + DIFF_EVOSUITE='DIFFERENTIAL-EVOSUITE' + CODELLAMA='CODELLAMA' \ No newline at end of file diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py new file mode 100644 index 00000000..fc3687d0 --- /dev/null +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -0,0 +1,149 @@ +import os +from typing import Dict, List +from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis + +from nimrod.test_suite_generation.generators.test_suite_generator import \ + TestSuiteGenerator +from nimrod.tools.java import Java +from nimrod.utils import generate_classpath + +from mlc_chat import ChatModule + + +class CodellamaTestSuiteGenerator(TestSuiteGenerator): + TARGET_METHODS_LIST_FILENAME = 'methods_to_test.txt' + TARGET_CLASS_LIST_FILENAME = 'classes_to_test.txt' + + def get_generator_tool_name(self) -> str: + return "CODELLAMA" + + def _get_test_suite_class_paths(self, path: str) -> List[str]: + paths = [] + + for node in os.listdir(path): + if os.path.isdir(os.path.join(path, node)): + paths += self._get_test_suite_class_paths(os.path.join(path, node)) + elif node.endswith(".java"): + paths.append(os.path.join(path, node)) + + return paths + + + def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: + class_names = [] + + for class_path in self._get_test_suite_class_paths(test_suite_path): + class_fqcn = os.path.relpath(class_path, os.path.join( + test_suite_path, "codellama-tests")).replace(os.sep, ".") + class_names.append(class_fqcn[:-5]) + + return class_names + + def generate_prompts(self, prompts_file, class_name, methods, code): + for method in methods: + temp_method = method.split("(")[0] + + try: + with open(code, 'r') as file: + lines = file.readlines() + + method_found = False + method_code = [] + for line in lines: + if temp_method in line and ("private" in line or "public" in line): + method_found = True + elif temp_method not in line and ("private" in line or "public" in line): + if method_found: + break + if method_found: + method_code.append(line) + + if method_found: + modified_lines = [] + modified_lines.append(f"/*{''.join(method_code)}*/\n\n") + modified_lines.append("import org.junit.FixMethodOrder;\n") + modified_lines.append("import org.junit.Test;\n") + modified_lines.append("import org.junit.runners.MethodSorters;\n\n") + print(f"Generating prompt for method {method}") + modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") + modified_lines.append(f"public class {class_name}Test {{\n") + modified_lines.append(f" public static boolean debug = false;\n\n") + modified_lines.append(f" @Test\n") + modified_lines.append(f" public void {temp_method}_001() throws Throwable {{\n") + modified_lines.append(f" if (debug) {{\n") + modified_lines.append(f"\"\"\"\n") + + with open(prompts_file, "w") as file: + file.writelines(modified_lines) + + except Exception as e: + print(f"Error while generating prompt for method {method}: {e}") + + def read_prompts(self, file_path): + prompts = [] + current_prompt = "" + + with open(file_path, "r") as file: + lines = file.readlines() + + for line in lines: + if line.strip() != '"""': + current_prompt += line + else: + if current_prompt.strip() != "": + prompts.append(current_prompt) + current_prompt = "" + + return prompts + + def create_chat_module(self, mpath, lpath, model, lib): + return ChatModule( + model=f"{mpath}/{model}", + model_lib_path=f"{lpath}/{lib}", + ) + + + def generate_output(self, chat_module, prompt): + return chat_module.generate(prompt=prompt) + + + def save_output(self, prompt, output, dir, output_file_name): + if dir: + if not os.path.exists(dir): + os.makedirs(dir) + + output_file_path = f"{dir}/{output_file_name}.txt" + with open(output_file_path, "w") as f: + f.write(prompt + "\n" + output) + + + def reset_chat_module(self, chat_module): + chat_module.reset_chat() + + def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: + class_name, methods = list(scenario.targets.items())[0] + + mpath = "/home/nfab/dist" + lpath = "/home/nfab/dist/libs" + model = "CodeLlama-7b-Instruct-hf-q4f16_1-MLC" + lib = "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" + outputs_dir = f"{output_path}" + prompts_path = f"{output_path}/prompts.txt" + + #code_base = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_base.java" + code_left = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_left.java" + #code_right = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_right.java" + #code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" + + self.generate_prompts(prompts_path, class_name, methods, code_left) + prompts_list = self.read_prompts(prompts_path) + + cm = self.create_chat_module(mpath, lpath, model, lib) + + for i, prompt in enumerate(prompts_list): + for j in range(0, 2): + print(f"----------------------------- Generating output {i}-{j}") + output_file_name = f"output{i}-{j}" + output = self.generate_output(cm, prompt) + self.save_output(prompt, output, outputs_dir, output_file_name) + self.reset_chat_module(cm) \ No newline at end of file diff --git a/nimrod/tools/codellama.py b/nimrod/tools/codellama.py new file mode 100644 index 00000000..952607c1 --- /dev/null +++ b/nimrod/tools/codellama.py @@ -0,0 +1,12 @@ +import os + +from nimrod.tools.suite_generator import Suite, SuiteGenerator +from nimrod.utils import generate_classpath + +class Codellama(SuiteGenerator): + + def _get_tool_name(self): + return "codellama" + + def _test_classes(self): + return ['RegressionTest', 'ErrorTest'] From 52ad4a28719b194009e72cef55dbbd7aa7e0dc74 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 13 May 2024 21:38:17 -0300 Subject: [PATCH 02/69] gera outputs para diferentes branches --- .../codellama_test_suite_generator.py | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index fc3687d0..ae0fd173 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -71,7 +71,8 @@ def generate_prompts(self, prompts_file, class_name, methods, code): modified_lines.append(f" @Test\n") modified_lines.append(f" public void {temp_method}_001() throws Throwable {{\n") modified_lines.append(f" if (debug) {{\n") - modified_lines.append(f"\"\"\"\n") + modified_lines.append(f" // test here\n") + modified_lines.append(f"\"\"\"") with open(prompts_file, "w") as file: file.writelines(modified_lines) @@ -114,8 +115,17 @@ def save_output(self, prompt, output, dir, output_file_name): output_file_path = f"{dir}/{output_file_name}.txt" with open(output_file_path, "w") as f: - f.write(prompt + "\n" + output) - + f.write(prompt + output) + + def get_branch(self, input_jar, code_base, code_left, code_right, code_merge): + if 'base' in input_jar: + return code_base, "base" + if 'left' in input_jar: + return code_left, "left" + if 'right' in input_jar: + return code_right, "right" + if 'merge' in input_jar: + return code_merge, "merge" def reset_chat_module(self, chat_module): chat_module.reset_chat() @@ -127,15 +137,18 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s lpath = "/home/nfab/dist/libs" model = "CodeLlama-7b-Instruct-hf-q4f16_1-MLC" lib = "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" - outputs_dir = f"{output_path}" prompts_path = f"{output_path}/prompts.txt" - #code_base = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_base.java" + code_base = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_base.java" code_left = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_left.java" - #code_right = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_right.java" - #code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" + code_right = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_right.java" + code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" + + code, branch = self.get_branch(input_jar, code_base, code_left, code_right, code_merge) + print (f"Branch: {branch}") + + self.generate_prompts(prompts_path, class_name, methods, code) - self.generate_prompts(prompts_path, class_name, methods, code_left) prompts_list = self.read_prompts(prompts_path) cm = self.create_chat_module(mpath, lpath, model, lib) @@ -143,7 +156,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s for i, prompt in enumerate(prompts_list): for j in range(0, 2): print(f"----------------------------- Generating output {i}-{j}") - output_file_name = f"output{i}-{j}" + output_file_name = f"output{i}-{j}-{class_name}-{branch}" output = self.generate_output(cm, prompt) - self.save_output(prompt, output, outputs_dir, output_file_name) + self.save_output(prompt, output, output_path, output_file_name) self.reset_chat_module(cm) \ No newline at end of file From 77a5566c8674b4646fd88fe72012eed72dc7b690 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 16 May 2024 21:05:55 -0300 Subject: [PATCH 03/69] =?UTF-8?q?altera=C3=A7=C3=B5es=20nas=20fun=C3=A7?= =?UTF-8?q?=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codellama_test_suite_generator.py | 89 +++++++++++++++++-- nimrod/tools/codellama.py | 87 +++++++++++++++++- 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index ae0fd173..73c73f48 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -39,6 +39,81 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return class_names + def _create_method_list(self, methods: "List[str]"): + rectified_methods = [self._convert_method_signature( + method) for method in methods] + return (":").join(rectified_methods) + + def _convert_method_signature(self, meth_signature: str) -> str: + method_return = "" + try: + method_return = meth_signature.split(")")[1] + except Exception as e: + print(e) + meth_name = meth_signature[:meth_signature.rfind("(")] + meth_args = meth_signature[meth_signature.find( + "(") + 1:meth_signature.rfind(")")].split(",") + asm_meth_format = self._asm_based_method_method_descriptor( + meth_args, method_return) + + return meth_name+asm_meth_format + + # See at: https://asm.ow2.io/asm4-guide.pdf -- Section 2.1.3 and 2.1.4 + # Java type Type descriptor + # boolean Z + # char C + # byte B + # short S + # int I + # float F + # long J + # double D + # Object Ljava/lang/Object; + # int[] [I + # Object[][] [[Ljava/lang/Object; + def _asm_based_method_method_descriptor(self, method_arguments, method_return): + result = '(' + for arg in method_arguments: + arg = arg.strip() + result = result + self._asm_based_type_descriptor(arg) + result = result + ')' + result = result + self._asm_based_type_descriptor(method_return) + return result + + def _asm_based_type_descriptor(self, arg): + result = '' + if '[]' in arg: + result = result + '[' + arg = arg.replace('[]', '') + + if arg == '': + result = result + '' + elif arg == 'int': + result = result + 'I' + elif arg == 'float': + result = result + 'F' + elif arg == 'boolean': + result = result + 'Z' + elif arg == 'char': + result = result + 'C' + elif arg == 'byte': + result = result + 'B' + elif arg == 'short': + result = result + 'S' + elif arg == 'long': + result = result + 'J' + elif arg == 'double': + result = result + 'D' + elif arg == 'void': + result = result + 'V' + elif arg == 'String': + result = result + 'Ljava/lang/String;' + else: + temp = "L" + arg.replace('.', '/') + ';' + result = result + temp + + return result + def generate_prompts(self, prompts_file, class_name, methods, code): for method in methods: temp_method = method.split("(")[0] @@ -66,12 +141,8 @@ def generate_prompts(self, prompts_file, class_name, methods, code): modified_lines.append("import org.junit.runners.MethodSorters;\n\n") print(f"Generating prompt for method {method}") modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") - modified_lines.append(f"public class {class_name}Test {{\n") - modified_lines.append(f" public static boolean debug = false;\n\n") - modified_lines.append(f" @Test\n") - modified_lines.append(f" public void {temp_method}_001() throws Throwable {{\n") - modified_lines.append(f" if (debug) {{\n") - modified_lines.append(f" // test here\n") + modified_lines.append(f"public class {class_name.split('.')[-1]}Test {{\n") + modified_lines.append(f"//continue with code only:\n") modified_lines.append(f"\"\"\"") with open(prompts_file, "w") as file: @@ -113,7 +184,7 @@ def save_output(self, prompt, output, dir, output_file_name): if not os.path.exists(dir): os.makedirs(dir) - output_file_path = f"{dir}/{output_file_name}.txt" + output_file_path = f"{dir}/{output_file_name}.java" with open(output_file_path, "w") as f: f.write(prompt + output) @@ -154,9 +225,9 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(mpath, lpath, model, lib) for i, prompt in enumerate(prompts_list): - for j in range(0, 2): + for j in range(0, 10): print(f"----------------------------- Generating output {i}-{j}") - output_file_name = f"output{i}-{j}-{class_name}-{branch}" + output_file_name = f"{i}{j}-{class_name.replace('.', '_')}-{branch}" output = self.generate_output(cm, prompt) self.save_output(prompt, output, output_path, output_file_name) self.reset_chat_module(cm) \ No newline at end of file diff --git a/nimrod/tools/codellama.py b/nimrod/tools/codellama.py index 952607c1..1072a97e 100644 --- a/nimrod/tools/codellama.py +++ b/nimrod/tools/codellama.py @@ -9,4 +9,89 @@ def _get_tool_name(self): return "codellama" def _test_classes(self): - return ['RegressionTest', 'ErrorTest'] + classes = [] + + for class_file in sorted(get_class_files(self.suite_classes_dir)): + filename, _ = os.path.splitext(class_file) + if not filename.endswith('_scaffolding'): + classes.append(filename.replace(os.sep, '.')) + + return classes + + def _get_suite_dir(self): + return os.path.join(self.suite_dir, 'codellama-tests') + + def create_method_list(self, methods: "list[str]"): + rectified_methods = [self.convert_method_signature( + method) for method in methods] + return (":").join(rectified_methods) + + def convert_method_signature(self, meth_signature: str) -> str: + method_return = "" + try: + method_return = meth_signature.split(")")[1] + except Exception as e: + print(e) + meth_name = meth_signature[:meth_signature.rfind("(")] + meth_args = meth_signature[meth_signature.find( + "(") + 1:meth_signature.rfind(")")].split(",") + asm_meth_format = self.asm_based_method_method_descriptor( + meth_args, method_return) + + return meth_name+asm_meth_format + + # See at: https://asm.ow2.io/asm4-guide.pdf -- Section 2.1.3 and 2.1.4 + # Java type Type descriptor + # boolean Z + # char C + # byte B + # short S + # int I + # float F + # long J + # double D + # Object Ljava/lang/Object; + # int[] [I + # Object[][] [[Ljava/lang/Object; + def asm_based_method_method_descriptor(self, method_arguments, method_return): + result = '(' + for arg in method_arguments: + arg = arg.strip() + result = result + self._asm_based_type_descriptor(arg) + result = result + ')' + result = result + self._asm_based_type_descriptor(method_return) + return result + + def _asm_based_type_descriptor(self, arg): + result = '' + if '[]' in arg: + result = result + '[' + arg = arg.replace('[]', '') + + if arg == '': + result = result + '' + elif arg == 'int': + result = result + 'I' + elif arg == 'float': + result = result + 'F' + elif arg == 'boolean': + result = result + 'Z' + elif arg == 'char': + result = result + 'C' + elif arg == 'byte': + result = result + 'B' + elif arg == 'short': + result = result + 'S' + elif arg == 'long': + result = result + 'J' + elif arg == 'double': + result = result + 'D' + elif arg == 'void': + result = result + 'V' + elif arg == 'String': + result = result + 'Ljava/lang/String;' + else: + temp = "L" + arg.replace('.', '/') + ';' + result = result + temp + + return result \ No newline at end of file From e585dba5411b9b151d016d196de531dddb5bb7d8 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 28 May 2024 17:16:34 -0300 Subject: [PATCH 04/69] Add try except block on compile test suite method --- .../generators/test_suite_generator.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 6de93e15..2c1aae47 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -68,8 +68,12 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: def _compile_test_suite(self, input_jar: str, test_suite_path: str, extra_class_path: List[str] = []) -> str: compiled_classes_path = path.join(test_suite_path, 'classes') class_path = generate_classpath([input_jar, test_suite_path, compiled_classes_path, JUNIT, HAMCREST] + extra_class_path) - for java_file in self._get_test_suite_class_paths(test_suite_path): - self._java.exec_javac(java_file, test_suite_path, None, None, - '-classpath', class_path, '-d', compiled_classes_path) + print("java_file: ", java_file) + try: + self._java.exec_javac(java_file, test_suite_path, None, None, + '-classpath', class_path, '-d', compiled_classes_path) + except Exception as e: + logging.error(f"Error while compiling test suite class {java_file}:\n{e}\n") + pass return class_path From a5aeba5f984860feeb9420e9605eb19debbdc1fe Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 28 May 2024 17:17:39 -0300 Subject: [PATCH 05/69] Add some functions for better compilation work --- .../codellama_test_suite_generator.py | 82 ++++++++++++++++--- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 73c73f48..d21ede12 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -138,11 +138,11 @@ def generate_prompts(self, prompts_file, class_name, methods, code): modified_lines.append(f"/*{''.join(method_code)}*/\n\n") modified_lines.append("import org.junit.FixMethodOrder;\n") modified_lines.append("import org.junit.Test;\n") - modified_lines.append("import org.junit.runners.MethodSorters;\n\n") - print(f"Generating prompt for method {method}") + modified_lines.append("import org.junit.runners.MethodSorters;\n") + modified_lines.append("import static org.junit.Assert.*;\n\n") modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") modified_lines.append(f"public class {class_name.split('.')[-1]}Test {{\n") - modified_lines.append(f"//continue with code only:\n") + modified_lines.append(f"//continue the test with code only:\n") modified_lines.append(f"\"\"\"") with open(prompts_file, "w") as file: @@ -151,6 +151,20 @@ def generate_prompts(self, prompts_file, class_name, methods, code): except Exception as e: print(f"Error while generating prompt for method {method}: {e}") + def get_imports(self, code): + with open(code, 'r') as file: + lines = file.readlines() + + imports = [] + for line in lines: + if line.strip().startswith("import"): + imports.append(line) + elif line.strip().startswith("package"): + package_name = line.split()[1].rstrip(';') + imports.append(f'import {package_name}.*;\n') + + return imports + def read_prompts(self, file_path): prompts = [] current_prompt = "" @@ -201,6 +215,49 @@ def get_branch(self, input_jar, code_base, code_left, code_right, code_merge): def reset_chat_module(self, chat_module): chat_module.reset_chat() + def change_method_name(self, prompt, method_name): + new_prompt = prompt.split("public class")[0] + new_prompt += f"public class {method_name} {{\n" + return new_prompt + + def get_individual_tests(self, output_path, prompt, class_name, imports, i): + counter = 0 + + if not os.path.exists(f"{output_path}/individual_tests"): + os.makedirs(f"{output_path}/individual_tests") + + for file in os.listdir(output_path): + lines = [] + test = [] + open_brackets_count = 0 + test_found = False + + if file.endswith(".java") and file.startswith(f"{i}"): + with open(os.path.join(output_path, file), "r") as f: + lines.extend(f.readlines()) + + for j, line in enumerate(lines): + if "{" in line: + open_brackets_count += 1 + + if "}" in line: + open_brackets_count -= 1 + + if ("@Test" in line or "import" in line or "package" in line) and test_found: + test_found = False + method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" + with open(f"{output_path}/individual_tests/{method_name}.java", "w") as f: + full_prompt = "".join(imports) + self.change_method_name(prompt, method_name) + f.write(full_prompt + "".join(test) + open_brackets_count * "}") + counter += 1 + test = [] + + if "@Test" in line and not test_found: + test_found = True + + if test_found: + test.append(line) + def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: class_name, methods = list(scenario.targets.items())[0] @@ -216,18 +273,23 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" code, branch = self.get_branch(input_jar, code_base, code_left, code_right, code_merge) - print (f"Branch: {branch}") self.generate_prompts(prompts_path, class_name, methods, code) + imports = self.get_imports(code) prompts_list = self.read_prompts(prompts_path) cm = self.create_chat_module(mpath, lpath, model, lib) for i, prompt in enumerate(prompts_list): - for j in range(0, 10): - print(f"----------------------------- Generating output {i}-{j}") - output_file_name = f"{i}{j}-{class_name.replace('.', '_')}-{branch}" - output = self.generate_output(cm, prompt) - self.save_output(prompt, output, output_path, output_file_name) - self.reset_chat_module(cm) \ No newline at end of file + for j in range(0, 2): + try: + print(f"----------------------------- Generating output {i}{j} in branch \"{branch}\" -----------------------------") + output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" + output = self.generate_output(cm, prompt) + self.save_output(prompt, output, output_path, output_file_name) + self.reset_chat_module(cm) + except Exception as e: + print(f"Error while generating output {i}-{j} in branch {branch}: {e}") + pass + self.get_individual_tests(output_path, prompt, class_name, imports, i) \ No newline at end of file From cec679ae7fd1d8accb6314c2016f7b77b1268bac Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Wed, 29 May 2024 11:57:46 -0300 Subject: [PATCH 06/69] removed print --- .../test_suite_generation/generators/test_suite_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 2c1aae47..1a415ee5 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -69,11 +69,10 @@ def _compile_test_suite(self, input_jar: str, test_suite_path: str, extra_class_ compiled_classes_path = path.join(test_suite_path, 'classes') class_path = generate_classpath([input_jar, test_suite_path, compiled_classes_path, JUNIT, HAMCREST] + extra_class_path) for java_file in self._get_test_suite_class_paths(test_suite_path): - print("java_file: ", java_file) try: self._java.exec_javac(java_file, test_suite_path, None, None, '-classpath', class_path, '-d', compiled_classes_path) except Exception as e: - logging.error(f"Error while compiling test suite class {java_file}:\n{e}\n") + logging.error("Error while compiling test suite class %s: %s", java_file, e) pass return class_path From 4ee7db1c0438d8c69ee21e50397f194c2659ba7f Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Wed, 29 May 2024 11:58:26 -0300 Subject: [PATCH 07/69] added/fixed logging --- .../codellama_test_suite_generator.py | 40 +++++++------------ .../test_suite_executor.py | 5 ++- nimrod/tests/utils.py | 29 ++++++++++++-- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index d21ede12..54905063 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,4 +1,4 @@ -import os +import os, logging from typing import Dict, List from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis @@ -49,7 +49,7 @@ def _convert_method_signature(self, meth_signature: str) -> str: try: method_return = meth_signature.split(")")[1] except Exception as e: - print(e) + logging.error("Error while converting method signature: %s", e) meth_name = meth_signature[:meth_signature.rfind("(")] meth_args = meth_signature[meth_signature.find( "(") + 1:meth_signature.rfind(")")].split(",") @@ -58,19 +58,6 @@ def _convert_method_signature(self, meth_signature: str) -> str: return meth_name+asm_meth_format - # See at: https://asm.ow2.io/asm4-guide.pdf -- Section 2.1.3 and 2.1.4 - # Java type Type descriptor - # boolean Z - # char C - # byte B - # short S - # int I - # float F - # long J - # double D - # Object Ljava/lang/Object; - # int[] [I - # Object[][] [[Ljava/lang/Object; def _asm_based_method_method_descriptor(self, method_arguments, method_return): result = '(' for arg in method_arguments: @@ -149,7 +136,7 @@ def generate_prompts(self, prompts_file, class_name, methods, code): file.writelines(modified_lines) except Exception as e: - print(f"Error while generating prompt for method {method}: {e}") + logging.error("Error while generating prompt for method %s: %s", method, e) def get_imports(self, code): with open(code, 'r') as file: @@ -197,8 +184,10 @@ def save_output(self, prompt, output, dir, output_file_name): if dir: if not os.path.exists(dir): os.makedirs(dir) + if not os.path.exists(f"{dir}/llm_outputs"): + os.makedirs(f"{dir}/llm_outputs") - output_file_path = f"{dir}/{output_file_name}.java" + output_file_path = f"{dir}/llm_outputs/{output_file_name}.txt" with open(output_file_path, "w") as f: f.write(prompt + output) @@ -223,17 +212,16 @@ def change_method_name(self, prompt, method_name): def get_individual_tests(self, output_path, prompt, class_name, imports, i): counter = 0 - if not os.path.exists(f"{output_path}/individual_tests"): - os.makedirs(f"{output_path}/individual_tests") + llm_outputs_path = f"{output_path}/llm_outputs/" - for file in os.listdir(output_path): + for file in os.listdir(llm_outputs_path): lines = [] test = [] open_brackets_count = 0 test_found = False - if file.endswith(".java") and file.startswith(f"{i}"): - with open(os.path.join(output_path, file), "r") as f: + if file.endswith(".txt") and file.startswith(f"{i}"): + with open(os.path.join(llm_outputs_path, file), "r") as f: lines.extend(f.readlines()) for j, line in enumerate(lines): @@ -246,7 +234,7 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i): if ("@Test" in line or "import" in line or "package" in line) and test_found: test_found = False method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" - with open(f"{output_path}/individual_tests/{method_name}.java", "w") as f: + with open(f"{output_path}/{method_name}.java", "w") as f: full_prompt = "".join(imports) + self.change_method_name(prompt, method_name) f.write(full_prompt + "".join(test) + open_brackets_count * "}") counter += 1 @@ -282,14 +270,14 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(mpath, lpath, model, lib) for i, prompt in enumerate(prompts_list): - for j in range(0, 2): + for j in range(0, 10): try: - print(f"----------------------------- Generating output {i}{j} in branch \"{branch}\" -----------------------------") + logging.info("Generating output %d%d in branch \"%s\"", i, j, branch) output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" output = self.generate_output(cm, prompt) self.save_output(prompt, output, output_path, output_file_name) self.reset_chat_module(cm) except Exception as e: - print(f"Error while generating output {i}-{j} in branch {branch}: {e}") + logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) pass self.get_individual_tests(output_path, prompt, class_name, imports, i) \ No newline at end of file diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index e8176e19..d7666119 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -35,9 +35,12 @@ def execute_test_suite(self, test_suite: TestSuite, jar: str, number_of_executio results: Dict[str, TestCaseResult] = dict() for test_class in test_suite.test_classes_names: + logging.debug("Test class: %s", test_class) + for i in range(0, number_of_executions): - logging.debug("Starting execution %d of %s from suite %s", i + 1, test_class, test_suite.path) + logging.info("Starting execution %d of %s from suite %s", i + 1, test_class, test_suite.path) response = self._execute_junit(test_suite, jar, test_class) + logging.debug("response: %s", response) for test_case, test_case_result in response.items(): test_fqname = f"{test_class}#{test_case}" if results.get(test_fqname) and results.get(test_fqname) != test_case_result: diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index 0f199c3b..34dc13a8 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -78,12 +78,35 @@ def setup_logging(): config = get_config() config_level = config.get('logger_level') level = logging._nameToLevel[config_level] if config_level else logging.INFO - logging.basicConfig( - level=level, - format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s', + + # Obtém o logger raiz + logger = logging.getLogger() + logger.setLevel(level) + + # Remove handlers antigos se existirem para evitar duplicação + if logger.hasHandlers(): + logger.handlers.clear() + + # Configura o formato e handlers + formatter = logging.Formatter( + '%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) + if os.path.exists('logfile.log'): + os.remove('logfile.log') + + # File handler para salvar logs em arquivo + file_handler = logging.FileHandler('logfile.log') + file_handler.setFormatter(formatter) + + # Stream handler para exibir logs no terminal + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + + # Adiciona handlers ao logger + logger.addHandler(file_handler) + logger.addHandler(stream_handler) def get_base_output_path() -> str: return os.getcwd().replace("/nimrod/proj", "/")+'/output-test-dest/' if os.getcwd().__contains__("/nimrod/proj") else os.getcwd() + "/output-test-dest/" From aa84b018c94cc40dbc0b0fa5307b8bd761914710 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 13 Jun 2024 21:08:25 -0300 Subject: [PATCH 08/69] fix class and test naming --- .../codellama_test_suite_generator.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 54905063..a9b71ebf 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -33,10 +33,8 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: class_names = [] for class_path in self._get_test_suite_class_paths(test_suite_path): - class_fqcn = os.path.relpath(class_path, os.path.join( - test_suite_path, "codellama-tests")).replace(os.sep, ".") - class_names.append(class_fqcn[:-5]) - + class_fqcn = os.path.basename(class_path).replace(".java", "") + class_names.append(class_fqcn) return class_names def _create_method_list(self, methods: "List[str]"): @@ -209,7 +207,7 @@ def change_method_name(self, prompt, method_name): new_prompt += f"public class {method_name} {{\n" return new_prompt - def get_individual_tests(self, output_path, prompt, class_name, imports, i): + def get_individual_tests(self, output_path, prompt, class_name, imports, i, test_counter): counter = 0 llm_outputs_path = f"{output_path}/llm_outputs/" @@ -219,6 +217,7 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i): test = [] open_brackets_count = 0 test_found = False + test_signature = "" if file.endswith(".txt") and file.startswith(f"{i}"): with open(os.path.join(llm_outputs_path, file), "r") as f: @@ -242,6 +241,11 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i): if "@Test" in line and not test_found: test_found = True + test_signature = lines[j+1].strip() + test_counter += 1 + + if test_signature in line and test_found: + line = line.replace(test_signature, f"public void test{test_counter:03d}() {{") if test_found: test.append(line) @@ -269,6 +273,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(mpath, lpath, model, lib) + test_counter = 0 for i, prompt in enumerate(prompts_list): for j in range(0, 10): try: @@ -280,4 +285,4 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) pass - self.get_individual_tests(output_path, prompt, class_name, imports, i) \ No newline at end of file + self.get_individual_tests(output_path, prompt, class_name, imports, i, test_counter) \ No newline at end of file From ddf12e795bc610c2115bea215084941b51c472b6 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 13 Jun 2024 21:09:13 -0300 Subject: [PATCH 09/69] change test not found result --- nimrod/test_suites_execution/test_suite_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index d7666119..8a6e8ec5 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -91,7 +91,7 @@ def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResu for i in range(0, test_run_count): test_case_name = 'test{number:0{width}d}'.format(width=len(str(test_run_count)), number=i) if not results.get(test_case_name): - results[test_case_name] = TestCaseResult.PASS + results[test_case_name] = TestCaseResult.NOT_EXECUTABLE return results From b5c8f1b565cc27eef42737720fdeb2b0ae38fc6c Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 13 Jun 2024 23:45:13 -0300 Subject: [PATCH 10/69] removed unused functions and cleaned the code --- .../codellama_test_suite_generator.py | 95 ++++--------------- nimrod/tools/codellama.py | 84 +--------------- 2 files changed, 21 insertions(+), 158 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index a9b71ebf..15280d3c 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -11,8 +11,6 @@ class CodellamaTestSuiteGenerator(TestSuiteGenerator): - TARGET_METHODS_LIST_FILENAME = 'methods_to_test.txt' - TARGET_CLASS_LIST_FILENAME = 'classes_to_test.txt' def get_generator_tool_name(self) -> str: return "CODELLAMA" @@ -25,7 +23,7 @@ def _get_test_suite_class_paths(self, path: str) -> List[str]: paths += self._get_test_suite_class_paths(os.path.join(path, node)) elif node.endswith(".java"): paths.append(os.path.join(path, node)) - + return paths @@ -35,69 +33,9 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: for class_path in self._get_test_suite_class_paths(test_suite_path): class_fqcn = os.path.basename(class_path).replace(".java", "") class_names.append(class_fqcn) + return class_names - def _create_method_list(self, methods: "List[str]"): - rectified_methods = [self._convert_method_signature( - method) for method in methods] - return (":").join(rectified_methods) - - def _convert_method_signature(self, meth_signature: str) -> str: - method_return = "" - try: - method_return = meth_signature.split(")")[1] - except Exception as e: - logging.error("Error while converting method signature: %s", e) - meth_name = meth_signature[:meth_signature.rfind("(")] - meth_args = meth_signature[meth_signature.find( - "(") + 1:meth_signature.rfind(")")].split(",") - asm_meth_format = self._asm_based_method_method_descriptor( - meth_args, method_return) - - return meth_name+asm_meth_format - - def _asm_based_method_method_descriptor(self, method_arguments, method_return): - result = '(' - for arg in method_arguments: - arg = arg.strip() - result = result + self._asm_based_type_descriptor(arg) - result = result + ')' - result = result + self._asm_based_type_descriptor(method_return) - return result - - def _asm_based_type_descriptor(self, arg): - result = '' - if '[]' in arg: - result = result + '[' - arg = arg.replace('[]', '') - - if arg == '': - result = result + '' - elif arg == 'int': - result = result + 'I' - elif arg == 'float': - result = result + 'F' - elif arg == 'boolean': - result = result + 'Z' - elif arg == 'char': - result = result + 'C' - elif arg == 'byte': - result = result + 'B' - elif arg == 'short': - result = result + 'S' - elif arg == 'long': - result = result + 'J' - elif arg == 'double': - result = result + 'D' - elif arg == 'void': - result = result + 'V' - elif arg == 'String': - result = result + 'Ljava/lang/String;' - else: - temp = "L" + arg.replace('.', '/') + ';' - result = result + temp - - return result def generate_prompts(self, prompts_file, class_name, methods, code): for method in methods: @@ -136,6 +74,7 @@ def generate_prompts(self, prompts_file, class_name, methods, code): except Exception as e: logging.error("Error while generating prompt for method %s: %s", method, e) + def get_imports(self, code): with open(code, 'r') as file: lines = file.readlines() @@ -150,6 +89,7 @@ def get_imports(self, code): return imports + def read_prompts(self, file_path): prompts = [] current_prompt = "" @@ -167,6 +107,7 @@ def read_prompts(self, file_path): return prompts + def create_chat_module(self, mpath, lpath, model, lib): return ChatModule( model=f"{mpath}/{model}", @@ -174,6 +115,10 @@ def create_chat_module(self, mpath, lpath, model, lib): ) + def reset_chat_module(self, chat_module): + chat_module.reset_chat() + + def generate_output(self, chat_module, prompt): return chat_module.generate(prompt=prompt) @@ -189,6 +134,7 @@ def save_output(self, prompt, output, dir, output_file_name): with open(output_file_path, "w") as f: f.write(prompt + output) + def get_branch(self, input_jar, code_base, code_left, code_right, code_merge): if 'base' in input_jar: return code_base, "base" @@ -199,15 +145,8 @@ def get_branch(self, input_jar, code_base, code_left, code_right, code_merge): if 'merge' in input_jar: return code_merge, "merge" - def reset_chat_module(self, chat_module): - chat_module.reset_chat() - - def change_method_name(self, prompt, method_name): - new_prompt = prompt.split("public class")[0] - new_prompt += f"public class {method_name} {{\n" - return new_prompt - def get_individual_tests(self, output_path, prompt, class_name, imports, i, test_counter): + def get_individual_tests(self, output_path, prompt, class_name, imports, i): counter = 0 llm_outputs_path = f"{output_path}/llm_outputs/" @@ -234,7 +173,9 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i, test test_found = False method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" with open(f"{output_path}/{method_name}.java", "w") as f: - full_prompt = "".join(imports) + self.change_method_name(prompt, method_name) + new_prompt = prompt.split("public class")[0] + new_prompt += f"public class {method_name} {{\n" + full_prompt = "".join(imports) + new_prompt f.write(full_prompt + "".join(test) + open_brackets_count * "}") counter += 1 test = [] @@ -242,10 +183,9 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i, test if "@Test" in line and not test_found: test_found = True test_signature = lines[j+1].strip() - test_counter += 1 if test_signature in line and test_found: - line = line.replace(test_signature, f"public void test{test_counter:03d}() {{") + line = line.replace(test_signature, f"public void test{i}{counter}() {{") if test_found: test.append(line) @@ -273,11 +213,10 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(mpath, lpath, model, lib) - test_counter = 0 for i, prompt in enumerate(prompts_list): for j in range(0, 10): try: - logging.info("Generating output %d%d in branch \"%s\"", i, j, branch) + logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" output = self.generate_output(cm, prompt) self.save_output(prompt, output, output_path, output_file_name) @@ -285,4 +224,4 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) pass - self.get_individual_tests(output_path, prompt, class_name, imports, i, test_counter) \ No newline at end of file + self.get_individual_tests(output_path, prompt, class_name, imports, i) \ No newline at end of file diff --git a/nimrod/tools/codellama.py b/nimrod/tools/codellama.py index 1072a97e..c6021b6b 100644 --- a/nimrod/tools/codellama.py +++ b/nimrod/tools/codellama.py @@ -1,7 +1,7 @@ import os from nimrod.tools.suite_generator import Suite, SuiteGenerator -from nimrod.utils import generate_classpath +from nimrod.utils import get_class_files class Codellama(SuiteGenerator): @@ -13,85 +13,9 @@ def _test_classes(self): for class_file in sorted(get_class_files(self.suite_classes_dir)): filename, _ = os.path.splitext(class_file) - if not filename.endswith('_scaffolding'): - classes.append(filename.replace(os.sep, '.')) + classes.append(filename.replace(os.sep, '.')) return classes - def _get_suite_dir(self): - return os.path.join(self.suite_dir, 'codellama-tests') - - def create_method_list(self, methods: "list[str]"): - rectified_methods = [self.convert_method_signature( - method) for method in methods] - return (":").join(rectified_methods) - - def convert_method_signature(self, meth_signature: str) -> str: - method_return = "" - try: - method_return = meth_signature.split(")")[1] - except Exception as e: - print(e) - meth_name = meth_signature[:meth_signature.rfind("(")] - meth_args = meth_signature[meth_signature.find( - "(") + 1:meth_signature.rfind(")")].split(",") - asm_meth_format = self.asm_based_method_method_descriptor( - meth_args, method_return) - - return meth_name+asm_meth_format - - # See at: https://asm.ow2.io/asm4-guide.pdf -- Section 2.1.3 and 2.1.4 - # Java type Type descriptor - # boolean Z - # char C - # byte B - # short S - # int I - # float F - # long J - # double D - # Object Ljava/lang/Object; - # int[] [I - # Object[][] [[Ljava/lang/Object; - def asm_based_method_method_descriptor(self, method_arguments, method_return): - result = '(' - for arg in method_arguments: - arg = arg.strip() - result = result + self._asm_based_type_descriptor(arg) - result = result + ')' - result = result + self._asm_based_type_descriptor(method_return) - return result - - def _asm_based_type_descriptor(self, arg): - result = '' - if '[]' in arg: - result = result + '[' - arg = arg.replace('[]', '') - - if arg == '': - result = result + '' - elif arg == 'int': - result = result + 'I' - elif arg == 'float': - result = result + 'F' - elif arg == 'boolean': - result = result + 'Z' - elif arg == 'char': - result = result + 'C' - elif arg == 'byte': - result = result + 'B' - elif arg == 'short': - result = result + 'S' - elif arg == 'long': - result = result + 'J' - elif arg == 'double': - result = result + 'D' - elif arg == 'void': - result = result + 'V' - elif arg == 'String': - result = result + 'Ljava/lang/String;' - else: - temp = "L" + arg.replace('.', '/') + ';' - result = result + temp - - return result \ No newline at end of file + def _get_suite_dir(self): + return os.path.join(self.suite_dir, 'codellama-tests') \ No newline at end of file From 655d95ddb90d525fe95da253ef1fa8c0aa5ed440 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 13 Jun 2024 23:46:04 -0300 Subject: [PATCH 11/69] simply not include not found tests --- nimrod/test_suites_execution/test_suite_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 8a6e8ec5..79e60801 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -40,7 +40,7 @@ def execute_test_suite(self, test_suite: TestSuite, jar: str, number_of_executio for i in range(0, number_of_executions): logging.info("Starting execution %d of %s from suite %s", i + 1, test_class, test_suite.path) response = self._execute_junit(test_suite, jar, test_class) - logging.debug("response: %s", response) + logging.debug("RESULTS: %s", response) for test_case, test_case_result in response.items(): test_fqname = f"{test_class}#{test_case}" if results.get(test_fqname) and results.get(test_fqname) != test_case_result: @@ -91,7 +91,7 @@ def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResu for i in range(0, test_run_count): test_case_name = 'test{number:0{width}d}'.format(width=len(str(test_run_count)), number=i) if not results.get(test_case_name): - results[test_case_name] = TestCaseResult.NOT_EXECUTABLE + pass return results From f4a6e29c6bc3c5271e6312f15fccaa936a2d8dd9 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 13 Jun 2024 23:50:19 -0300 Subject: [PATCH 12/69] remove unused imports --- .../generators/codellama_test_suite_generator.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 15280d3c..6be655e9 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,15 +1,11 @@ import os, logging -from typing import Dict, List +from typing import List from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis - from nimrod.test_suite_generation.generators.test_suite_generator import \ TestSuiteGenerator -from nimrod.tools.java import Java -from nimrod.utils import generate_classpath from mlc_chat import ChatModule - class CodellamaTestSuiteGenerator(TestSuiteGenerator): def get_generator_tool_name(self) -> str: From 1d427f11c165e780ae688511e1cb0c1a62127945 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Fri, 21 Jun 2024 12:22:25 -0300 Subject: [PATCH 13/69] varias coisa --- .../codellama_test_suite_generator.py | 120 +++++++++++++----- .../test_suite_executor.py | 10 +- 2 files changed, 96 insertions(+), 34 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 6be655e9..d3002575 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -33,34 +33,76 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return class_names + def get_method_code(self, file_path, method_name): + with open(file_path, 'r') as file: + lines = file.readlines() + + method_found = False + method_code = [] + class_attributes = [] + current_method = [] + other_methods = [] + search_other_methods = False + + for line in lines: + stripped_line = line.strip() + + # Check for class attributes + if ';' in stripped_line and ('private' in stripped_line or 'public' in stripped_line): + class_attributes.append(line) + + # Check for method definition + if stripped_line.startswith(('private', 'public')) and method_name.split('(')[0] in stripped_line: + method_found = True + current_method = [line] + elif method_found and stripped_line.startswith(('private', 'public')): + break + elif method_found: + current_method.append(line) + + if method_found: + method_code = current_method + + if search_other_methods: + for line in method_code: + try: + method_call = line.split('.')[1].strip() + if '(' in line and ')' in line and '=' not in line and ';' in line and method_name not in line: + method_call = method_call.split(';')[0] + other_methods.append(method_call) + except IndexError: + continue + + return method_found, class_attributes, method_code, other_methods + + def generate_prompts(self, prompts_file, class_name, methods, code): for method in methods: - temp_method = method.split("(")[0] - try: - with open(code, 'r') as file: - lines = file.readlines() - - method_found = False - method_code = [] - for line in lines: - if temp_method in line and ("private" in line or "public" in line): - method_found = True - elif temp_method not in line and ("private" in line or "public" in line): - if method_found: - break - if method_found: - method_code.append(line) + method_found, class_attributes, method_code, other_methods = self.get_method_code(code, method) + other_method_codes = [] if method_found: + if other_methods: + for other_method in other_methods: + other_method_found, ca, other_method_code, om = self.get_method_code(code, other_method) + + if other_method_found: + other_method_codes.append(other_method_code) + modified_lines = [] - modified_lines.append(f"/*{''.join(method_code)}*/\n\n") + modified_lines.append(f"/*\n{''.join(class_attributes)}\n{''.join(method_code)}") + if other_methods: + modified_lines.append("\n") + for i in range(len(other_method_codes)): + modified_lines.append(f"{''.join(other_method_codes[i])}") + modified_lines.append("*/\n\n") modified_lines.append("import org.junit.FixMethodOrder;\n") modified_lines.append("import org.junit.Test;\n") modified_lines.append("import org.junit.runners.MethodSorters;\n") modified_lines.append("import static org.junit.Assert.*;\n\n") modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") - modified_lines.append(f"public class {class_name.split('.')[-1]}Test {{\n") + modified_lines.append(f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n") modified_lines.append(f"//continue the test with code only:\n") modified_lines.append(f"\"\"\"") @@ -150,8 +192,10 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i): for file in os.listdir(llm_outputs_path): lines = [] test = [] - open_brackets_count = 0 + before = [] + open_brackets_count = 1 test_found = False + before_found = False test_signature = "" if file.endswith(".txt") and file.startswith(f"{i}"): @@ -159,31 +203,42 @@ def get_individual_tests(self, output_path, prompt, class_name, imports, i): lines.extend(f.readlines()) for j, line in enumerate(lines): - if "{" in line: - open_brackets_count += 1 - - if "}" in line: - open_brackets_count -= 1 - - if ("@Test" in line or "import" in line or "package" in line) and test_found: + if ("@Test" in line or "import" in line or "package" in line or not open_brackets_count) and test_found: test_found = False method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" with open(f"{output_path}/{method_name}.java", "w") as f: new_prompt = prompt.split("public class")[0] new_prompt += f"public class {method_name} {{\n" full_prompt = "".join(imports) + new_prompt + if before: + full_prompt += "".join(before) f.write(full_prompt + "".join(test) + open_brackets_count * "}") counter += 1 test = [] if "@Test" in line and not test_found: test_found = True - test_signature = lines[j+1].strip() + before_found = False + if lines[j+1]: + test_signature = lines[j+1].strip() + + if before_found: + before.append(line) - if test_signature in line and test_found: - line = line.replace(test_signature, f"public void test{i}{counter}() {{") + if ("@Before" in line and not before_found): + before_found = True + before.append(line) if test_found: + if test_signature in line: + line = line.replace(test_signature, f"public void test{i}{counter}() {{") + + if "{" in line: + open_brackets_count += 1 + + if "}" in line: + open_brackets_count -= 1 + test.append(line) def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: @@ -200,6 +255,13 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s code_right = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_right.java" code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" + """ + code_base = f"/mnt/c/Users/natha/Downloads/smat/base.java" + code_left = f"/mnt/c/Users/natha/Downloads/smat/left.java" + code_right = f"/mnt/c/Users/natha/Downloads/smat/right.java" + code_merge = f"/mnt/c/Users/natha/Downloads/smat/merge.java" + """ + code, branch = self.get_branch(input_jar, code_base, code_left, code_right, code_merge) self.generate_prompts(prompts_path, class_name, methods, code) @@ -210,7 +272,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(mpath, lpath, model, lib) for i, prompt in enumerate(prompts_list): - for j in range(0, 10): + for j in range(0, 5): try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 79e60801..c1cb0023 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -88,11 +88,11 @@ def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResu test_run_count = 0 if tests_run: test_run_count = int(tests_run.group('tests_run_count')) - for i in range(0, test_run_count): - test_case_name = 'test{number:0{width}d}'.format(width=len(str(test_run_count)), number=i) - if not results.get(test_case_name): - pass - + if results: + for i in range(0, test_run_count): + test_case_name = 'test{number:0{width}d}'.format(width=len(str(test_run_count)), number=i) + if not results.get(test_case_name): + results[test_case_name] = TestCaseResult.PASS return results def execute_test_suite_with_coverage(self, test_suite: TestSuite, target_jar: str, test_cases: List[str]) -> str: From 4db60de37016bee1d645b6ac204f53d01bb0c365 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 20 Aug 2024 22:28:45 -0300 Subject: [PATCH 14/69] calledprocesserror --- .../generators/test_suite_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 1a415ee5..72adc990 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -3,6 +3,7 @@ from os import makedirs, path from time import time from typing import List +from subprocess import CalledProcessError from nimrod.tests.utils import get_config from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis @@ -72,7 +73,6 @@ def _compile_test_suite(self, input_jar: str, test_suite_path: str, extra_class_ try: self._java.exec_javac(java_file, test_suite_path, None, None, '-classpath', class_path, '-d', compiled_classes_path) - except Exception as e: - logging.error("Error while compiling test suite class %s: %s", java_file, e) - pass + except CalledProcessError: + logging.error("Error while compiling %s", java_file) return class_path From e28295d31a47dfcc051efbff1066b38685fc8ed7 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 20 Aug 2024 22:28:57 -0300 Subject: [PATCH 15/69] update requirements --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 91bbd0f5..33739f17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,5 @@ -BeautifulSoup4 +beautifulsoup4==4.12.3 +mlc_chat_nightly_cu122==0.1.dev962 +setuptools==68.2.2 +tree_sitter==0.22.3 +tree_sitter_java==0.21.0 From 8a7c153161c38de76635100b50bdf090c9450329 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 20 Aug 2024 22:31:08 -0300 Subject: [PATCH 16/69] =?UTF-8?q?varias=20refatora=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codellama_test_suite_generator.py | 385 +++++++++++------- 1 file changed, 246 insertions(+), 139 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index d3002575..e14852bf 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,5 +1,7 @@ import os, logging -from typing import List +from typing import List, Generator +import tree_sitter_java as tsjava +from tree_sitter import Language, Parser, Node, Tree from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import \ TestSuiteGenerator @@ -33,81 +35,110 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return class_names - def get_method_code(self, file_path, method_name): - with open(file_path, 'r') as file: - lines = file.readlines() + def get_method_code(self, file_path, method_name, full_class_name): + JAVA_LANGUAGE = Language(tsjava.language()) + parser = Parser(JAVA_LANGUAGE) + + class_name = full_class_name.split('.')[-1] + source_code = open(file_path).read() + tree = parser.parse(bytes(source_code, "utf8")) + + def traverse_tree(tree: Tree) -> Generator[Node, None, None]: + cursor = tree.walk() + visited_children = False + + while True: + if not visited_children: + yield cursor.node + visited_children = not cursor.goto_first_child() + elif cursor.goto_next_sibling(): + visited_children = False + elif not cursor.goto_parent(): + break + + def get_snippet(start, end): + return '\n'.join([line[start.column:] if i == start.row else line[:end.column] if i == end.row else line + for i, line in enumerate(source_code.splitlines()) + if start.row <= i <= end.row]) + + def get_class_node(tree: Tree) -> list: + classes = [node for node in traverse_tree(tree) if node.type == 'class_declaration'] + for class_node in classes: + for child in class_node.children: + if child.type == 'identifier': + start = child.start_point + end = child.end_point + child_class_name = get_snippet(start, end).split()[0] + if child_class_name == class_name: + return class_node + return None + + def get_class_attributes_nodes(class_node: Node) -> list: + for child in class_node.children: + if child.type == 'class_body': + attributes = [node for node in child.children if node.type == 'field_declaration'] + break + return attributes if attributes else None + + def get_constructor_nodes(class_node: Node) -> list: + for child in class_node.children: + if child.type == 'class_body': + constructors = [node for node in child.children if node.type == 'constructor_declaration'] + break + return constructors if constructors else None + + def get_method_node(class_node: Node) -> list: + for child in class_node.children: + if child.type == 'class_body': + methods = [node for node in child.children if node.type == 'method_declaration'] + break + + for method in methods: + for child in method.children: + if child.type == 'identifier': + start = child.start_point + end = child.end_point + child_method_name = get_snippet(start, end).split()[0] + if child_method_name == method_name: + return method + return None + + class_node = get_class_node(tree) + class_attributes = [get_snippet(attribute.start_point, attribute.end_point) for attribute in get_class_attributes_nodes(class_node)] + constructor_codes = [get_snippet(constructor.start_point, constructor.end_point) for constructor in get_constructor_nodes(class_node)] + method_node = get_method_node(class_node) + method_code = get_snippet(method_node.start_point, method_node.end_point) - method_found = False - method_code = [] - class_attributes = [] - current_method = [] - other_methods = [] - search_other_methods = False - - for line in lines: - stripped_line = line.strip() - - # Check for class attributes - if ';' in stripped_line and ('private' in stripped_line or 'public' in stripped_line): - class_attributes.append(line) - - # Check for method definition - if stripped_line.startswith(('private', 'public')) and method_name.split('(')[0] in stripped_line: - method_found = True - current_method = [line] - elif method_found and stripped_line.startswith(('private', 'public')): - break - elif method_found: - current_method.append(line) - - if method_found: - method_code = current_method - - if search_other_methods: - for line in method_code: - try: - method_call = line.split('.')[1].strip() - if '(' in line and ')' in line and '=' not in line and ';' in line and method_name not in line: - method_call = method_call.split(';')[0] - other_methods.append(method_call) - except IndexError: - continue - - return method_found, class_attributes, method_code, other_methods + return class_attributes, constructor_codes, method_code def generate_prompts(self, prompts_file, class_name, methods, code): for method in methods: try: - method_found, class_attributes, method_code, other_methods = self.get_method_code(code, method) - - other_method_codes = [] - if method_found: - if other_methods: - for other_method in other_methods: - other_method_found, ca, other_method_code, om = self.get_method_code(code, other_method) - - if other_method_found: - other_method_codes.append(other_method_code) - - modified_lines = [] - modified_lines.append(f"/*\n{''.join(class_attributes)}\n{''.join(method_code)}") - if other_methods: - modified_lines.append("\n") - for i in range(len(other_method_codes)): - modified_lines.append(f"{''.join(other_method_codes[i])}") - modified_lines.append("*/\n\n") - modified_lines.append("import org.junit.FixMethodOrder;\n") - modified_lines.append("import org.junit.Test;\n") - modified_lines.append("import org.junit.runners.MethodSorters;\n") - modified_lines.append("import static org.junit.Assert.*;\n\n") - modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") - modified_lines.append(f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n") - modified_lines.append(f"//continue the test with code only:\n") - modified_lines.append(f"\"\"\"") + class_attributes, constructor_codes, method_code = self.get_method_code(code, method, class_name) + + modified_lines = [] + class_attributes_str = '\n'.join(class_attributes) + method_code_str = ''.join(method_code) + + if constructor_codes: + constructor_code_str = '\n'.join(constructor_codes) + modified_lines.append(f"/*\n{class_attributes_str}\n\n{constructor_code_str}\n\n{method_code_str}") + else: + modified_lines.append(f"/*\n{class_attributes_str}\n\n{method_code_str}") + + modified_lines.append("\n*/\n\n") + modified_lines.append("import org.junit.FixMethodOrder;\n") + modified_lines.append("import org.junit.Test;\n") + modified_lines.append("import org.junit.runners.MethodSorters;\n") + modified_lines.append("import static org.junit.Assert.*;\n\n") + modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") + modified_lines.append(f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n") + modified_lines.append(f"//continue the test with code only:\n") + modified_lines.append(f"\"\"\"") - with open(prompts_file, "w") as file: - file.writelines(modified_lines) + with open(prompts_file, "w") as file: + file.writelines(modified_lines) except Exception as e: logging.error("Error while generating prompt for method %s: %s", method, e) @@ -173,73 +204,161 @@ def save_output(self, prompt, output, dir, output_file_name): f.write(prompt + output) - def get_branch(self, input_jar, code_base, code_left, code_right, code_merge): - if 'base' in input_jar: - return code_base, "base" - if 'left' in input_jar: - return code_left, "left" - if 'right' in input_jar: - return code_right, "right" - if 'merge' in input_jar: - return code_merge, "merge" + def get_branch(self, input_jar, code_paths): + branches = ["base", "left", "right", "merge"] + for branch in branches: + if branch in input_jar: + return code_paths[branch], branch + raise ValueError(f"Nenhuma correspondência de branch encontrada no caminho: {input_jar}") def get_individual_tests(self, output_path, prompt, class_name, imports, i): - counter = 0 - llm_outputs_path = f"{output_path}/llm_outputs/" - for file in os.listdir(llm_outputs_path): - lines = [] - test = [] - before = [] - open_brackets_count = 1 - test_found = False - before_found = False - test_signature = "" + JAVA_LANGUAGE = Language(tsjava.language()) + parser = Parser(JAVA_LANGUAGE) + for file in os.listdir(llm_outputs_path): if file.endswith(".txt") and file.startswith(f"{i}"): - with open(os.path.join(llm_outputs_path, file), "r") as f: - lines.extend(f.readlines()) - - for j, line in enumerate(lines): - if ("@Test" in line or "import" in line or "package" in line or not open_brackets_count) and test_found: - test_found = False - method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" - with open(f"{output_path}/{method_name}.java", "w") as f: + source_code = open(os.path.join(llm_outputs_path, file)).read() + tree = parser.parse(bytes(source_code, "utf8")) + + def traverse_tree(tree: Tree) -> Generator[Node, None, None]: + cursor = tree.walk() + visited_children = False + + while True: + if not visited_children: + yield cursor.node + visited_children = not cursor.goto_first_child() + elif cursor.goto_next_sibling(): + visited_children = False + elif not cursor.goto_parent(): + break + + def collect_methods(tree: Tree) -> list: + methods = [node for node in traverse_tree(tree) if node.type == 'method_declaration'] + return methods + + def has_missing_brace(body: Node): + if body.children[-1].start_point.column == body.children[-1].end_point.column: + return True + return False + + def separate_tests(methods): + before_block = [] + test_block = [] + + def get_snippet(start, end): + return '\n'.join([line[start.column:] if i == start.row else line[:end.column] if i == end.row else line + for i, line in enumerate(source_code.splitlines()) + if start.row <= i <= end.row]) + + for method in methods: + children = method.children + for child in children: + if child.type == 'modifiers': + if child.children[0].type == 'marker_annotation': + start = child.children[0].start_point + end = child.children[0].end_point + + annotation_snippet = get_snippet(start, end) + + if annotation_snippet in ["@Before", "@BeforeEach", "@BeforeAll", "@BeforeClass"]: + start = method.start_point + end = method.end_point + before_block.append({"snippet": get_snippet(start, end), "missing_braces": has_missing_brace(method.children[-1])}) + + elif annotation_snippet == "@Test": + start = method.start_point + end = method.end_point + test_block.append({"snippet": get_snippet(start, end), "missing_braces": has_missing_brace(method.children[-1])}) + + print(before_block) + print(test_block) + return before_block, test_block + + methods = collect_methods(tree) + before_block, test_block = separate_tests(methods) + + for k, test in enumerate(test_block): + method_name = f"{class_name.split('.')[-1]}Test_{i}_{k}" + file_path = f"{output_path}/{method_name}.java" + + with open(file_path, "w") as f: new_prompt = prompt.split("public class")[0] new_prompt += f"public class {method_name} {{\n" full_prompt = "".join(imports) + new_prompt - if before: - full_prompt += "".join(before) - f.write(full_prompt + "".join(test) + open_brackets_count * "}") - counter += 1 - test = [] - - if "@Test" in line and not test_found: - test_found = True - before_found = False - if lines[j+1]: - test_signature = lines[j+1].strip() - - if before_found: - before.append(line) - - if ("@Before" in line and not before_found): - before_found = True - before.append(line) - - if test_found: - if test_signature in line: - line = line.replace(test_signature, f"public void test{i}{counter}() {{") - - if "{" in line: - open_brackets_count += 1 - - if "}" in line: - open_brackets_count -= 1 + + if before_block: + for before in before_block: + full_prompt += "".join(before['snippet']) + + snippet = test['snippet'] + test_signature = snippet.split("{")[0].strip() + new_signature = f"public void test{i}{k}()" + new_snippet = snippet.replace(test_signature, new_signature, 1) + + f.write(full_prompt + new_snippet) + + if test['missing_braces']: + f.write("}") + f.write("}\n") + + + def find_source_code_paths(self, jar_path, class_name): + return {"base": "/mnt/c/Users/natha/Downloads/smat/base.java", + "left": "/mnt/c/Users/natha/Downloads/smat/left.java", + "right": "/mnt/c/Users/natha/Downloads/smat/right.java", + "merge": "/mnt/c/Users/natha/Downloads/smat/merge.java"} + # Dividir o caminho em partes + path_parts = jar_path.split(os.sep) + + # Verificar se a estrutura do caminho é a esperada + if "transformed" not in path_parts: + raise ValueError("O caminho fornecido não contém a parte 'transformed'.") + + # Encontrar o índice da parte "transformed" + transformed_index = path_parts.index("transformed") + + # Substituir "transformed" por "source" e remover todas as partes subsequentes + source_path_parts = path_parts[:transformed_index] + ["source"] + + # Construir o caminho base + base_path = os.path.join("/", *source_path_parts) - test.append(line) + # Verificar se o diretório base existe antes de procurar + if not os.path.exists(base_path): + raise FileNotFoundError(f"O diretório base não existe: {base_path}") + + # Inicializar dicionário para armazenar caminhos de arquivos + java_files = {"base": "", "left": "", "right": "", "merge": ""} + + # Procurar por arquivos .java recursivamente + for root, dirs, files in os.walk(base_path): + # Verificar se a pasta com o nome da classe existe e dar preferência aos arquivos dentro dela + if os.path.basename(root) == class_name: + for file in files: + if file.endswith(".java"): + file_key = file.replace(".java", "") + if file_key in java_files: + java_files[file_key] = os.path.join(root, file) + + # Verificar arquivos fora da pasta com o nome da classe + for file in files: + if file.endswith(".java"): + file_key = file.replace(".java", "") + if file_key in java_files and not java_files[file_key]: + java_files[file_key] = os.path.join(root, file) + if all(java_files.values()): + break + + # Verifica se todos os arquivos foram encontrados, caso contrário, lança um erro + missing_files = [key for key, value in java_files.items() if not value] + if missing_files: + raise FileNotFoundError(f"Os seguintes arquivos não foram encontrados: {', '.join(missing_files)}") + + return java_files def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: class_name, methods = list(scenario.targets.items())[0] @@ -249,20 +368,8 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s model = "CodeLlama-7b-Instruct-hf-q4f16_1-MLC" lib = "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" prompts_path = f"{output_path}/prompts.txt" - - code_base = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_base.java" - code_left = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_left.java" - code_right = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_right.java" - code_merge = f"/mnt/c/Users/natha/Downloads/mergedataset/mergedataset/spring-boot/ea8107b6a53fa60b5f23b33e1b6d2e88bb60133c/source/UndertowEmbeddedServletContainerFactory_merge.java" - - """ - code_base = f"/mnt/c/Users/natha/Downloads/smat/base.java" - code_left = f"/mnt/c/Users/natha/Downloads/smat/left.java" - code_right = f"/mnt/c/Users/natha/Downloads/smat/right.java" - code_merge = f"/mnt/c/Users/natha/Downloads/smat/merge.java" - """ - - code, branch = self.get_branch(input_jar, code_base, code_left, code_right, code_merge) + code_paths = self.find_source_code_paths(input_jar, class_name.split('.')[-1]) + code, branch = self.get_branch(input_jar, code_paths) self.generate_prompts(prompts_path, class_name, methods, code) imports = self.get_imports(code) From 973adc346956a030b65d8ddbcd46daa226361c10 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 27 Aug 2024 17:37:00 -0300 Subject: [PATCH 17/69] =?UTF-8?q?=C3=A9=20o=20refatoras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codellama_test_suite_generator.py | 467 ++++++++---------- 1 file changed, 199 insertions(+), 268 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index e14852bf..7974679f 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,312 +1,224 @@ -import os, logging -from typing import List, Generator +import json +import logging +import os +from typing import List + import tree_sitter_java as tsjava -from tree_sitter import Language, Parser, Node, Tree -from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis -from nimrod.test_suite_generation.generators.test_suite_generator import \ - TestSuiteGenerator +from tree_sitter import Language, Parser from mlc_chat import ChatModule +from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis +from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator class CodellamaTestSuiteGenerator(TestSuiteGenerator): def get_generator_tool_name(self) -> str: return "CODELLAMA" + def _get_test_suite_class_paths(self, path: str) -> List[str]: paths = [] - - for node in os.listdir(path): - if os.path.isdir(os.path.join(path, node)): - paths += self._get_test_suite_class_paths(os.path.join(path, node)) - elif node.endswith(".java"): - paths.append(os.path.join(path, node)) - + for root, _, files in os.walk(path): + paths.extend(os.path.join(root, file) for file in files if file.endswith(".java")) return paths def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: - class_names = [] + return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] + - for class_path in self._get_test_suite_class_paths(test_suite_path): - class_fqcn = os.path.basename(class_path).replace(".java", "") - class_names.append(class_fqcn) + def create_chat_module(self, mpath: str, lpath: str, model: str, lib: str) -> ChatModule: + return ChatModule( + model=f"{mpath}/{model}", + model_lib_path=f"{lpath}/{lib}", + ) + + + def reset_chat_module(self, chat_module: ChatModule) -> None: + chat_module.reset_chat() + + + def generate_output(self, chat_module: ChatModule, prompt: str) -> str: + return chat_module.generate(prompt=prompt) - return class_names + def save_output(self, prompt: str, output: str, dir: str, output_file_name: str) -> None: + llm_outputs_dir = os.path.join(dir, "llm_outputs") + output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") + os.makedirs(llm_outputs_dir, exist_ok=True) + with open(output_file_path, "w") as file: + file.write(prompt + output) - def get_method_code(self, file_path, method_name, full_class_name): + + def parse_code(self, file_path: str) -> tuple: JAVA_LANGUAGE = Language(tsjava.language()) parser = Parser(JAVA_LANGUAGE) - - class_name = full_class_name.split('.')[-1] - source_code = open(file_path).read() + with open(file_path, 'r') as f: + source_code = f.read() tree = parser.parse(bytes(source_code, "utf8")) + return JAVA_LANGUAGE, source_code, tree + - def traverse_tree(tree: Tree) -> Generator[Node, None, None]: - cursor = tree.walk() - visited_children = False - - while True: - if not visited_children: - yield cursor.node - visited_children = not cursor.goto_first_child() - elif cursor.goto_next_sibling(): - visited_children = False - elif not cursor.goto_parent(): - break + def extract_snippet(self, source_code: str, start_byte: int, end_byte: int) -> str: + return source_code[start_byte:end_byte] - def get_snippet(start, end): - return '\n'.join([line[start.column:] if i == start.row else line[:end.column] if i == end.row else line - for i, line in enumerate(source_code.splitlines()) - if start.row <= i <= end.row]) - - def get_class_node(tree: Tree) -> list: - classes = [node for node in traverse_tree(tree) if node.type == 'class_declaration'] - for class_node in classes: - for child in class_node.children: - if child.type == 'identifier': - start = child.start_point - end = child.end_point - child_class_name = get_snippet(start, end).split()[0] - if child_class_name == class_name: - return class_node - return None - - def get_class_attributes_nodes(class_node: Node) -> list: - for child in class_node.children: - if child.type == 'class_body': - attributes = [node for node in child.children if node.type == 'field_declaration'] - break - return attributes if attributes else None - - def get_constructor_nodes(class_node: Node) -> list: - for child in class_node.children: - if child.type == 'class_body': - constructors = [node for node in child.children if node.type == 'constructor_declaration'] - break - return constructors if constructors else None - def get_method_node(class_node: Node) -> list: - for child in class_node.children: - if child.type == 'class_body': - methods = [node for node in child.children if node.type == 'method_declaration'] + def get_class_info(self, file_path: str, method_name: str, full_class_name: str) -> tuple: + class_name = full_class_name.split('.')[-1] + JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + + query_text = """ + (class_declaration name: (identifier) @class_name) + (field_declaration) @field_declaration + (constructor_declaration) @constructor_declaration + (method_declaration) @method_def + """ + query = JAVA_LANGUAGE.query(query_text) + captures = query.captures(tree.root_node) + + class_attributes = [] + class_constructors = [] + class_method = "" + found_class = False + + for node, capture_name in captures: + captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) + + if capture_name == "class_name": + if captured_text == class_name: + found_class = True + elif found_class: + # Stop processing if we encounter a different class after finding the target class break + continue - for method in methods: - for child in method.children: - if child.type == 'identifier': - start = child.start_point - end = child.end_point - child_method_name = get_snippet(start, end).split()[0] - if child_method_name == method_name: - return method - return None - - class_node = get_class_node(tree) - class_attributes = [get_snippet(attribute.start_point, attribute.end_point) for attribute in get_class_attributes_nodes(class_node)] - constructor_codes = [get_snippet(constructor.start_point, constructor.end_point) for constructor in get_constructor_nodes(class_node)] - method_node = get_method_node(class_node) - method_code = get_snippet(method_node.start_point, method_node.end_point) + if found_class: + if capture_name == "field_declaration": + class_attributes.append(captured_text) + elif capture_name == "constructor_declaration": + class_constructors.append(captured_text) + elif capture_name == "method_def" and method_name in captured_text: + class_method = captured_text + + return class_attributes, class_constructors, class_method - return class_attributes, constructor_codes, method_code + def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str], file_path: str) -> None: + prompts = [] - def generate_prompts(self, prompts_file, class_name, methods, code): for method in methods: try: - class_attributes, constructor_codes, method_code = self.get_method_code(code, method, class_name) + class_attributes, constructor_codes, method_code = self.get_class_info(file_path, method, class_name) - modified_lines = [] - class_attributes_str = '\n'.join(class_attributes) + class_attributes_str = '\n'.join(class_attributes) if class_attributes else "" + constructor_code_str = '\n'.join(constructor_codes) if constructor_codes else "" method_code_str = ''.join(method_code) - if constructor_codes: - constructor_code_str = '\n'.join(constructor_codes) - modified_lines.append(f"/*\n{class_attributes_str}\n\n{constructor_code_str}\n\n{method_code_str}") - else: - modified_lines.append(f"/*\n{class_attributes_str}\n\n{method_code_str}") - - modified_lines.append("\n*/\n\n") - modified_lines.append("import org.junit.FixMethodOrder;\n") - modified_lines.append("import org.junit.Test;\n") - modified_lines.append("import org.junit.runners.MethodSorters;\n") - modified_lines.append("import static org.junit.Assert.*;\n\n") - modified_lines.append(f"@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n") - modified_lines.append(f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n") - modified_lines.append(f"//continue the test with code only:\n") - modified_lines.append(f"\"\"\"") + prompt = ( + f"/*\n{class_attributes_str}\n\n{constructor_code_str}\n\n{method_code_str}\n*/\n\n" + "import org.junit.FixMethodOrder;\n" + "import org.junit.Test;\n" + "import org.junit.runners.MethodSorters;\n" + "import static org.junit.Assert.*;\n\n" + "@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n" + f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" + "//continue the test with code only:\n" + ) - with open(prompts_file, "w") as file: - file.writelines(modified_lines) + prompts.append(prompt) except Exception as e: logging.error("Error while generating prompt for method %s: %s", method, e) + with open(prompts_file, "w") as file: + json.dump(prompts, file, indent=4) + - def get_imports(self, code): - with open(code, 'r') as file: - lines = file.readlines() + def get_imports(self, file_path: str) -> List[str]: + JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + + query_text = """ + (import_declaration) @import + (package_declaration) @package + """ + query = JAVA_LANGUAGE.query(query_text) + captures = query.captures(tree.root_node) imports = [] - for line in lines: - if line.strip().startswith("import"): - imports.append(line) - elif line.strip().startswith("package"): - package_name = line.split()[1].rstrip(';') + + for node, capture_name in captures: + start_byte, end_byte = node.start_byte, node.end_byte + captured_text = source_code[start_byte:end_byte].strip() + + if capture_name == "import": + imports.append(f'{captured_text}\n') + elif capture_name == "package": + package_name = captured_text.split()[1].rstrip(';') imports.append(f'import {package_name}.*;\n') return imports def read_prompts(self, file_path): - prompts = [] - current_prompt = "" - with open(file_path, "r") as file: - lines = file.readlines() - - for line in lines: - if line.strip() != '"""': - current_prompt += line - else: - if current_prompt.strip() != "": - prompts.append(current_prompt) - current_prompt = "" - + prompts = json.load(file) + return prompts - def create_chat_module(self, mpath, lpath, model, lib): - return ChatModule( - model=f"{mpath}/{model}", - model_lib_path=f"{lpath}/{lib}", - ) + def get_individual_tests(self, output_path: str, prompt: str, class_name: str, imports: str, i: int) -> None: + llm_outputs_path = os.path.join(output_path, "llm_outputs") + counter = 0 + def classify_annotations(captures, source_code): + before_block = [] + test_block = [] - def reset_chat_module(self, chat_module): - chat_module.reset_chat() + for node, _ in captures: + captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) + if "@Test" in captured_text: + test_block.append({"snippet": captured_text}) + elif any(annotation in captured_text for annotation in ["@Before", "@BeforeClass", "@BeforeAll"]): + before_block.append({"snippet": captured_text}) + return before_block, test_block - def generate_output(self, chat_module, prompt): - return chat_module.generate(prompt=prompt) + for file in os.listdir(llm_outputs_path): + if file.endswith(".txt") and file.startswith(f"{i}"): + file_path = os.path.join(llm_outputs_path, file) + JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + query_text = """ + (method_declaration + (modifiers + (marker_annotation))) @method_def + """ + query = JAVA_LANGUAGE.query(query_text) + captures = query.captures(tree.root_node) - def save_output(self, prompt, output, dir, output_file_name): - if dir: - if not os.path.exists(dir): - os.makedirs(dir) - if not os.path.exists(f"{dir}/llm_outputs"): - os.makedirs(f"{dir}/llm_outputs") + before_block, test_block = classify_annotations(captures, source_code) - output_file_path = f"{dir}/llm_outputs/{output_file_name}.txt" - with open(output_file_path, "w") as f: - f.write(prompt + output) + for test in test_block: + method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" + output_file_path = os.path.join(output_path, f"{method_name}.java") + new_prompt = prompt.split("public class")[0] + f"public class {method_name} {{\n" + full_prompt = "".join(imports) + new_prompt - def get_branch(self, input_jar, code_paths): - branches = ["base", "left", "right", "merge"] - for branch in branches: - if branch in input_jar: - return code_paths[branch], branch - raise ValueError(f"Nenhuma correspondência de branch encontrada no caminho: {input_jar}") + if before_block: + full_prompt += "".join(before['snippet'] for before in before_block) + snippet = test['snippet'] + test_method_name = snippet.split('(')[0].split()[-1] + new_snippet = snippet.replace(test_method_name, f"test{i}{counter}") - def get_individual_tests(self, output_path, prompt, class_name, imports, i): - llm_outputs_path = f"{output_path}/llm_outputs/" + with open(output_file_path, "w") as f: + f.write(full_prompt + new_snippet + "\n}") - JAVA_LANGUAGE = Language(tsjava.language()) - parser = Parser(JAVA_LANGUAGE) - - for file in os.listdir(llm_outputs_path): - if file.endswith(".txt") and file.startswith(f"{i}"): - source_code = open(os.path.join(llm_outputs_path, file)).read() - tree = parser.parse(bytes(source_code, "utf8")) - - def traverse_tree(tree: Tree) -> Generator[Node, None, None]: - cursor = tree.walk() - visited_children = False - - while True: - if not visited_children: - yield cursor.node - visited_children = not cursor.goto_first_child() - elif cursor.goto_next_sibling(): - visited_children = False - elif not cursor.goto_parent(): - break - - def collect_methods(tree: Tree) -> list: - methods = [node for node in traverse_tree(tree) if node.type == 'method_declaration'] - return methods - - def has_missing_brace(body: Node): - if body.children[-1].start_point.column == body.children[-1].end_point.column: - return True - return False - - def separate_tests(methods): - before_block = [] - test_block = [] - - def get_snippet(start, end): - return '\n'.join([line[start.column:] if i == start.row else line[:end.column] if i == end.row else line - for i, line in enumerate(source_code.splitlines()) - if start.row <= i <= end.row]) - - for method in methods: - children = method.children - for child in children: - if child.type == 'modifiers': - if child.children[0].type == 'marker_annotation': - start = child.children[0].start_point - end = child.children[0].end_point - - annotation_snippet = get_snippet(start, end) - - if annotation_snippet in ["@Before", "@BeforeEach", "@BeforeAll", "@BeforeClass"]: - start = method.start_point - end = method.end_point - before_block.append({"snippet": get_snippet(start, end), "missing_braces": has_missing_brace(method.children[-1])}) - - elif annotation_snippet == "@Test": - start = method.start_point - end = method.end_point - test_block.append({"snippet": get_snippet(start, end), "missing_braces": has_missing_brace(method.children[-1])}) - - print(before_block) - print(test_block) - return before_block, test_block - - methods = collect_methods(tree) - before_block, test_block = separate_tests(methods) - - for k, test in enumerate(test_block): - method_name = f"{class_name.split('.')[-1]}Test_{i}_{k}" - file_path = f"{output_path}/{method_name}.java" - - with open(file_path, "w") as f: - new_prompt = prompt.split("public class")[0] - new_prompt += f"public class {method_name} {{\n" - full_prompt = "".join(imports) + new_prompt - - if before_block: - for before in before_block: - full_prompt += "".join(before['snippet']) - - snippet = test['snippet'] - test_signature = snippet.split("{")[0].strip() - new_signature = f"public void test{i}{k}()" - new_snippet = snippet.replace(test_signature, new_signature, 1) - - f.write(full_prompt + new_snippet) - - if test['missing_braces']: - f.write("}") - f.write("}\n") + counter += 1 - def find_source_code_paths(self, jar_path, class_name): + def find_source_code_paths(self, jar_path: str, class_name: str) -> dict: return {"base": "/mnt/c/Users/natha/Downloads/smat/base.java", "left": "/mnt/c/Users/natha/Downloads/smat/left.java", "right": "/mnt/c/Users/natha/Downloads/smat/right.java", @@ -359,34 +271,53 @@ def find_source_code_paths(self, jar_path, class_name): raise FileNotFoundError(f"Os seguintes arquivos não foram encontrados: {', '.join(missing_files)}") return java_files + - def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: - class_name, methods = list(scenario.targets.items())[0] - - mpath = "/home/nfab/dist" - lpath = "/home/nfab/dist/libs" - model = "CodeLlama-7b-Instruct-hf-q4f16_1-MLC" - lib = "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" - prompts_path = f"{output_path}/prompts.txt" - code_paths = self.find_source_code_paths(input_jar, class_name.split('.')[-1]) - code, branch = self.get_branch(input_jar, code_paths) - - self.generate_prompts(prompts_path, class_name, methods, code) - imports = self.get_imports(code) + def get_branch_info(self, input_jar: str, class_name: str) -> tuple: + source_code_paths = self.find_source_code_paths(input_jar, class_name.split('.')[-1]) + branches = ["base", "left", "right", "merge"] - prompts_list = self.read_prompts(prompts_path) + branch = next((b for b in branches if b in input_jar), None) - cm = self.create_chat_module(mpath, lpath, model, lib) + if branch: + file_path = source_code_paths.get(branch) + if file_path: + return file_path, branch + + available_branches = ", ".join(branches) + raise ValueError(f"No corresponding branch found in '{input_jar}'. Available branches: {available_branches}") + + + def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: + class_name, methods = next(iter(scenario.targets.items())) + model_info = { + "mpath": "/home/nfab/dist", + "lpath": "/home/nfab/dist/libs", + "model": "CodeLlama-7b-Instruct-hf-q4f16_1-MLC", + "lib": "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" + } + prompts_path = os.path.join(output_path, "prompts.json") + + file_path, branch = self.get_branch_info(input_jar, class_name) + self.generate_prompts(prompts_path, class_name, methods, file_path) + imports = self.get_imports(file_path) + prompts_list = self.read_prompts(prompts_path) + + cm = self.create_chat_module(**model_info) for i, prompt in enumerate(prompts_list): - for j in range(0, 5): - try: - logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) - output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" - output = self.generate_output(cm, prompt) - self.save_output(prompt, output, output_path, output_file_name) - self.reset_chat_module(cm) - except Exception as e: - logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) - pass - self.get_individual_tests(output_path, prompt, class_name, imports, i) \ No newline at end of file + self._process_prompts(prompt, output_path, branch, class_name, imports, i, cm) + + def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: str, i: int, cm, num_outputs=5) -> None: + for j in range(num_outputs): + output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" + try: + logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) + output = self.generate_output(cm, prompt) + self.save_output(prompt, output, output_path, output_file_name) + except Exception as e: + logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) + finally: + self.reset_chat_module(cm) + + self.get_individual_tests(output_path, prompt, class_name, imports, i) From b70c0520ec738c104def3e7391cfa69dd5e94e27 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 3 Sep 2024 19:13:27 -0300 Subject: [PATCH 18/69] =?UTF-8?q?melhor=20logging=20e=20mudan=C3=A7a=20no?= =?UTF-8?q?=20path=20do=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nimrod/tests/utils.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index 34dc13a8..74084e24 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -76,37 +76,30 @@ def calculator_sum_aor_1(): def setup_logging(): config = get_config() - config_level = config.get('logger_level') - level = logging._nameToLevel[config_level] if config_level else logging.INFO - - # Obtém o logger raiz + config_level = config.get('logger_level', 'INFO').upper() + level = logging._nameToLevel.get(config_level, logging.INFO) + logger = logging.getLogger() logger.setLevel(level) - - # Remove handlers antigos se existirem para evitar duplicação + if logger.hasHandlers(): logger.handlers.clear() - - # Configura o formato e handlers + formatter = logging.Formatter( - '%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s', + '%(asctime)s %(levelname)s %(module)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) - if os.path.exists('logfile.log'): - os.remove('logfile.log') - - # File handler para salvar logs em arquivo - file_handler = logging.FileHandler('logfile.log') + file_handler = logging.FileHandler('logfile.log', mode='a') file_handler.setFormatter(formatter) - - # Stream handler para exibir logs no terminal + stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) - - # Adiciona handlers ao logger + logger.addHandler(file_handler) logger.addHandler(stream_handler) def get_base_output_path() -> str: - return os.getcwd().replace("/nimrod/proj", "/")+'/output-test-dest/' if os.getcwd().__contains__("/nimrod/proj") else os.getcwd() + "/output-test-dest/" + current_dir = os.getcwd() + base_dir = current_dir.replace("/nimrod/proj", "") if "/nimrod/proj" in current_dir else current_dir + return os.path.join(base_dir, "output-test-dest", "projects") From e25abcb335a45223eb77429d71579df874837adf Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 3 Sep 2024 19:13:42 -0300 Subject: [PATCH 19/69] reports nao sao sobrescritos --- nimrod/output_generation/output_generator.py | 27 +++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/nimrod/output_generation/output_generator.py b/nimrod/output_generation/output_generator.py index b7a53440..b3eb8792 100644 --- a/nimrod/output_generation/output_generator.py +++ b/nimrod/output_generation/output_generator.py @@ -11,7 +11,8 @@ class OutputGenerator(ABC, Generic[T]): - REPORTS_DIRECTORY = path.join(get_base_output_path(), "reports") + parent_dir = path.dirname(get_base_output_path()) + REPORTS_DIRECTORY = path.join(parent_dir, "reports") def __init__(self, report_name: str) -> None: super().__init__() @@ -27,9 +28,27 @@ def write_report(self, context: OutputGeneratorContext) -> None: file_path = path.join(self.REPORTS_DIRECTORY, self._report_name) logging.info(f"Starting data processing of {self._report_name} report") - data = self._generate_report_data(context) + new_data = self._generate_report_data(context) logging.info(f"Finished data processing of {self._report_name} report") - with open(file_path, "w") as write: - json.dump(data, write) + existing_data = self._load_existing_data(file_path) + + if not isinstance(existing_data, list): + existing_data = [existing_data] if existing_data else [] + existing_data.append(new_data) + + self._write_json(file_path, existing_data) logging.info(f"Finished generation of {self._report_name} report") + + def _load_existing_data(self, file_path: str): + if not path.exists(file_path): + return [] + try: + with open(file_path, "r") as read_file: + return json.load(read_file) + except json.JSONDecodeError: + return [] + + def _write_json(self, file_path: str, data) -> None: + with open(file_path, "w") as write_file: + json.dump(data, write_file, indent=4) From 131d0015d607634feeea0d96d0cf22579bed850a Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 3 Sep 2024 19:14:11 -0300 Subject: [PATCH 20/69] conserto de varios bugs --- .../codellama_test_suite_generator.py | 202 ++++++++++-------- 1 file changed, 110 insertions(+), 92 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 7974679f..61afe2d3 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,7 +1,7 @@ import json import logging import os -from typing import List +from typing import List, Dict import tree_sitter_java as tsjava from tree_sitter import Language, Parser @@ -63,51 +63,71 @@ def extract_snippet(self, source_code: str, start_byte: int, end_byte: int) -> s return source_code[start_byte:end_byte] - def get_class_info(self, file_path: str, method_name: str, full_class_name: str) -> tuple: - class_name = full_class_name.split('.')[-1] - JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) - - query_text = """ - (class_declaration name: (identifier) @class_name) - (field_declaration) @field_declaration - (constructor_declaration) @constructor_declaration - (method_declaration) @method_def - """ - query = JAVA_LANGUAGE.query(query_text) - captures = query.captures(tree.root_node) - - class_attributes = [] - class_constructors = [] - class_method = "" - found_class = False + def get_class_info(self, file_path: str, full_method_name: str, full_class_name: str) -> tuple: + try: + class_name = full_class_name.split('.')[-1] + method_name = full_method_name.split('(')[0] + JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + + query_text = f""" + (class_declaration + name: (identifier) @class_name + body: (class_body + [ + (field_declaration) @field_declaration + (constructor_declaration + name: (identifier) @constructor_name) @constructor_declaration + (method_declaration + name: (identifier) @method_name) @method_def + (#eq? @method_name "{method_name}") + (#eq? @constructor_name "{class_name}") + ] + ) + (#eq? @class_name "{class_name}") + ) + """ + query = JAVA_LANGUAGE.query(query_text) + captures = query.captures(tree.root_node) - for node, capture_name in captures: - captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) + if not captures: + raise Exception(f"No captures found for the class '{class_name}' in '{file_path}'") - if capture_name == "class_name": - if captured_text == class_name: - found_class = True - elif found_class: - # Stop processing if we encounter a different class after finding the target class - break - continue + class_attributes = [] + class_constructors = [] + class_method = "" - if found_class: + for node, capture_name in captures: + captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) + if capture_name == "field_declaration": class_attributes.append(captured_text) elif capture_name == "constructor_declaration": class_constructors.append(captured_text) - elif capture_name == "method_def" and method_name in captured_text: + elif capture_name == "method_def": class_method = captured_text - return class_attributes, class_constructors, class_method + return class_attributes, class_constructors, class_method + except Exception as e: + logging.error(f"An error occurred while extracting class info for '{full_class_name}': {e}") + raise e def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str], file_path: str) -> None: - prompts = [] + if os.path.exists(prompts_file): + with open(prompts_file, "r") as file: + try: + prompts_dict = json.load(file) + except json.JSONDecodeError: + prompts_dict = {} + else: + prompts_dict = {} + + if class_name not in prompts_dict: + prompts_dict[class_name] = [] for method in methods: try: + logging.info("Generating prompt for method '%s'", method) class_attributes, constructor_codes, method_code = self.get_class_info(file_path, method, class_name) class_attributes_str = '\n'.join(class_attributes) if class_attributes else "" @@ -125,16 +145,16 @@ def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str "//continue the test with code only:\n" ) - prompts.append(prompt) + prompts_dict[class_name].append(prompt) except Exception as e: logging.error("Error while generating prompt for method %s: %s", method, e) with open(prompts_file, "w") as file: - json.dump(prompts, file, indent=4) + json.dump(prompts_dict, file, indent=4) - def get_imports(self, file_path: str) -> List[str]: + def get_imports(self, class_name: str, file_path: str, imports_path: str) -> None: JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) query_text = """ @@ -144,29 +164,38 @@ def get_imports(self, file_path: str) -> List[str]: query = JAVA_LANGUAGE.query(query_text) captures = query.captures(tree.root_node) - imports = [] - + if os.path.exists(imports_path): + with open(imports_path, "r") as file: + try: + imports_dict = json.load(file) + except json.JSONDecodeError: + imports_dict = {} + else: + imports_dict = {} + + class_imports = imports_dict.setdefault(class_name, []) + for node, capture_name in captures: start_byte, end_byte = node.start_byte, node.end_byte captured_text = source_code[start_byte:end_byte].strip() if capture_name == "import": - imports.append(f'{captured_text}\n') + class_imports.append(f'{captured_text}\n') elif capture_name == "package": package_name = captured_text.split()[1].rstrip(';') - imports.append(f'import {package_name}.*;\n') + class_imports.append(f'import {package_name}.*;\n') - return imports + with open(imports_path, "w") as file: + json.dump(imports_dict, file, indent=4) - def read_prompts(self, file_path): + def load_json(self, file_path): with open(file_path, "r") as file: - prompts = json.load(file) - - return prompts + content = json.load(file) + return content - def get_individual_tests(self, output_path: str, prompt: str, class_name: str, imports: str, i: int) -> None: + def get_individual_tests(self, output_path: str, prompt: str, class_name: str, imports: List[str], i: int) -> None: llm_outputs_path = os.path.join(output_path, "llm_outputs") counter = 0 @@ -217,58 +246,40 @@ def classify_annotations(captures, source_code): counter += 1 - + def find_source_code_paths(self, jar_path: str, class_name: str) -> dict: - return {"base": "/mnt/c/Users/natha/Downloads/smat/base.java", - "left": "/mnt/c/Users/natha/Downloads/smat/left.java", - "right": "/mnt/c/Users/natha/Downloads/smat/right.java", - "merge": "/mnt/c/Users/natha/Downloads/smat/merge.java"} - # Dividir o caminho em partes path_parts = jar_path.split(os.sep) - # Verificar se a estrutura do caminho é a esperada if "transformed" not in path_parts: - raise ValueError("O caminho fornecido não contém a parte 'transformed'.") + raise ValueError("The provided path does not contain the 'transformed' directory") - # Encontrar o índice da parte "transformed" transformed_index = path_parts.index("transformed") - - # Substituir "transformed" por "source" e remover todas as partes subsequentes - source_path_parts = path_parts[:transformed_index] + ["source"] - - # Construir o caminho base - base_path = os.path.join("/", *source_path_parts) + base_path = os.path.join("/", *path_parts[:transformed_index], "source") - # Verificar se o diretório base existe antes de procurar if not os.path.exists(base_path): - raise FileNotFoundError(f"O diretório base não existe: {base_path}") + raise FileNotFoundError(f"The base path '{base_path}' does not exist") - # Inicializar dicionário para armazenar caminhos de arquivos - java_files = {"base": "", "left": "", "right": "", "merge": ""} + java_files = {key: "" for key in ["base", "left", "right", "merge"]} - # Procurar por arquivos .java recursivamente - for root, dirs, files in os.walk(base_path): - # Verificar se a pasta com o nome da classe existe e dar preferência aos arquivos dentro dela + for root, _, files in os.walk(base_path): + java_candidates = [file for file in files if file.endswith(".java")] + + # Prioritize files within the class-named folder if os.path.basename(root) == class_name: - for file in files: - if file.endswith(".java"): - file_key = file.replace(".java", "") - if file_key in java_files: - java_files[file_key] = os.path.join(root, file) - - # Verificar arquivos fora da pasta com o nome da classe - for file in files: - if file.endswith(".java"): + for file in java_candidates: file_key = file.replace(".java", "") - if file_key in java_files and not java_files[file_key]: + if file_key in java_files: java_files[file_key] = os.path.join(root, file) - if all(java_files.values()): - break - # Verifica se todos os arquivos foram encontrados, caso contrário, lança um erro - missing_files = [key for key, value in java_files.items() if not value] + # If any file is still missing, try to fill it in + for file in java_candidates: + file_key = file.replace(".java", "") + if file_key in java_files and not java_files[file_key]: + java_files[file_key] = os.path.join(root, file) + + missing_files = [key for key, path in java_files.items() if not path] if missing_files: - raise FileNotFoundError(f"Os seguintes arquivos não foram encontrados: {', '.join(missing_files)}") + raise FileNotFoundError(f"The following files were not found: {', '.join(missing_files)}") return java_files @@ -289,7 +300,6 @@ def get_branch_info(self, input_jar: str, class_name: str) -> tuple: def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: - class_name, methods = next(iter(scenario.targets.items())) model_info = { "mpath": "/home/nfab/dist", "lpath": "/home/nfab/dist/libs", @@ -297,27 +307,35 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s "lib": "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" } prompts_path = os.path.join(output_path, "prompts.json") + imports_path = os.path.join(output_path, "imports.json") + targets = scenario.targets - file_path, branch = self.get_branch_info(input_jar, class_name) - self.generate_prompts(prompts_path, class_name, methods, file_path) - imports = self.get_imports(file_path) - prompts_list = self.read_prompts(prompts_path) - + for class_name, methods in targets.items(): + file_path, branch = self.get_branch_info(input_jar, class_name) + self.generate_prompts(prompts_path, class_name, methods, file_path) + self.get_imports(class_name, file_path, imports_path) + + prompts_dict = self.load_json(prompts_path) + imports_dict = self.load_json(imports_path) cm = self.create_chat_module(**model_info) - for i, prompt in enumerate(prompts_list): - self._process_prompts(prompt, output_path, branch, class_name, imports, i, cm) + for class_name, prompts_list in prompts_dict.items(): + logging.info("Generating tests for target methods in class '%s'", class_name) + for i, prompt in enumerate(prompts_list): + self._process_prompts(prompt, output_path, branch, class_name, imports=imports_dict.get(class_name, []), i=i, chat_module=cm) + + os.remove(imports_path) # Remove imports file after generating tests - def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: str, i: int, cm, num_outputs=5) -> None: + def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, chat_module: ChatModule, num_outputs: int = 5) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) - output = self.generate_output(cm, prompt) + output = self.generate_output(chat_module, prompt) self.save_output(prompt, output, output_path, output_file_name) except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) finally: - self.reset_chat_module(cm) + self.reset_chat_module(chat_module) self.get_individual_tests(output_path, prompt, class_name, imports, i) From e42e1b746f1cb2bd668a60fec44034e06492e3dd Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 16 Sep 2024 21:44:30 -0300 Subject: [PATCH 21/69] fix jar type search --- .../codellama_test_suite_generator.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 61afe2d3..fdb58464 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -247,14 +247,13 @@ def classify_annotations(captures, source_code): counter += 1 - def find_source_code_paths(self, jar_path: str, class_name: str) -> dict: - path_parts = jar_path.split(os.sep) + def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) -> Dict[str, str]: + path_parts = input_jar.split(os.sep) + if jar_type != "transformed" and jar_type != "original": + raise ValueError("The provided path does not contain the expected jar type (transformed/original)") - if "transformed" not in path_parts: - raise ValueError("The provided path does not contain the 'transformed' directory") - - transformed_index = path_parts.index("transformed") - base_path = os.path.join("/", *path_parts[:transformed_index], "source") + type_index = path_parts.index(jar_type) + base_path = os.path.join("/", *path_parts[:type_index], "source") if not os.path.exists(base_path): raise FileNotFoundError(f"The base path '{base_path}' does not exist") @@ -284,8 +283,8 @@ def find_source_code_paths(self, jar_path: str, class_name: str) -> dict: return java_files - def get_branch_info(self, input_jar: str, class_name: str) -> tuple: - source_code_paths = self.find_source_code_paths(input_jar, class_name.split('.')[-1]) + def get_branch_info(self, input_jar: str, jar_type: str, class_name: str) -> tuple: + source_code_paths = self.find_source_code_paths(input_jar, jar_type, class_name.split('.')[-1]) branches = ["base", "left", "right", "merge"] branch = next((b for b in branches if b in input_jar), None) @@ -309,9 +308,10 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s prompts_path = os.path.join(output_path, "prompts.json") imports_path = os.path.join(output_path, "imports.json") targets = scenario.targets + jar_type = scenario.jar_type for class_name, methods in targets.items(): - file_path, branch = self.get_branch_info(input_jar, class_name) + file_path, branch = self.get_branch_info(input_jar, jar_type, class_name) self.generate_prompts(prompts_path, class_name, methods, file_path) self.get_imports(class_name, file_path, imports_path) From ae8badc7ea6c88f9f1ad3bdba110baaeb078f939 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 30 Sep 2024 20:57:35 -0300 Subject: [PATCH 22/69] update logging --- nimrod/tests/utils.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index 74084e24..4f361387 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -85,8 +85,26 @@ def setup_logging(): if logger.hasHandlers(): logger.handlers.clear() + modules_to_ignore = [ + 'mlc_chat.support.auto_device', + 'mlc_chat.support', + 'mlc_chat', + 'mlc_chat.support.config', + 'mlc_chat.chat_module', + 'mlc_chat.serve.engine', + 'mlc_chat.serve', + 'auto_device', + 'chat_module', + 'model_metadata' + ] + + for module in modules_to_ignore: + mod_logger = logging.getLogger(module) + mod_logger.setLevel(logging.ERROR) + mod_logger.propagate = False + formatter = logging.Formatter( - '%(asctime)s %(levelname)s %(module)s - %(message)s', + '[%(asctime)s] %(levelname)s %(filename)s:%(lineno)d: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) From 660838aa92ff88fb1f00d8e32d099702ab050ec2 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 30 Sep 2024 20:58:04 -0300 Subject: [PATCH 23/69] calc time --- .../codellama_test_suite_generator.py | 66 +++++++++++++++++-- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index fdb58464..fc7dba75 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -127,7 +127,7 @@ def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str for method in methods: try: - logging.info("Generating prompt for method '%s'", method) + logging.debug("Generating prompt for method '%s' in class '%s'", method, class_name) class_attributes, constructor_codes, method_code = self.get_class_info(file_path, method, class_name) class_attributes_str = '\n'.join(class_attributes) if class_attributes else "" @@ -148,7 +148,7 @@ def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str prompts_dict[class_name].append(prompt) except Exception as e: - logging.error("Error while generating prompt for method %s: %s", method, e) + logging.error("Error while generating prompt for method '%s': %s", method, e) with open(prompts_file, "w") as file: json.dump(prompts_dict, file, indent=4) @@ -248,6 +248,10 @@ def classify_annotations(captures, source_code): def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) -> Dict[str, str]: + if not os.path.exists(input_jar): + logging.error("The provided jar path '%s' does not exist", input_jar) + raise FileNotFoundError(f"The provided path '{input_jar}' does not exist") + path_parts = input_jar.split(os.sep) if jar_type != "transformed" and jar_type != "original": raise ValueError("The provided path does not contain the expected jar type (transformed/original)") @@ -278,7 +282,7 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) missing_files = [key for key, path in java_files.items() if not path] if missing_files: - raise FileNotFoundError(f"The following files were not found: {', '.join(missing_files)}") + raise FileNotFoundError(f"The following source code files were not found: {', '.join(missing_files)}") return java_files @@ -298,6 +302,44 @@ def get_branch_info(self, input_jar: str, jar_type: str, class_name: str) -> tup raise ValueError(f"No corresponding branch found in '{input_jar}'. Available branches: {available_branches}") + def calc_time_spent_per_output(self, time_spent_path: str, output_path: str, class_name: str, output_file_name: str, time_spent: float, project_name: str) -> None: + logging.debug("Calculating time spent in output '%s' for class '%s'", output_file_name, class_name) + os.makedirs(os.path.dirname(time_spent_path), exist_ok=True) + + try: + with open(time_spent_path, "r") as file: + time_spent_dict = json.load(file) + except (FileNotFoundError, json.JSONDecodeError): + time_spent_dict = {} + + project_data = time_spent_dict.setdefault(project_name, {}) + class_data = project_data.setdefault(class_name, {"total_time_spent": 0, "outputs": {}}) + + key_name = output_path.split(os.sep)[-1] + '_' + output_file_name + + time_spent_rounded = round(time_spent, 2) + class_data["outputs"][key_name] = time_spent_rounded + class_data["total_time_spent"] = round(class_data["total_time_spent"] + time_spent_rounded, 2) + + try: + with open(time_spent_path, "w") as file: + json.dump(time_spent_dict, file, indent=4) + except Exception as e: + logging.error("Error while saving time spent data to '%s': %s", time_spent_path, e) + raise + + + def free_gpu_memory(self, chat_module: ChatModule) -> None: + import torch + + try: + torch.cuda.empty_cache() + chat_module._unload() + + except Exception as e: + logging.error(f"Error during memory cleanup: {e}") + + def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: model_info = { "mpath": "/home/nfab/dist", @@ -307,6 +349,10 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s } prompts_path = os.path.join(output_path, "prompts.json") imports_path = os.path.join(output_path, "imports.json") + + time_spent_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(output_path))), "reports", "codellama_time_spent.json") + + project_name = scenario.project_name targets = scenario.targets jar_type = scenario.jar_type @@ -320,22 +366,28 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s cm = self.create_chat_module(**model_info) for class_name, prompts_list in prompts_dict.items(): - logging.info("Generating tests for target methods in class '%s'", class_name) + logging.debug("Generating tests for target methods in class '%s'", class_name) for i, prompt in enumerate(prompts_list): - self._process_prompts(prompt, output_path, branch, class_name, imports=imports_dict.get(class_name, []), i=i, chat_module=cm) - + self._process_prompts(prompt, output_path, branch, class_name, imports=imports_dict.get(class_name, []), i=i, chat_module=cm, time_spent_path=time_spent_path, project_name=project_name) + os.remove(imports_path) # Remove imports file after generating tests + self.free_gpu_memory(cm) - def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, chat_module: ChatModule, num_outputs: int = 5) -> None: + def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, chat_module: ChatModule, time_spent_path: str, project_name: str, num_outputs: int = 5) -> None: + import time for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) + start_time = time.time() output = self.generate_output(chat_module, prompt) self.save_output(prompt, output, output_path, output_file_name) except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) finally: self.reset_chat_module(chat_module) + end_time = time.time() + time_spent = end_time - start_time + self.calc_time_spent_per_output(time_spent_path, output_path, class_name, output_file_name, time_spent, project_name) self.get_individual_tests(output_path, prompt, class_name, imports, i) From 6dfed76932e38de80897db3a355cc176c47e8900 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 30 Sep 2024 20:58:23 -0300 Subject: [PATCH 24/69] better output --- nimrod/output_generation/test_suites_output_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nimrod/output_generation/test_suites_output_generator.py b/nimrod/output_generation/test_suites_output_generator.py index 2d14765c..f477221a 100644 --- a/nimrod/output_generation/test_suites_output_generator.py +++ b/nimrod/output_generation/test_suites_output_generator.py @@ -23,6 +23,7 @@ def _generate_report_data(self, context: OutputGeneratorContext) -> List[TestSui for test_suite in context.test_suites: report_data.append({ "project_name": context.scenario.project_name, + "targets": context.scenario.targets, "generator_name": test_suite.generator_name, "path": test_suite.path, "detected_semantic_conflicts": self._has_detected_semantic_conflicts_in_test_suite(test_suite, context.semantic_conflicts), From 8c88634cb88634e256a671477cad3cb97066f10c Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 30 Sep 2024 21:08:53 -0300 Subject: [PATCH 25/69] update req --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 33739f17..351a835e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,8 @@ +--extra-index-url https://mlc.ai/wheels beautifulsoup4==4.12.3 -mlc_chat_nightly_cu122==0.1.dev962 setuptools==68.2.2 +torch==2.2.1 tree_sitter==0.22.3 tree_sitter_java==0.21.0 +mlc-llm-nightly-cu122 +mlc-ai-nightly-cu122 From ea9df201eb0abb7f5bba3fbdb14d22774ebfd3c7 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 30 Sep 2024 21:18:56 -0300 Subject: [PATCH 26/69] a --- nimrod/output_generation/test_suites_output_generator.py | 1 + .../generators/codellama_test_suite_generator.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nimrod/output_generation/test_suites_output_generator.py b/nimrod/output_generation/test_suites_output_generator.py index f477221a..3cfd265b 100644 --- a/nimrod/output_generation/test_suites_output_generator.py +++ b/nimrod/output_generation/test_suites_output_generator.py @@ -7,6 +7,7 @@ class TestSuitesOutput(TypedDict): project_name: str + targets: List[str] generator_name: str path: str detected_semantic_conflicts: bool diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index fc7dba75..15ec9357 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -17,7 +17,7 @@ def get_generator_tool_name(self) -> str: def _get_test_suite_class_paths(self, path: str) -> List[str]: - paths = [] + paths: list[str] = [] for root, _, files in os.walk(path): paths.extend(os.path.join(root, file) for file in files if file.endswith(".java")) return paths @@ -92,9 +92,9 @@ def get_class_info(self, file_path: str, full_method_name: str, full_class_name: if not captures: raise Exception(f"No captures found for the class '{class_name}' in '{file_path}'") - class_attributes = [] - class_constructors = [] - class_method = "" + class_attributes: List[str] = [] + class_constructors: List[str] = [] + class_method: str = "" for node, capture_name in captures: captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) From fad3c28443b1ce0844c1a811229c80cd2a8239cc Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 19 Nov 2024 19:09:25 -0300 Subject: [PATCH 27/69] fix: "targets" attr type --- nimrod/output_generation/test_suites_output_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimrod/output_generation/test_suites_output_generator.py b/nimrod/output_generation/test_suites_output_generator.py index 3cfd265b..b9a9039c 100644 --- a/nimrod/output_generation/test_suites_output_generator.py +++ b/nimrod/output_generation/test_suites_output_generator.py @@ -7,7 +7,7 @@ class TestSuitesOutput(TypedDict): project_name: str - targets: List[str] + targets: dict[str, list[str]] generator_name: str path: str detected_semantic_conflicts: bool From 44c3ff6d3acfc621dc11cc909019a0a5dc46f002 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 19 Nov 2024 19:09:46 -0300 Subject: [PATCH 28/69] remove ignored modules from logging --- nimrod/tests/utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index 4f361387..7386a1b0 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -85,24 +85,6 @@ def setup_logging(): if logger.hasHandlers(): logger.handlers.clear() - modules_to_ignore = [ - 'mlc_chat.support.auto_device', - 'mlc_chat.support', - 'mlc_chat', - 'mlc_chat.support.config', - 'mlc_chat.chat_module', - 'mlc_chat.serve.engine', - 'mlc_chat.serve', - 'auto_device', - 'chat_module', - 'model_metadata' - ] - - for module in modules_to_ignore: - mod_logger = logging.getLogger(module) - mod_logger.setLevel(logging.ERROR) - mod_logger.propagate = False - formatter = logging.Formatter( '[%(asctime)s] %(levelname)s %(filename)s:%(lineno)d: %(message)s', datefmt='%Y-%m-%d %H:%M:%S' From 7169eebfb2bd56c5fe8cb2e46a7a6bf3379ec9ac Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Wed, 20 Nov 2024 19:09:29 -0300 Subject: [PATCH 29/69] feat(model): migrate from local codellama-7b to API-based codellama-70b with refactoring --- .../codellama_test_suite_generator.py | 335 +++++++++++------- 1 file changed, 199 insertions(+), 136 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 15ec9357..8284f4e5 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -1,14 +1,15 @@ import json import logging import os +import requests from typing import List, Dict import tree_sitter_java as tsjava from tree_sitter import Language, Parser -from mlc_chat import ChatModule from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator +from nimrod.tests.utils import get_config class CodellamaTestSuiteGenerator(TestSuiteGenerator): @@ -27,47 +28,131 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] - def create_chat_module(self, mpath: str, lpath: str, model: str, lib: str) -> ChatModule: - return ChatModule( - model=f"{mpath}/{model}", - model_lib_path=f"{lpath}/{lib}", - ) + def load_json(self, file_path): + """Loads a JSON file and return its content as a dictionary""" + with open(file_path, "r") as file: + try: + content = json.load(file) + except json.JSONDecodeError: + content = {} + return content + + + def save_json(self, file_path, content): + """Saves a dictionary as a JSON file""" + with open(file_path, "w") as file: + json.dump(content, file, indent=4) + + + def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) -> List[Dict[str, str]]: + """Builds the test prompt messages for the given method information and class name""" + class_name = full_class_name.split('.')[-1] + class_fields = method_info.get("class_fields", []) + constructor_codes = method_info.get("constructor_codes", []) + method_code = method_info.get("method_code", "") + test_template = method_info.get("test_template", "") + messages = [ + { + "role": "user", + "content": f"""You are a Java test generator for JUnit. You can only write tests for Java classes. + You will first receive information about the class, and then be asked to create tests based on that information. + When asked to generate test code, follow the next rules: + 1. Complete the provided test template with Java code only, starting your output with the annotation '@Test', using JUnit format and nothing else. + 2. Do not provide informations about the tests or your reasoning. You should only write the test code. + 3. Additional information, like titles, descriptions, comments, or any other text must be inside /* */ or // comments. + 4. The output should compile and run as is, without errors. + Below, you will find additional information regarding the Class under Test ({class_name}): + Attributes: + {class_fields} + + Constructor: + """ + "\n".join(constructor_codes) + f""" + + Target Method Under Test: + {method_code}""" + }, + { + "role": "assistant", + "content": f"//Okay, I understand all rules and all details about the class {class_name}. Let's start generating tests for it." + }, + { + "role": "user", + "content": f"Now, complete the JUnit test template below:\n{test_template}" + } + ] + + return messages + + def generate_output(self, messages: str, api_url: str) -> Dict[str, str]: + """Generates the output using the API and returns the response and time duration""" + url = api_url + headers = {"Content-Type": "application/json"} + payload = { + "model": "codellama:70b", + "messages": messages, + "stream": False, + "options": {"temperature": 0.7}, + } + logging.debug("Starting API request...") + timeout_seconds = 500 - def reset_chat_module(self, chat_module: ChatModule) -> None: - chat_module.reset_chat() + try: + response = requests.post(url, headers=headers, json=payload, timeout=timeout_seconds) + response.raise_for_status() + logging.debug("Request successful. Status: %s", response.status_code) + result = response.json() - def generate_output(self, chat_module: ChatModule, prompt: str) -> str: - return chat_module.generate(prompt=prompt) + return { + "response": result.get("message", {}).get("content", "Response not found."), + "total_duration": result.get("total_duration", 0) + } + + except requests.exceptions.Timeout: + logging.error("Exceeded total timeout of %s seconds. Aborting.", timeout_seconds) + return {"error": "Total timeout exceeded"} + except requests.exceptions.RequestException as e: + logging.error("Request error: %s", e) + return {"error": "Request error"} - def save_output(self, prompt: str, output: str, dir: str, output_file_name: str) -> None: + except json.JSONDecodeError: + logging.error("JSON decoding error: %s", response.text) + return {"error": "Response decoding error"} + + + def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: + """Saves the output generated by the model to a file""" llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") + os.makedirs(llm_outputs_dir, exist_ok=True) with open(output_file_path, "w") as file: - file.write(prompt + output) + file.write(test_template + output) - def parse_code(self, file_path: str) -> tuple: + def parse_code(self, source_code_path: str) -> tuple: + """Parses the Java source code using the Tree-sitter parser and return the language, source code, and generated AST""" JAVA_LANGUAGE = Language(tsjava.language()) parser = Parser(JAVA_LANGUAGE) - with open(file_path, 'r') as f: + with open(source_code_path, 'r') as f: source_code = f.read() tree = parser.parse(bytes(source_code, "utf8")) return JAVA_LANGUAGE, source_code, tree def extract_snippet(self, source_code: str, start_byte: int, end_byte: int) -> str: + """Extracts a snippet of code from the source code using the start and end byte offsets""" return source_code[start_byte:end_byte] - def get_class_info(self, file_path: str, full_method_name: str, full_class_name: str) -> tuple: + def extract_class_info(self, source_code_path: str, full_method_name: str, full_class_name: str) -> tuple: + """Extracts the class fields, constructor, and body of the method under test from the source code using the generated AST to query the information""" try: class_name = full_class_name.split('.')[-1] method_name = full_method_name.split('(')[0] - JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) query_text = f""" (class_declaration @@ -90,9 +175,9 @@ def get_class_info(self, file_path: str, full_method_name: str, full_class_name: captures = query.captures(tree.root_node) if not captures: - raise Exception(f"No captures found for the class '{class_name}' in '{file_path}'") + raise Exception(f"No captures found for the class '{class_name}' in '{source_code_path}'") - class_attributes: List[str] = [] + class_fields: List[str] = [] class_constructors: List[str] = [] class_method: str = "" @@ -100,62 +185,56 @@ def get_class_info(self, file_path: str, full_method_name: str, full_class_name: captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) if capture_name == "field_declaration": - class_attributes.append(captured_text) + class_fields.append(captured_text) elif capture_name == "constructor_declaration": class_constructors.append(captured_text) elif capture_name == "method_def": class_method = captured_text - return class_attributes, class_constructors, class_method + return class_fields, class_constructors, class_method except Exception as e: logging.error(f"An error occurred while extracting class info for '{full_class_name}': {e}") raise e - def generate_prompts(self, prompts_file: str, class_name: str, methods: List[str], file_path: str) -> None: - if os.path.exists(prompts_file): - with open(prompts_file, "r") as file: - try: - prompts_dict = json.load(file) - except json.JSONDecodeError: - prompts_dict = {} + def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods: List[str], source_code_path: str) -> None: + """Stores relevant scenario information (for each class and method) in a JSON file""" + if os.path.exists(scenario_infos_path): + scenario_infos_dict = self.load_json(scenario_infos_path) else: - prompts_dict = {} + scenario_infos_dict = {} - if class_name not in prompts_dict: - prompts_dict[class_name] = [] + if class_name not in scenario_infos_dict: + scenario_infos_dict[class_name] = [] for method in methods: try: - logging.debug("Generating prompt for method '%s' in class '%s'", method, class_name) - class_attributes, constructor_codes, method_code = self.get_class_info(file_path, method, class_name) - - class_attributes_str = '\n'.join(class_attributes) if class_attributes else "" - constructor_code_str = '\n'.join(constructor_codes) if constructor_codes else "" - method_code_str = ''.join(method_code) - - prompt = ( - f"/*\n{class_attributes_str}\n\n{constructor_code_str}\n\n{method_code_str}\n*/\n\n" - "import org.junit.FixMethodOrder;\n" - "import org.junit.Test;\n" - "import org.junit.runners.MethodSorters;\n" - "import static org.junit.Assert.*;\n\n" - "@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n" - f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" - "//continue the test with code only:\n" - ) - - prompts_dict[class_name].append(prompt) + logging.debug("Saving scenario information for method '%s' in class '%s'", method, class_name) + class_fields, constructor_codes, method_code = self.extract_class_info(source_code_path, method, class_name) + + scenario_infos_dict[class_name].append({ + 'class_fields': class_fields if class_fields else [], + 'constructor_codes': constructor_codes if constructor_codes else [], + 'method_code': method_code if method_code else "", + 'test_template': ( + "import org.junit.FixMethodOrder;\n" + "import org.junit.Test;\n" + "import org.junit.runners.MethodSorters;\n" + "import static org.junit.Assert.*;\n\n" + "@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n" + f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" + ) + }) except Exception as e: - logging.error("Error while generating prompt for method '%s': %s", method, e) + logging.error("Error while saving scenario information for method '%s' in class '%s': %s", method, class_name, e) - with open(prompts_file, "w") as file: - json.dump(prompts_dict, file, indent=4) + self.save_json(scenario_infos_path, scenario_infos_dict) - def get_imports(self, class_name: str, file_path: str, imports_path: str) -> None: - JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + def save_imports(self, class_name: str, source_code_path: str, imports_path: str) -> None: + """Extracts import statements from the Java source code and stores them in a JSON file""" + JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) query_text = """ (import_declaration) @import @@ -165,11 +244,7 @@ def get_imports(self, class_name: str, file_path: str, imports_path: str) -> Non captures = query.captures(tree.root_node) if os.path.exists(imports_path): - with open(imports_path, "r") as file: - try: - imports_dict = json.load(file) - except json.JSONDecodeError: - imports_dict = {} + imports_dict = self.load_json(imports_path) else: imports_dict = {} @@ -185,23 +260,18 @@ def get_imports(self, class_name: str, file_path: str, imports_path: str) -> Non package_name = captured_text.split()[1].rstrip(';') class_imports.append(f'import {package_name}.*;\n') - with open(imports_path, "w") as file: - json.dump(imports_dict, file, indent=4) + self.save_json(imports_path, imports_dict) - def load_json(self, file_path): - with open(file_path, "r") as file: - content = json.load(file) - return content - - - def get_individual_tests(self, output_path: str, prompt: str, class_name: str, imports: List[str], i: int) -> None: + def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int) -> None: + """Extracts individual tests from the generated test suite and saves them to separate files""" llm_outputs_path = os.path.join(output_path, "llm_outputs") counter = 0 - def classify_annotations(captures, source_code): - before_block = [] - test_block = [] + def classify_annotations(captures: List[tuple], source_code: str) -> tuple: + """Classifies annotations in the captured snippets as 'before' or 'test' blocks""" + before_block: List[Dict[str, str]] = [] + test_block: List[Dict[str, str]] = [] for node, _ in captures: captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) @@ -214,8 +284,8 @@ def classify_annotations(captures, source_code): for file in os.listdir(llm_outputs_path): if file.endswith(".txt") and file.startswith(f"{i}"): - file_path = os.path.join(llm_outputs_path, file) - JAVA_LANGUAGE, source_code, tree = self.parse_code(file_path) + source_code_path = os.path.join(llm_outputs_path, file) + JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) query_text = """ (method_declaration @@ -231,23 +301,24 @@ def classify_annotations(captures, source_code): method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" output_file_path = os.path.join(output_path, f"{method_name}.java") - new_prompt = prompt.split("public class")[0] + f"public class {method_name} {{\n" - full_prompt = "".join(imports) + new_prompt + new_template = test_template.split("public class")[0] + f"public class {method_name} {{\n" + full_template = "".join(imports) + new_template if before_block: - full_prompt += "".join(before['snippet'] for before in before_block) + full_template += "".join(before['snippet'] for before in before_block) snippet = test['snippet'] test_method_name = snippet.split('(')[0].split()[-1] new_snippet = snippet.replace(test_method_name, f"test{i}{counter}") with open(output_file_path, "w") as f: - f.write(full_prompt + new_snippet + "\n}") + f.write(full_template + new_snippet + "\n}") counter += 1 def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) -> Dict[str, str]: + """Finds the source code files for the given class name in the specified JAR path""" if not os.path.exists(input_jar): logging.error("The provided jar path '%s' does not exist", input_jar) raise FileNotFoundError(f"The provided path '{input_jar}' does not exist") @@ -287,107 +358,99 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) return java_files - def get_branch_info(self, input_jar: str, jar_type: str, class_name: str) -> tuple: + def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: str) -> tuple: + """Retrieves the source code path and branch for the given JAR path""" source_code_paths = self.find_source_code_paths(input_jar, jar_type, class_name.split('.')[-1]) branches = ["base", "left", "right", "merge"] branch = next((b for b in branches if b in input_jar), None) if branch: - file_path = source_code_paths.get(branch) - if file_path: - return file_path, branch + source_code_path = source_code_paths.get(branch) + if source_code_path: + return source_code_path, branch available_branches = ", ".join(branches) raise ValueError(f"No corresponding branch found in '{input_jar}'. Available branches: {available_branches}") - def calc_time_spent_per_output(self, time_spent_path: str, output_path: str, class_name: str, output_file_name: str, time_spent: float, project_name: str) -> None: - logging.debug("Calculating time spent in output '%s' for class '%s'", output_file_name, class_name) - os.makedirs(os.path.dirname(time_spent_path), exist_ok=True) + def record_output_duration(self, time_duration_path: str, output_path: str, class_name: str, output_file_name: str, total_duration: int, project_name: str) -> None: + """Records the duration of output generation for the given class and output file""" + logging.debug("Recording duration for output '%s' in class '%s'", output_file_name, class_name) + os.makedirs(os.path.dirname(time_duration_path), exist_ok=True) - try: - with open(time_spent_path, "r") as file: - time_spent_dict = json.load(file) - except (FileNotFoundError, json.JSONDecodeError): - time_spent_dict = {} + time_duration_dict = self.load_json(time_duration_path) - project_data = time_spent_dict.setdefault(project_name, {}) - class_data = project_data.setdefault(class_name, {"total_time_spent": 0, "outputs": {}}) + project_data = time_duration_dict.setdefault(project_name, {}) + class_data = project_data.setdefault(class_name, {"total_duration": 0, "outputs": {}}) key_name = output_path.split(os.sep)[-1] + '_' + output_file_name - time_spent_rounded = round(time_spent, 2) - class_data["outputs"][key_name] = time_spent_rounded - class_data["total_time_spent"] = round(class_data["total_time_spent"] + time_spent_rounded, 2) - - try: - with open(time_spent_path, "w") as file: - json.dump(time_spent_dict, file, indent=4) - except Exception as e: - logging.error("Error while saving time spent data to '%s': %s", time_spent_path, e) - raise + total_duration_seconds = total_duration / 1_000_000_000 + duration_rounded = round(total_duration_seconds, 2) + class_data["outputs"][key_name] = duration_rounded + class_data["total_duration"] = round(class_data["total_duration"] + duration_rounded, 2) - def free_gpu_memory(self, chat_module: ChatModule) -> None: - import torch - try: - torch.cuda.empty_cache() - chat_module._unload() - + self.save_json(time_duration_path, time_duration_dict) except Exception as e: - logging.error(f"Error during memory cleanup: {e}") + logging.error("Error while recording duration for output '%s' in class '%s': %s", output_file_name, class_name, e) + raise def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: - model_info = { - "mpath": "/home/nfab/dist", - "lpath": "/home/nfab/dist/libs", - "model": "CodeLlama-7b-Instruct-hf-q4f16_1-MLC", - "lib": "CodeLlama-7b-Instruct-hf-q4f16_1-cuda.so" - } - prompts_path = os.path.join(output_path, "prompts.json") + """Main method for generating tests using the CODELLAMA tool""" + config = get_config() + api_url = config.get("codellama_api_url", "") + if not api_url: + raise ValueError("The 'codellama_api_url' key is not defined in the configuration file") + + # Define paths for storing scenario information (for prompt generation), importing data (to be extracted from source code), and recording time duration (for each output) + scenario_infos_path = os.path.join(output_path, "scenario_infos.json") imports_path = os.path.join(output_path, "imports.json") - - time_spent_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(output_path))), "reports", "codellama_time_spent.json") + time_duration_path = os.path.join( + os.path.dirname( + os.path.dirname( + os.path.dirname(output_path))), + "reports", "codellama_time_duration.json") # Save time duration data in the 'reports' folder, located next to the 'projects' folder project_name = scenario.project_name targets = scenario.targets jar_type = scenario.jar_type + # Fetch the source code paths for each class and save the associated scenario information and import data for class_name, methods in targets.items(): - file_path, branch = self.get_branch_info(input_jar, jar_type, class_name) - self.generate_prompts(prompts_path, class_name, methods, file_path) - self.get_imports(class_name, file_path, imports_path) + source_code_path, branch = self.fetch_source_code_branch(input_jar, jar_type, class_name) + self.save_scenario_infos(scenario_infos_path, class_name, methods, source_code_path) + self.save_imports(class_name, source_code_path, imports_path) - prompts_dict = self.load_json(prompts_path) + # Load scenario information and import data into dictionaries + scenario_infos_dict = self.load_json(scenario_infos_path) imports_dict = self.load_json(imports_path) - cm = self.create_chat_module(**model_info) - for class_name, prompts_list in prompts_dict.items(): + # Generate tests for each method in every class and save the results + for class_name, scenario_infos_list in scenario_infos_dict.items(): logging.debug("Generating tests for target methods in class '%s'", class_name) - for i, prompt in enumerate(prompts_list): - self._process_prompts(prompt, output_path, branch, class_name, imports=imports_dict.get(class_name, []), i=i, chat_module=cm, time_spent_path=time_spent_path, project_name=project_name) + for i, method_info in enumerate(scenario_infos_list): + messages = self.build_test_prompt(method_info, class_name) + test_template = method_info.get("test_template", "") + self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), i=i, time_duration_path=time_duration_path, project_name=project_name, api_url=api_url) os.remove(imports_path) # Remove imports file after generating tests - self.free_gpu_memory(cm) - def _process_prompts(self, prompt: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, chat_module: ChatModule, time_spent_path: str, project_name: str, num_outputs: int = 5) -> None: - import time + def _process_prompts(self, messages: str, test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, api_url=str, num_outputs: int = 5) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) - start_time = time.time() - output = self.generate_output(chat_module, prompt) - self.save_output(prompt, output, output_path, output_file_name) + output = self.generate_output(messages, api_url) + response = output.get("response", "Response not found.") + total_duration = output.get("total_duration", 0) + self.save_output(test_template, response, output_path, output_file_name) except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) finally: - self.reset_chat_module(chat_module) - end_time = time.time() - time_spent = end_time - start_time - self.calc_time_spent_per_output(time_spent_path, output_path, class_name, output_file_name, time_spent, project_name) + self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) - self.get_individual_tests(output_path, prompt, class_name, imports, i) + self.extract_individual_tests(output_path, test_template, class_name, imports, i) \ No newline at end of file From 5fcc7ec3fdce20bc18980c4aad57e58c1d6aaf3c Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Mon, 9 Dec 2024 21:51:33 -0300 Subject: [PATCH 30/69] change test template logic and prompt messages --- .../codellama_test_suite_generator.py | 100 ++++++++++++++---- 1 file changed, 80 insertions(+), 20 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 8284f4e5..0f79f26d 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -52,32 +52,84 @@ def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) - method_code = method_info.get("method_code", "") test_template = method_info.get("test_template", "") messages = [ + { + "role": "system", + "content":""" +You are a senior Java developer with expertise in JUnit testing. +Your only task is to generate JUnit test methods based on the provided Java class details, using Java syntax. + +Follow these guidelines: + +1. Provide only the JUnit test code, written in Java syntax, and nothing else. +2. Fully implement the test methods, including the actual test logic (assertEquals, assertTrue, etc.), starting with @Test. +3. Exclude any setup/teardown code or content outside the test method itself. +4. Do not include explanations, titles, Markdown text, backticks (```), or list formatting. +5. Use comment blocks /* */ or // for extra text, such as comments, titles, explanations, or any additional details within the code. +6. Ensure that the generated output is completely functional as code, compiles successfully, and runs without errors. +""" + }, { "role": "user", - "content": f"""You are a Java test generator for JUnit. You can only write tests for Java classes. - You will first receive information about the class, and then be asked to create tests based on that information. - When asked to generate test code, follow the next rules: - 1. Complete the provided test template with Java code only, starting your output with the annotation '@Test', using JUnit format and nothing else. - 2. Do not provide informations about the tests or your reasoning. You should only write the test code. - 3. Additional information, like titles, descriptions, comments, or any other text must be inside /* */ or // comments. - 4. The output should compile and run as is, without errors. - Below, you will find additional information regarding the Class under Test ({class_name}): - Attributes: - {class_fields} - - Constructor: - """ + "\n".join(constructor_codes) + f""" - - Target Method Under Test: - {method_code}""" + "content":""" +Below, you will find additional information regarding the Class Under Test (DFPBaseSample): +Class fields: +public String text; + +Constructors: +public DFPBaseSample(String text) { + this.text = text; + } + +Target Method Under Test: +public void cleanText() { + DFPBaseSample inst = new DFPBaseSample(text); + inst.normalizeWhiteSpace(); + inst.removeComments(); + this.text = inst.text; + }""" + f""" + +Here is the test template: +{test_template} + +Tests to replace the #TEST_METHODS# placeholder:""" }, { "role": "assistant", - "content": f"//Okay, I understand all rules and all details about the class {class_name}. Let's start generating tests for it." + "content": """ +@Test +public void test00() { + DFPBaseSample sample = new DFPBaseSample("This is a sample text"); + sample.cleanText(); + assertEquals("This is a sample text", sample.getText()); +} + +@Test +public void test01() { + DFPBaseSample sample1 = new DFPBaseSample("Hello World"); + DFPBaseSample sample2 = sample1; + sample1.cleanText(); + sample2.normalizeWhiteSpace(); + sample2.removeComments(); + assertEquals(sample1.getText(), sample2.getText()); +}""" }, { "role": "user", - "content": f"Now, complete the JUnit test template below:\n{test_template}" + "content":f""" +Below, you will find additional information regarding the Class Under Test ({class_name}): +Class fields: +""" + "\n".join(class_fields) + f""" + +Constructors: +""" + "\n".join(constructor_codes) + f""" + +Target Method Under Test: +{method_code} + +Here is the test template: +{test_template} + +Tests to replace the #TEST_METHODS# placeholder:""" } ] @@ -123,13 +175,15 @@ def generate_output(self, messages: str, api_url: str) -> Dict[str, str]: def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: - """Saves the output generated by the model to a file""" + """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" + llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") + filled_template = test_template.replace("#TEST_METHODS#", output) os.makedirs(llm_outputs_dir, exist_ok=True) with open(output_file_path, "w") as file: - file.write(test_template + output) + file.write(filled_template) def parse_code(self, source_code_path: str) -> tuple: @@ -223,6 +277,8 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods "import static org.junit.Assert.*;\n\n" "@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n" f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" + "#TEST_METHODS#\n" + "}" ) }) @@ -379,6 +435,10 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas logging.debug("Recording duration for output '%s' in class '%s'", output_file_name, class_name) os.makedirs(os.path.dirname(time_duration_path), exist_ok=True) + if not os.path.exists(time_duration_path): + with open(time_duration_path, "w") as file: + json.dump({}, file) + time_duration_dict = self.load_json(time_duration_path) project_data = time_duration_dict.setdefault(project_name, {}) From 083f1407a8c7572971b48276a9773717d4f36034 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 7 Jan 2025 21:53:21 -0300 Subject: [PATCH 31/69] update dependencies versions --- nimrod/tests/example/pom.xml | 2 +- requirements.txt | 7 ++----- setup.py | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/nimrod/tests/example/pom.xml b/nimrod/tests/example/pom.xml index 56e5109d..47dc02f7 100644 --- a/nimrod/tests/example/pom.xml +++ b/nimrod/tests/example/pom.xml @@ -14,7 +14,7 @@ junit junit - 4.12 + 4.13.1 test diff --git a/requirements.txt b/requirements.txt index 351a835e..dfc7cfe6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,5 @@ ---extra-index-url https://mlc.ai/wheels beautifulsoup4==4.12.3 -setuptools==68.2.2 -torch==2.2.1 +setuptools==70.0.0 tree_sitter==0.22.3 tree_sitter_java==0.21.0 -mlc-llm-nightly-cu122 -mlc-ai-nightly-cu122 +types-requests==2.32.0.20241016 \ No newline at end of file diff --git a/setup.py b/setup.py index 64337572..c0854b14 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,8 @@ def readme(): install_requires=[ 'argparse==1.4.0', 'beautifulsoup4==4.6.0', - 'pygithub==1.43.7', - 'gitpython==2.1.11' + 'pygithub==2.5.0', + 'gitpython==3.1.41' ], test_suite='nose.collector', tests_require=[ From cbedd6ac96578204c31005d1f679668bddf5b3a6 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 7 Jan 2025 21:54:02 -0300 Subject: [PATCH 32/69] fixed var types --- .../generators/codellama_test_suite_generator.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 0f79f26d..387ff055 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -2,7 +2,7 @@ import logging import os import requests -from typing import List, Dict +from typing import List, Dict, Union import tree_sitter_java as tsjava from tree_sitter import Language, Parser @@ -47,11 +47,11 @@ def save_json(self, file_path, content): def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) -> List[Dict[str, str]]: """Builds the test prompt messages for the given method information and class name""" class_name = full_class_name.split('.')[-1] - class_fields = method_info.get("class_fields", []) - constructor_codes = method_info.get("constructor_codes", []) + class_fields: Union[str, List[str]] = method_info.get("class_fields", []) + constructor_codes: Union[str, List[str]] = method_info.get("constructor_codes", []) method_code = method_info.get("method_code", "") test_template = method_info.get("test_template", "") - messages = [ + messages: List[Dict[str, str]] = [ { "role": "system", "content":""" @@ -135,7 +135,7 @@ def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) - return messages - def generate_output(self, messages: str, api_url: str) -> Dict[str, str]: + def generate_output(self, messages: List[Dict[str, str]], api_url: str) -> Dict[str, str]: """Generates the output using the API and returns the response and time duration""" url = api_url headers = {"Content-Type": "application/json"} @@ -499,14 +499,14 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s os.remove(imports_path) # Remove imports file after generating tests - def _process_prompts(self, messages: str, test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, api_url=str, num_outputs: int = 5) -> None: + def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, api_url=str, num_outputs: int = 5) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) output = self.generate_output(messages, api_url) response = output.get("response", "Response not found.") - total_duration = output.get("total_duration", 0) + total_duration = int(output.get("total_duration", "0") or 0) self.save_output(test_template, response, output_path, output_file_name) except Exception as e: logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) From 4e2a94b5a1a77411ec12762d705d863a15ecdf01 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 7 Jan 2025 22:12:52 -0300 Subject: [PATCH 33/69] use the current repo name --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cdc454c2..2f1db392 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,11 +38,12 @@ jobs: uses: stCarolas/setup-maven@v4.1 - name: Creating env-config.json run: | - cd /home/runner/work/SMAT/SMAT/nimrod/tests/ + repo_name=$(basename $GITHUB_REPOSITORY) + cd /home/runner/work/$repo_name/$repo_name/nimrod/tests/ java_path="/opt/hostedtoolcache/Java_Adopt_jdk/$(ls /opt/hostedtoolcache/Java_Adopt_jdk)/x64" contents="$(jq --arg java_path "$java_path" '.java_home=$java_path | .maven_home = "/opt/hostedtoolcache/maven/3.5.4/x64"' env-config.json)" echo "${contents}" > env-config.json - cd /home/runner/work/SMAT/SMAT + cd /home/runner/work/$repo_name/$repo_name - name: Test with pytest run: | pytest -k 'not test_general_behavior_study_semantic_conflict' From 0453c4d3599040f2e718f87aba62a4b44d2dd535 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 7 Jan 2025 22:45:59 -0300 Subject: [PATCH 34/69] fixed some stylistic issues --- .../codellama_test_suite_generator.py | 62 +++++++++---------- nimrod/tools/codellama.py | 5 +- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 387ff055..5ea496f9 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -11,22 +11,20 @@ from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator from nimrod.tests.utils import get_config + class CodellamaTestSuiteGenerator(TestSuiteGenerator): def get_generator_tool_name(self) -> str: return "CODELLAMA" - def _get_test_suite_class_paths(self, path: str) -> List[str]: paths: list[str] = [] for root, _, files in os.walk(path): paths.extend(os.path.join(root, file) for file in files if file.endswith(".java")) return paths - def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] - def load_json(self, file_path): """Loads a JSON file and return its content as a dictionary""" @@ -36,14 +34,12 @@ def load_json(self, file_path): except json.JSONDecodeError: content = {} return content - def save_json(self, file_path, content): """Saves a dictionary as a JSON file""" with open(file_path, "w") as file: json.dump(content, file, indent=4) - def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) -> List[Dict[str, str]]: """Builds the test prompt messages for the given method information and class name""" class_name = full_class_name.split('.')[-1] @@ -118,7 +114,7 @@ def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) - "content":f""" Below, you will find additional information regarding the Class Under Test ({class_name}): Class fields: -""" + "\n".join(class_fields) + f""" +""" + "\n".join(class_fields) + """ Constructors: """ + "\n".join(constructor_codes) + f""" @@ -160,7 +156,7 @@ def generate_output(self, messages: List[Dict[str, str]], api_url: str) -> Dict[ "response": result.get("message", {}).get("content", "Response not found."), "total_duration": result.get("total_duration", 0) } - + except requests.exceptions.Timeout: logging.error("Exceeded total timeout of %s seconds. Aborting.", timeout_seconds) return {"error": "Total timeout exceeded"} @@ -173,10 +169,9 @@ def generate_output(self, messages: List[Dict[str, str]], api_url: str) -> Dict[ logging.error("JSON decoding error: %s", response.text) return {"error": "Response decoding error"} - def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" - + llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") filled_template = test_template.replace("#TEST_METHODS#", output) @@ -185,7 +180,6 @@ def save_output(self, test_template: str, output: str, dir: str, output_file_nam with open(output_file_path, "w") as file: file.write(filled_template) - def parse_code(self, source_code_path: str) -> tuple: """Parses the Java source code using the Tree-sitter parser and return the language, source code, and generated AST""" JAVA_LANGUAGE = Language(tsjava.language()) @@ -194,15 +188,16 @@ def parse_code(self, source_code_path: str) -> tuple: source_code = f.read() tree = parser.parse(bytes(source_code, "utf8")) return JAVA_LANGUAGE, source_code, tree - def extract_snippet(self, source_code: str, start_byte: int, end_byte: int) -> str: """Extracts a snippet of code from the source code using the start and end byte offsets""" return source_code[start_byte:end_byte] - def extract_class_info(self, source_code_path: str, full_method_name: str, full_class_name: str) -> tuple: - """Extracts the class fields, constructor, and body of the method under test from the source code using the generated AST to query the information""" + """ + Extracts the class fields, constructor, and body of the method under test from the source code + using the generated AST to query the information + """ try: class_name = full_class_name.split('.')[-1] method_name = full_method_name.split('(')[0] @@ -237,7 +232,7 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ for node, capture_name in captures: captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) - + if capture_name == "field_declaration": class_fields.append(captured_text) elif capture_name == "constructor_declaration": @@ -287,7 +282,6 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods self.save_json(scenario_infos_path, scenario_infos_dict) - def save_imports(self, class_name: str, source_code_path: str, imports_path: str) -> None: """Extracts import statements from the Java source code and stores them in a JSON file""" JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) @@ -318,7 +312,6 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str self.save_json(imports_path, imports_dict) - def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int) -> None: """Extracts individual tests from the generated test suite and saves them to separate files""" llm_outputs_path = os.path.join(output_path, "llm_outputs") @@ -372,7 +365,6 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: counter += 1 - def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) -> Dict[str, str]: """Finds the source code files for the given class name in the specified JAR path""" if not os.path.exists(input_jar): @@ -382,15 +374,15 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) path_parts = input_jar.split(os.sep) if jar_type != "transformed" and jar_type != "original": raise ValueError("The provided path does not contain the expected jar type (transformed/original)") - + type_index = path_parts.index(jar_type) base_path = os.path.join("/", *path_parts[:type_index], "source") if not os.path.exists(base_path): raise FileNotFoundError(f"The base path '{base_path}' does not exist") - + java_files = {key: "" for key in ["base", "left", "right", "merge"]} - + for root, _, files in os.walk(base_path): java_candidates = [file for file in files if file.endswith(".java")] @@ -412,7 +404,6 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) raise FileNotFoundError(f"The following source code files were not found: {', '.join(missing_files)}") return java_files - def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: str) -> tuple: """Retrieves the source code path and branch for the given JAR path""" @@ -420,7 +411,7 @@ def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: st branches = ["base", "left", "right", "merge"] branch = next((b for b in branches if b in input_jar), None) - + if branch: source_code_path = source_code_paths.get(branch) if source_code_path: @@ -429,8 +420,8 @@ def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: st available_branches = ", ".join(branches) raise ValueError(f"No corresponding branch found in '{input_jar}'. Available branches: {available_branches}") - - def record_output_duration(self, time_duration_path: str, output_path: str, class_name: str, output_file_name: str, total_duration: int, project_name: str) -> None: + def record_output_duration(self, time_duration_path: str, output_path: str, class_name: str, + output_file_name: str, total_duration: int, project_name: str) -> None: """Records the duration of output generation for the given class and output file""" logging.debug("Recording duration for output '%s' in class '%s'", output_file_name, class_name) os.makedirs(os.path.dirname(time_duration_path), exist_ok=True) @@ -458,7 +449,6 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas logging.error("Error while recording duration for output '%s' in class '%s': %s", output_file_name, class_name, e) raise - def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: """Main method for generating tests using the CODELLAMA tool""" config = get_config() @@ -466,14 +456,15 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s if not api_url: raise ValueError("The 'codellama_api_url' key is not defined in the configuration file") - # Define paths for storing scenario information (for prompt generation), importing data (to be extracted from source code), and recording time duration (for each output) + # Define paths for storing scenario information (for prompt generation), + # importing data (to be extracted from source code), and recording time duration (for each output) scenario_infos_path = os.path.join(output_path, "scenario_infos.json") imports_path = os.path.join(output_path, "imports.json") + # Save time duration data in the 'reports' folder, located next to the 'projects' folder time_duration_path = os.path.join( os.path.dirname( os.path.dirname( - os.path.dirname(output_path))), - "reports", "codellama_time_duration.json") # Save time duration data in the 'reports' folder, located next to the 'projects' folder + os.path.dirname(output_path))), "reports", "codellama_time_duration.json") project_name = scenario.project_name targets = scenario.targets @@ -484,7 +475,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s source_code_path, branch = self.fetch_source_code_branch(input_jar, jar_type, class_name) self.save_scenario_infos(scenario_infos_path, class_name, methods, source_code_path) self.save_imports(class_name, source_code_path, imports_path) - + # Load scenario information and import data into dictionaries scenario_infos_dict = self.load_json(scenario_infos_path) imports_dict = self.load_json(imports_path) @@ -495,11 +486,16 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s for i, method_info in enumerate(scenario_infos_list): messages = self.build_test_prompt(method_info, class_name) test_template = method_info.get("test_template", "") - self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), i=i, time_duration_path=time_duration_path, project_name=project_name, api_url=api_url) + self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, + branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), + i=i, time_duration_path=time_duration_path, project_name=project_name, api_url=api_url) - os.remove(imports_path) # Remove imports file after generating tests + # Remove imports file after generating tests + os.remove(imports_path) - def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, api_url=str, num_outputs: int = 5) -> None: + def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, + class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, + api_url=str, num_outputs: int = 5) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: @@ -513,4 +509,4 @@ def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, o finally: self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) - self.extract_individual_tests(output_path, test_template, class_name, imports, i) \ No newline at end of file + self.extract_individual_tests(output_path, test_template, class_name, imports, i) diff --git a/nimrod/tools/codellama.py b/nimrod/tools/codellama.py index c6021b6b..c1aa1462 100644 --- a/nimrod/tools/codellama.py +++ b/nimrod/tools/codellama.py @@ -1,8 +1,9 @@ import os -from nimrod.tools.suite_generator import Suite, SuiteGenerator +from nimrod.tools.suite_generator import SuiteGenerator from nimrod.utils import get_class_files + class Codellama(SuiteGenerator): def _get_tool_name(self): @@ -18,4 +19,4 @@ def _test_classes(self): return classes def _get_suite_dir(self): - return os.path.join(self.suite_dir, 'codellama-tests') \ No newline at end of file + return os.path.join(self.suite_dir, 'codellama-tests') From 344d54de407b7e4eaf0f5df3d1491ca4002ee621 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Wed, 12 Mar 2025 20:07:00 -0300 Subject: [PATCH 35/69] =?UTF-8?q?parametriza=C3=A7=C3=A3o=20dos=20dados=20?= =?UTF-8?q?da=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codellama_test_suite_generator.py | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 5ea496f9..dbe4e675 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -3,6 +3,7 @@ import os import requests from typing import List, Dict, Union +import re import tree_sitter_java as tsjava from tree_sitter import Language, Parser @@ -131,22 +132,25 @@ def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) - return messages - def generate_output(self, messages: List[Dict[str, str]], api_url: str) -> Dict[str, str]: + def generate_output(self, messages: List[Dict[str, str]], api_params: Dict[str, str]) -> Dict[str, str]: """Generates the output using the API and returns the response and time duration""" - url = api_url + timeout_seconds = api_params["codellama"].get("timeout_seconds", 60) + api_url = api_params["codellama"].get("api_url", "http://localhost:11434/api/chat") + temperature = api_params["codellama"].get("temperature", 0) + model = api_params["codellama"].get("model", "codellama:70b") + headers = {"Content-Type": "application/json"} payload = { - "model": "codellama:70b", + "model": model, "messages": messages, "stream": False, - "options": {"temperature": 0.7}, + "options": {"temperature": temperature}, } logging.debug("Starting API request...") - timeout_seconds = 500 try: - response = requests.post(url, headers=headers, json=payload, timeout=timeout_seconds) + response = requests.post(api_url, headers=headers, json=payload, timeout=timeout_seconds) response.raise_for_status() logging.debug("Request successful. Status: %s", response.status_code) @@ -171,6 +175,18 @@ def generate_output(self, messages: List[Dict[str, str]], api_url: str) -> Dict[ def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" + # Remove tags and Java code blocks + output = re.sub(r'.*?|```java|```', '', output, flags=re.DOTALL) + + # Remove lines starting with "number. " (e.g., "1. public void test() {...}") + output = re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) + + # Look for @Before, @BeforeClass, or @BeforeAll first; fallback to @Test if none are found + markers = ["@Before", "@BeforeClass", "@BeforeAll", "@Test"] + index = min((output.find(marker) for marker in markers if marker in output), default=-1) + + # Keep only the content starting from the first found annotation + output = output[index:] if index != -1 else output llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") @@ -332,7 +348,9 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: return before_block, test_block for file in os.listdir(llm_outputs_path): - if file.endswith(".txt") and file.startswith(f"{i}"): + # Avoid processing the wrong files (from different classes) + pattern = rf"^{i}\d+_(left|right)_{re.escape(class_name.split('.')[-1])}\.txt$" + if re.match(pattern, file): source_code_path = os.path.join(llm_outputs_path, file) JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) @@ -452,9 +470,12 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: """Main method for generating tests using the CODELLAMA tool""" config = get_config() - api_url = config.get("codellama_api_url", "") - if not api_url: - raise ValueError("The 'codellama_api_url' key is not defined in the configuration file") + api_params = config.get("api_params", {}) + if not api_params: + raise ValueError("The 'api_params' section is missing from the configuration file") + + if not api_params.get("codellama"): + raise ValueError("The 'codellama' section is missing from the 'api_params' configuration") # Define paths for storing scenario information (for prompt generation), # importing data (to be extracted from source code), and recording time duration (for each output) @@ -488,19 +509,16 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s test_template = method_info.get("test_template", "") self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), - i=i, time_duration_path=time_duration_path, project_name=project_name, api_url=api_url) - - # Remove imports file after generating tests - os.remove(imports_path) + i=i, time_duration_path=time_duration_path, project_name=project_name, api_params=api_params) def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, - api_url=str, num_outputs: int = 5) -> None: + api_params: Dict[str, str], num_outputs: int = 1) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) - output = self.generate_output(messages, api_url) + output = self.generate_output(messages, api_params) response = output.get("response", "Response not found.") total_duration = int(output.get("total_duration", "0") or 0) self.save_output(test_template, response, output_path, output_file_name) From e132accfbdb3a67e64989fce285ba04a515285ef Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Wed, 12 Mar 2025 20:09:46 -0300 Subject: [PATCH 36/69] creates a file to store compilation outputs --- .../generators/test_suite_generator.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 72adc990..dc0bbef0 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -4,6 +4,7 @@ from time import time from typing import List from subprocess import CalledProcessError +import json from nimrod.tests.utils import get_config from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis @@ -66,13 +67,37 @@ def _get_test_suite_class_paths(self, test_suite_path: str) -> List[str]: def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: pass + def _update_compilation_results(self, test_suite_path: str, java_file: str, output: str) -> None: + """Updates the compilation results file with the output of the compilation of a test suite class.""" + COMPILATION_LOG_FILE = "compilation_results.json" + + if path.exists(COMPILATION_LOG_FILE): + with open(COMPILATION_LOG_FILE, "r", encoding="utf-8") as f: + try: + compilation_results = json.load(f) + except json.JSONDecodeError: + compilation_results = {} + else: + compilation_results = {} + + test_suite_entry = compilation_results.setdefault(test_suite_path, {"compilation_output": {}}) + safe_output = output.strip() if output.strip() else "" + test_suite_entry["compilation_output"][java_file] = safe_output + + with open(COMPILATION_LOG_FILE, "w", encoding="utf-8") as f: + json.dump(compilation_results, f, indent=4) + def _compile_test_suite(self, input_jar: str, test_suite_path: str, extra_class_path: List[str] = []) -> str: compiled_classes_path = path.join(test_suite_path, 'classes') class_path = generate_classpath([input_jar, test_suite_path, compiled_classes_path, JUNIT, HAMCREST] + extra_class_path) for java_file in self._get_test_suite_class_paths(test_suite_path): + output = "" try: self._java.exec_javac(java_file, test_suite_path, None, None, '-classpath', class_path, '-d', compiled_classes_path) - except CalledProcessError: + except CalledProcessError as e: + output = (e.stdout or b'').decode("utf-8", errors="ignore") + (e.stderr or b'').decode("utf-8", errors="ignore") logging.error("Error while compiling %s", java_file) + self._update_compilation_results(test_suite_path, java_file, output) + return class_path From db8be5ce0124ab08956ed03d37919204763346c6 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 25 Mar 2025 15:57:25 -0300 Subject: [PATCH 37/69] fixed get_config() function to support dicts inside dicts --- .../generators/codellama_test_suite_generator.py | 6 +++--- nimrod/tests/utils.py | 12 ++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index dbe4e675..a7d8cfae 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -2,7 +2,7 @@ import logging import os import requests -from typing import List, Dict, Union +from typing import List, Dict, Union, Any import re import tree_sitter_java as tsjava @@ -132,7 +132,7 @@ def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) - return messages - def generate_output(self, messages: List[Dict[str, str]], api_params: Dict[str, str]) -> Dict[str, str]: + def generate_output(self, messages: List[Dict[str, str]], api_params: Any) -> Dict[str, str]: """Generates the output using the API and returns the response and time duration""" timeout_seconds = api_params["codellama"].get("timeout_seconds", 60) api_url = api_params["codellama"].get("api_url", "http://localhost:11434/api/chat") @@ -513,7 +513,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, - api_params: Dict[str, str], num_outputs: int = 1) -> None: + api_params: Any, num_outputs: int = 1) -> None: for j in range(num_outputs): output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" try: diff --git a/nimrod/tests/utils.py b/nimrod/tests/utils.py index 7386a1b0..d3726a3b 100644 --- a/nimrod/tests/utils.py +++ b/nimrod/tests/utils.py @@ -2,18 +2,14 @@ import os import json import shutil -from typing import Dict +from typing import Dict, Any PATH = os.path.dirname(os.path.abspath(__file__)) -def get_config() -> "Dict[str, str]": - config: "Dict[str, str]" = dict() - - with open(os.path.join(PATH, os.sep.join(['env-config.json'])), 'r') as j: - config = json.loads(j.read()) - - return config +def get_config() -> Dict[str, Any]: + with open(os.path.join(PATH, "env-config.json"), 'r') as j: + return json.load(j) def calculator_project_dir(): From 9d1f85eb2e72597cc792558c0528d60ee287050c Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 15 Apr 2025 11:27:14 -0300 Subject: [PATCH 38/69] move functions to utils --- nimrod/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nimrod/utils.py b/nimrod/utils.py index 6280f380..b51388d8 100644 --- a/nimrod/utils.py +++ b/nimrod/utils.py @@ -1,4 +1,5 @@ import os +import json def get_class_files(path): @@ -32,3 +33,19 @@ def package_to_dir(package): def dir_to_package(directory): return directory.replace(os.sep, '.') + + +def load_json(file_path): + """Loads a JSON file and return its content as a dictionary""" + with open(file_path, "r") as file: + try: + content = json.load(file) + except json.JSONDecodeError: + content = {} + return content + + +def save_json(file_path, content): + """Saves a dictionary as a JSON file""" + with open(file_path, "w") as file: + json.dump(content, file, indent=4) From 038f41be522d9d61dc4c0461cda8148b528d5914 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 15 Apr 2025 11:29:51 -0300 Subject: [PATCH 39/69] created api class --- .../codellama_test_suite_generator.py | 350 +++++++++--------- 1 file changed, 181 insertions(+), 169 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index a7d8cfae..85d639a9 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -2,7 +2,8 @@ import logging import os import requests -from typing import List, Dict, Union, Any +from typing import List, Dict, Union, Any, Optional +from itertools import combinations import re import tree_sitter_java as tsjava @@ -11,172 +12,156 @@ from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator from nimrod.tests.utils import get_config +from nimrod.utils import load_json, save_json -class CodellamaTestSuiteGenerator(TestSuiteGenerator): +class Api(): - def get_generator_tool_name(self) -> str: - return "CODELLAMA" - - def _get_test_suite_class_paths(self, path: str) -> List[str]: - paths: list[str] = [] - for root, _, files in os.walk(path): - paths.extend(os.path.join(root, file) for file in files if file.endswith(".java")) - return paths + def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model: str) -> None: + self.api_url = api_url + self.timeout_seconds = timeout_seconds + self.temperature = temperature + self.model = model + self.headers = {"Content-Type": "application/json"} + self.payload = { + "model": self.model, + "messages": [], + "stream": False, + "options": {"temperature": self.temperature, "num_ctx": 16384}, + } + self.branch = None # Initialize branch as None + + def set_branch(self, branch: str) -> None: + """Sets the branch to be used in the API requests.""" + self.branch = branch - def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: - return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] + def post(self, payload: Dict[str, Any]) -> Dict[str, Any]: + try: + response: requests.Response = requests.post(self.api_url, headers=self.headers, json=payload, timeout=self.timeout_seconds) + response.raise_for_status() + logging.debug("Request successful. Status: %s", response.status_code) + return response.json() + except requests.exceptions.Timeout: + logging.error("Request timed out.") + return {"error": "Request timed out"} + except requests.exceptions.RequestException as e: + logging.error(f"Request error: {e}") + return {"error": "Request error"} + except json.JSONDecodeError: + logging.error("JSON decoding error.") + return {"error": "JSON decoding error"} + + def set_payload_messages(self, messages: List[Dict[str, str]]) -> None: + self.payload["messages"] = messages - def load_json(self, file_path): - """Loads a JSON file and return its content as a dictionary""" - with open(file_path, "r") as file: - try: - content = json.load(file) - except json.JSONDecodeError: - content = {} - return content - - def save_json(self, file_path, content): - """Saves a dictionary as a JSON file""" - with open(file_path, "w") as file: - json.dump(content, file, indent=4) - - def build_test_prompt(self, method_info: Dict[str, str], full_class_name: str) -> List[Dict[str, str]]: - """Builds the test prompt messages for the given method information and class name""" + def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Union[str, int]]: + try: + logging.debug("Generating output...") + self.set_payload_messages(messages) + response = self.post(self.payload) + logging.debug("Response: %s", response) + return { + "response": response.get("message", {}).get("content", "Response not found."), + "total_duration": response.get("total_duration", self.timeout_seconds) + } + except Exception as e: + logging.error(f"Error generating output: {e}") + return {"error": "Output generation error"} + + def generate_messages_list(self, method_info: Dict[str, str], full_class_name: str, branch: str, output_path: str) -> List[List[Dict[str, str]]]: + """ + Generates the messages for the API requests. + Each list of messages contains different information about the method under test. + """ + self.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] class_fields: Union[str, List[str]] = method_info.get("class_fields", []) constructor_codes: Union[str, List[str]] = method_info.get("constructor_codes", []) method_code = method_info.get("method_code", "") - test_template = method_info.get("test_template", "") - messages: List[Dict[str, str]] = [ - { - "role": "system", - "content":""" -You are a senior Java developer with expertise in JUnit testing. -Your only task is to generate JUnit test methods based on the provided Java class details, using Java syntax. - -Follow these guidelines: - -1. Provide only the JUnit test code, written in Java syntax, and nothing else. -2. Fully implement the test methods, including the actual test logic (assertEquals, assertTrue, etc.), starting with @Test. -3. Exclude any setup/teardown code or content outside the test method itself. -4. Do not include explanations, titles, Markdown text, backticks (```), or list formatting. -5. Use comment blocks /* */ or // for extra text, such as comments, titles, explanations, or any additional details within the code. -6. Ensure that the generated output is completely functional as code, compiles successfully, and runs without errors. + + # Define the base system message + system_message = { + "role": "system", + "content": """You are a senior Java developer with expertise in JUnit testing. +Your task is to provide JUnit tests for the given method in the class under test, considering the changes introduced in the left and right branches. +You have to answer with the test code only, inside code blocks (```). +The tests should start with @Test. """ - }, + } + + user_init_msg = { + "role": "user", + "content": f"""Here is the context of the method under test in the class {class_name} on the {branch} branch:""" + } + + # Define the user message templates + user_msg_templates = [ { "role": "user", - "content":""" -Below, you will find additional information regarding the Class Under Test (DFPBaseSample): -Class fields: -public String text; - -Constructors: -public DFPBaseSample(String text) { - this.text = text; - } - -Target Method Under Test: -public void cleanText() { - DFPBaseSample inst = new DFPBaseSample(text); - inst.normalizeWhiteSpace(); - inst.removeComments(); - this.text = inst.text; - }""" + f""" - -Here is the test template: -{test_template} - -Tests to replace the #TEST_METHODS# placeholder:""" - }, - { - "role": "assistant", - "content": """ -@Test -public void test00() { - DFPBaseSample sample = new DFPBaseSample("This is a sample text"); - sample.cleanText(); - assertEquals("This is a sample text", sample.getText()); -} - -@Test -public void test01() { - DFPBaseSample sample1 = new DFPBaseSample("Hello World"); - DFPBaseSample sample2 = sample1; - sample1.cleanText(); - sample2.normalizeWhiteSpace(); - sample2.removeComments(); - assertEquals(sample1.getText(), sample2.getText()); -}""" + "content": f"""Class fields: +""" + "\n".join(class_fields) + "" }, { "role": "user", - "content":f""" -Below, you will find additional information regarding the Class Under Test ({class_name}): -Class fields: -""" + "\n".join(class_fields) + """ - -Constructors: -""" + "\n".join(constructor_codes) + f""" - -Target Method Under Test: -{method_code} - -Here is the test template: -{test_template} - -Tests to replace the #TEST_METHODS# placeholder:""" - } + "content": f"""Constructors: +""" + "\n".join(constructor_codes) + "" + }, ] - return messages + user_method_ctx_msg = { + "role": "user", + "content": f"""Target Method Under Test: +{method_code} - def generate_output(self, messages: List[Dict[str, str]], api_params: Any) -> Dict[str, str]: - """Generates the output using the API and returns the response and time duration""" - timeout_seconds = api_params["codellama"].get("timeout_seconds", 60) - api_url = api_params["codellama"].get("api_url", "http://localhost:11434/api/chat") - temperature = api_params["codellama"].get("temperature", 0) - model = api_params["codellama"].get("model", "codellama:70b") - - headers = {"Content-Type": "application/json"} - payload = { - "model": model, - "messages": messages, - "stream": False, - "options": {"temperature": temperature}, +Now generate tests for the method under test, considering the given context. +Write all tests inside code blocks (```), and start each test with @Test.""" } - logging.debug("Starting API request...") + # Build the list of lists of messages + messages_lists: List[List[Dict[str, str]]] = [] + for r in range(1, len(user_msg_templates) + 1): + for user_msgs_combination in combinations(user_msg_templates, r): + messages_list = [system_message, user_init_msg, *user_msgs_combination, user_method_ctx_msg] + messages_lists.append(messages_list) - try: - response = requests.post(api_url, headers=headers, json=payload, timeout=timeout_seconds) - response.raise_for_status() - logging.debug("Request successful. Status: %s", response.status_code) + return messages_lists - result = response.json() - return { - "response": result.get("message", {}).get("content", "Response not found."), - "total_duration": result.get("total_duration", 0) - } +class CodellamaTestSuiteGenerator(TestSuiteGenerator): - except requests.exceptions.Timeout: - logging.error("Exceeded total timeout of %s seconds. Aborting.", timeout_seconds) - return {"error": "Total timeout exceeded"} + def get_generator_tool_name(self) -> str: + return "CODELLAMA" - except requests.exceptions.RequestException as e: - logging.error("Request error: %s", e) - return {"error": "Request error"} + def _get_test_suite_class_paths(self, path: str) -> List[str]: + paths: List[str] = [] + for root, _, files in os.walk(path): + paths.extend(os.path.join(root, file) for file in files if file.endswith(".java")) + return paths - except json.JSONDecodeError: - logging.error("JSON decoding error: %s", response.text) - return {"error": "Response decoding error"} + def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: + return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] + + def generate_outputs(self, messages: List[Dict[str, str]]) -> Dict[str, str]: + """Generates the output using the API and returns the response and time duration""" + try: + output = self.api.generate_output(messages) + if isinstance(output, dict): + return output + else: + logging.error(f"Unexpected output format: {output}") + return {"error": "Unexpected output format"} + except Exception as e: + logging.error(f"Error generating outputs: {e}") + return {"error": "Output generation error"} def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" - # Remove tags and Java code blocks - output = re.sub(r'.*?|```java|```', '', output, flags=re.DOTALL) + # Remove content between tags + output = re.sub(r'.*?', '', output, flags=re.DOTALL) + + # Extract only the content inside ``` blocks (excluding the ``` markers) + matches = re.findall(r'```(?:\w+)?\n?(.*?)```', output, flags=re.DOTALL) + output = '\n'.join(matches).strip() # Remove lines starting with "number. " (e.g., "1. public void test() {...}") output = re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) @@ -265,7 +250,7 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods: List[str], source_code_path: str) -> None: """Stores relevant scenario information (for each class and method) in a JSON file""" if os.path.exists(scenario_infos_path): - scenario_infos_dict = self.load_json(scenario_infos_path) + scenario_infos_dict = load_json(scenario_infos_path) else: scenario_infos_dict = {} @@ -282,11 +267,8 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods 'constructor_codes': constructor_codes if constructor_codes else [], 'method_code': method_code if method_code else "", 'test_template': ( - "import org.junit.FixMethodOrder;\n" "import org.junit.Test;\n" - "import org.junit.runners.MethodSorters;\n" "import static org.junit.Assert.*;\n\n" - "@FixMethodOrder(MethodSorters.NAME_ASCENDING)\n" f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" "#TEST_METHODS#\n" "}" @@ -296,7 +278,7 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods except Exception as e: logging.error("Error while saving scenario information for method '%s' in class '%s': %s", method, class_name, e) - self.save_json(scenario_infos_path, scenario_infos_dict) + save_json(scenario_infos_path, scenario_infos_dict) def save_imports(self, class_name: str, source_code_path: str, imports_path: str) -> None: """Extracts import statements from the Java source code and stores them in a JSON file""" @@ -310,7 +292,7 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str captures = query.captures(tree.root_node) if os.path.exists(imports_path): - imports_dict = self.load_json(imports_path) + imports_dict = load_json(imports_path) else: imports_dict = {} @@ -326,7 +308,7 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str package_name = captured_text.split()[1].rstrip(';') class_imports.append(f'import {package_name}.*;\n') - self.save_json(imports_path, imports_dict) + save_json(imports_path, imports_dict) def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int) -> None: """Extracts individual tests from the generated test suite and saves them to separate files""" @@ -383,8 +365,25 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: counter += 1 - def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) -> Dict[str, str]: + def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str, project_name: str) -> Dict[str, str]: """Finds the source code files for the given class name in the specified JAR path""" + """ + base_path = os.path.join("/mnt/c/Users/natha/Downloads/tests", project_name) + if not os.path.exists(base_path): + raise FileNotFoundError(f"The base path '{base_path}' does not exist") + + # Converter class_name para path relativo dentro do diretório base + relative_path = class_name.replace(".", "/") + ".java" + + # Procurar o arquivo no diretório base + for root, _, files in os.walk(base_path): + if relative_path.split('/')[-1] in files: + full_path = os.path.join(root, relative_path.split('/')[-1]) + logging.debug("Source code found for class '%s' in '%s'", class_name, full_path) + return {key: full_path for key in ["base", "left", "right", "merge"]} + + raise FileNotFoundError(f"Source code for class '{class_name}' not found in '{base_path}'") + """ if not os.path.exists(input_jar): logging.error("The provided jar path '%s' does not exist", input_jar) raise FileNotFoundError(f"The provided path '{input_jar}' does not exist") @@ -423,9 +422,9 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str) return java_files - def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: str) -> tuple: + def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: str, project_name: str) -> tuple: """Retrieves the source code path and branch for the given JAR path""" - source_code_paths = self.find_source_code_paths(input_jar, jar_type, class_name.split('.')[-1]) + source_code_paths = self.find_source_code_paths(input_jar, jar_type, class_name.split('.')[-1], project_name) branches = ["base", "left", "right", "merge"] branch = next((b for b in branches if b in input_jar), None) @@ -433,7 +432,7 @@ def fetch_source_code_branch(self, input_jar: str, jar_type: str, class_name: st if branch: source_code_path = source_code_paths.get(branch) if source_code_path: - return source_code_path, branch + return source_code_path, branch, source_code_paths available_branches = ", ".join(branches) raise ValueError(f"No corresponding branch found in '{input_jar}'. Available branches: {available_branches}") @@ -448,7 +447,7 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas with open(time_duration_path, "w") as file: json.dump({}, file) - time_duration_dict = self.load_json(time_duration_path) + time_duration_dict = load_json(time_duration_path) project_data = time_duration_dict.setdefault(project_name, {}) class_data = project_data.setdefault(class_name, {"total_duration": 0, "outputs": {}}) @@ -462,13 +461,12 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas class_data["total_duration"] = round(class_data["total_duration"] + duration_rounded, 2) try: - self.save_json(time_duration_path, time_duration_dict) + save_json(time_duration_path, time_duration_dict) except Exception as e: logging.error("Error while recording duration for output '%s' in class '%s': %s", output_file_name, class_name, e) raise def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: - """Main method for generating tests using the CODELLAMA tool""" config = get_config() api_params = config.get("api_params", {}) if not api_params: @@ -477,6 +475,14 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s if not api_params.get("codellama"): raise ValueError("The 'codellama' section is missing from the 'api_params' configuration") + codellama_params = api_params.get("codellama", {}) + self.api = Api( + api_url=codellama_params.get("api_url", "http://localhost:11434/api/chat"), + timeout_seconds=codellama_params.get("timeout_seconds", 60), + temperature=codellama_params.get("temperature", 0), + model=codellama_params.get("model", "codellama:70b") + ) + # Define paths for storing scenario information (for prompt generation), # importing data (to be extracted from source code), and recording time duration (for each output) scenario_infos_path = os.path.join(output_path, "scenario_infos.json") @@ -493,38 +499,44 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s # Fetch the source code paths for each class and save the associated scenario information and import data for class_name, methods in targets.items(): - source_code_path, branch = self.fetch_source_code_branch(input_jar, jar_type, class_name) + source_code_path, branch, source_paths = self.fetch_source_code_branch(input_jar, jar_type, class_name, project_name) self.save_scenario_infos(scenario_infos_path, class_name, methods, source_code_path) self.save_imports(class_name, source_code_path, imports_path) # Load scenario information and import data into dictionaries - scenario_infos_dict = self.load_json(scenario_infos_path) - imports_dict = self.load_json(imports_path) + scenario_infos_dict = load_json(scenario_infos_path) + imports_dict = load_json(imports_path) # Generate tests for each method in every class and save the results for class_name, scenario_infos_list in scenario_infos_dict.items(): logging.debug("Generating tests for target methods in class '%s'", class_name) for i, method_info in enumerate(scenario_infos_list): - messages = self.build_test_prompt(method_info, class_name) + messages = self.api.generate_messages_list(method_info, class_name, branch, output_path) test_template = method_info.get("test_template", "") self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), - i=i, time_duration_path=time_duration_path, project_name=project_name, api_params=api_params) + i=i, time_duration_path=time_duration_path, project_name=project_name) def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, - api_params: Any, num_outputs: int = 1) -> None: + num_outputs: int = 1) -> None: for j in range(num_outputs): - output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}" - try: - logging.debug("Generating output %d%d in branch \"%s\"", i, j, branch) - output = self.generate_output(messages, api_params) - response = output.get("response", "Response not found.") - total_duration = int(output.get("total_duration", "0") or 0) - self.save_output(test_template, response, output_path, output_file_name) - except Exception as e: - logging.error("Error while generating output %d%d in branch \"%s\": %s", i, j, branch, e) - finally: - self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) + for k, message_set in enumerate(messages): + output_file_name = f"{i}{j}{k}_{branch}_{class_name.split('.')[-1]}" + self._process_single_prompt(message_set, test_template, output_path, branch, class_name, imports, i, j, k, time_duration_path, project_name, output_file_name) + + def _process_single_prompt(self, message_set: List[Dict[str, str]], test_template: str, output_path: str, branch: str, + class_name: str, imports: List[str], i: int, j: int, k: int, time_duration_path: str, + project_name: str, output_file_name: str) -> None: + try: + logging.debug("Processing output %d%d%d in branch \"%s\"", i, j, k, branch) + output = self.generate_outputs(message_set) + response = output.get("response", "Response not found.") + total_duration = int(output.get("total_duration", "0") or 0) + self.save_output(test_template, response, output_path, output_file_name) + except Exception as e: + logging.error("Error while processing output %d%d%d in branch \"%s\": %s", i, j, k, branch, e) + finally: + self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) self.extract_individual_tests(output_path, test_template, class_name, imports, i) From e8ad4445053d16784c7c7ebc03b643df29820482 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 15 Apr 2025 14:43:09 -0300 Subject: [PATCH 40/69] fixed scenario infos format and remove useless function --- .../codellama_test_suite_generator.py | 102 ++++++++---------- 1 file changed, 47 insertions(+), 55 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 85d639a9..72d8dd0d 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -15,7 +15,7 @@ from nimrod.utils import load_json, save_json -class Api(): +class Api: def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model: str) -> None: self.api_url = api_url @@ -29,15 +29,21 @@ def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model "stream": False, "options": {"temperature": self.temperature, "num_ctx": 16384}, } - self.branch = None # Initialize branch as None + self.branch = None def set_branch(self, branch: str) -> None: """Sets the branch to be used in the API requests.""" self.branch = branch def post(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """Sends a POST request to the API and handles the response.""" try: - response: requests.Response = requests.post(self.api_url, headers=self.headers, json=payload, timeout=self.timeout_seconds) + response: requests.Response = requests.post( + self.api_url, + headers=self.headers, + json=payload, + timeout=self.timeout_seconds + ) response.raise_for_status() logging.debug("Request successful. Status: %s", response.status_code) return response.json() @@ -52,78 +58,72 @@ def post(self, payload: Dict[str, Any]) -> Dict[str, Any]: return {"error": "JSON decoding error"} def set_payload_messages(self, messages: List[Dict[str, str]]) -> None: + """Sets the messages in the payload.""" self.payload["messages"] = messages def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Union[str, int]]: + """Generates output by sending messages to the API.""" try: - logging.debug("Generating output...") self.set_payload_messages(messages) response = self.post(self.payload) logging.debug("Response: %s", response) return { "response": response.get("message", {}).get("content", "Response not found."), - "total_duration": response.get("total_duration", self.timeout_seconds) + "total_duration": response.get("total_duration", self.timeout_seconds), } except Exception as e: logging.error(f"Error generating output: {e}") return {"error": "Output generation error"} - def generate_messages_list(self, method_info: Dict[str, str], full_class_name: str, branch: str, output_path: str) -> List[List[Dict[str, str]]]: + def generate_messages_list(self, method_info: Dict[str, str], full_class_name: str, + branch: str, output_path: str) -> List[List[Dict[str, str]]]: """ Generates the messages for the API requests. Each list of messages contains different information about the method under test. """ self.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] - class_fields: Union[str, List[str]] = method_info.get("class_fields", []) - constructor_codes: Union[str, List[str]] = method_info.get("constructor_codes", []) + class_fields = method_info.get("class_fields", []) + constructor_codes = method_info.get("constructor_codes", []) method_code = method_info.get("method_code", "") + left_changes_summary = method_info.get("left_changes_summary", "") + right_changes_summary = method_info.get("right_changes_summary", "") - # Define the base system message system_message = { "role": "system", - "content": """You are a senior Java developer with expertise in JUnit testing. -Your task is to provide JUnit tests for the given method in the class under test, considering the changes introduced in the left and right branches. -You have to answer with the test code only, inside code blocks (```). -The tests should start with @Test. -""" + "content": ( + "You are a senior Java developer with expertise in JUnit testing.\n" + "Your task is to provide JUnit tests for the given method in the class under test, " + "considering the changes introduced in the left and right branches.\n" + "You have to answer with the test code only, inside code blocks (```).\n" + "The tests should start with @Test." + ), } user_init_msg = { "role": "user", - "content": f"""Here is the context of the method under test in the class {class_name} on the {branch} branch:""" + "content": f"""{left_changes_summary}\n{right_changes_summary}\nHere is the context of the method under test in the class {class_name} on the {branch} branch:""", } - # Define the user message templates user_msg_templates = [ - { - "role": "user", - "content": f"""Class fields: -""" + "\n".join(class_fields) + "" - }, - { - "role": "user", - "content": f"""Constructors: -""" + "\n".join(constructor_codes) + "" - }, + {"role": "user", "content": f"Class fields:\n" + "\n".join(class_fields)}, + {"role": "user", "content": f"Constructors:\n" + "\n".join(constructor_codes)}, ] user_method_ctx_msg = { "role": "user", - "content": f"""Target Method Under Test: -{method_code} - -Now generate tests for the method under test, considering the given context. -Write all tests inside code blocks (```), and start each test with @Test.""" + "content": ( + f"Target Method Under Test:\n{method_code}\n\n" + "Now generate tests for the method under test, considering the given context.\n" + "Write all tests inside code blocks (```), and start each test with @Test." + ), } - # Build the list of lists of messages messages_lists: List[List[Dict[str, str]]] = [] for r in range(1, len(user_msg_templates) + 1): for user_msgs_combination in combinations(user_msg_templates, r): messages_list = [system_message, user_init_msg, *user_msgs_combination, user_method_ctx_msg] messages_lists.append(messages_list) - return messages_lists @@ -141,19 +141,6 @@ def _get_test_suite_class_paths(self, path: str) -> List[str]: def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: return [os.path.basename(path).replace(".java", "") for path in self._get_test_suite_class_paths(test_suite_path)] - def generate_outputs(self, messages: List[Dict[str, str]]) -> Dict[str, str]: - """Generates the output using the API and returns the response and time duration""" - try: - output = self.api.generate_output(messages) - if isinstance(output, dict): - return output - else: - logging.error(f"Unexpected output format: {output}") - return {"error": "Unexpected output format"} - except Exception as e: - logging.error(f"Error generating outputs: {e}") - return {"error": "Output generation error"} - def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" # Remove content between tags @@ -257,7 +244,10 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods if class_name not in scenario_infos_dict: scenario_infos_dict[class_name] = [] - for method in methods: + for method_item in methods: + method = method_item.get("method", method_item) + left_changes_summary = method_item.get("leftChangesSummary", "") + right_changes_summary = method_item.get("rightChangesSummary", "") try: logging.debug("Saving scenario information for method '%s' in class '%s'", method, class_name) class_fields, constructor_codes, method_code = self.extract_class_info(source_code_path, method, class_name) @@ -266,6 +256,8 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods 'class_fields': class_fields if class_fields else [], 'constructor_codes': constructor_codes if constructor_codes else [], 'method_code': method_code if method_code else "", + 'left_changes_summary': left_changes_summary, + 'right_changes_summary': right_changes_summary, 'test_template': ( "import org.junit.Test;\n" "import static org.junit.Assert.*;\n\n" @@ -511,28 +503,28 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s for class_name, scenario_infos_list in scenario_infos_dict.items(): logging.debug("Generating tests for target methods in class '%s'", class_name) for i, method_info in enumerate(scenario_infos_list): - messages = self.api.generate_messages_list(method_info, class_name, branch, output_path) + messages_list = self.api.generate_messages_list(method_info, class_name, branch, output_path) test_template = method_info.get("test_template", "") - self._process_prompts(messages=messages, test_template=test_template, output_path=output_path, + self._process_prompts(messages_list=messages_list, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), i=i, time_duration_path=time_duration_path, project_name=project_name) - def _process_prompts(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, + def _process_prompts(self, messages_list: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, num_outputs: int = 1) -> None: for j in range(num_outputs): - for k, message_set in enumerate(messages): + for k, messages in enumerate(messages_list): output_file_name = f"{i}{j}{k}_{branch}_{class_name.split('.')[-1]}" - self._process_single_prompt(message_set, test_template, output_path, branch, class_name, imports, i, j, k, time_duration_path, project_name, output_file_name) + self._process_single_prompt(messages, test_template, output_path, branch, class_name, imports, i, j, k, time_duration_path, project_name, output_file_name) - def _process_single_prompt(self, message_set: List[Dict[str, str]], test_template: str, output_path: str, branch: str, + def _process_single_prompt(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, j: int, k: int, time_duration_path: str, project_name: str, output_file_name: str) -> None: try: logging.debug("Processing output %d%d%d in branch \"%s\"", i, j, k, branch) - output = self.generate_outputs(message_set) + output = self.api.generate_output(messages) response = output.get("response", "Response not found.") - total_duration = int(output.get("total_duration", "0") or 0) + total_duration = int(output.get("total_duration", self.api.timeout_seconds)) self.save_output(test_template, response, output_path, output_file_name) except Exception as e: logging.error("Error while processing output %d%d%d in branch \"%s\": %s", i, j, k, branch, e) From 8ba504df70985d6775bb3eae78f82e6f6835ead6 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 22 Apr 2025 16:13:24 -0300 Subject: [PATCH 41/69] =?UTF-8?q?conserto=20da=20l=C3=B3gica?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../codellama_test_suite_generator.py | 99 +++++++++++++------ 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 72d8dd0d..9fb736eb 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -27,7 +27,11 @@ def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model "model": self.model, "messages": [], "stream": False, - "options": {"temperature": self.temperature, "num_ctx": 16384}, + "options": { + "temperature": self.temperature, + "num_ctx": 16384, + "seed": 123 + }, } self.branch = None @@ -66,7 +70,7 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Union[str try: self.set_payload_messages(messages) response = self.post(self.payload) - logging.debug("Response: %s", response) + #logging.debug("Response: %s", response) return { "response": response.get("message", {}).get("content", "Response not found."), "total_duration": response.get("total_duration", self.timeout_seconds), @@ -76,13 +80,14 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Union[str return {"error": "Output generation error"} def generate_messages_list(self, method_info: Dict[str, str], full_class_name: str, - branch: str, output_path: str) -> List[List[Dict[str, str]]]: + branch: str, output_path: str) -> Dict[str, List[Dict[str, str]]]: """ Generates the messages for the API requests. Each list of messages contains different information about the method under test. """ self.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] + method_name = method_info.get("method_name", "") class_fields = method_info.get("class_fields", []) constructor_codes = method_info.get("constructor_codes", []) method_code = method_info.get("method_code", "") @@ -102,10 +107,11 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s user_init_msg = { "role": "user", - "content": f"""{left_changes_summary}\n{right_changes_summary}\nHere is the context of the method under test in the class {class_name} on the {branch} branch:""", + "content": f"""Here is the context of the method under test in the class {class_name} on the {branch} branch:""", } user_msg_templates = [ + {"role": "user", "content": f"{left_changes_summary}\n{right_changes_summary}"}, {"role": "user", "content": f"Class fields:\n" + "\n".join(class_fields)}, {"role": "user", "content": f"Constructors:\n" + "\n".join(constructor_codes)}, ] @@ -114,17 +120,37 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s "role": "user", "content": ( f"Target Method Under Test:\n{method_code}\n\n" - "Now generate tests for the method under test, considering the given context.\n" + "Now generate JUnit tests for the method under test, considering the given context. Remember to create meaningful assertions.\n" "Write all tests inside code blocks (```), and start each test with @Test." ), } - messages_lists: List[List[Dict[str, str]]] = [] + messages_dict: Dict[str, List[Dict[str, str]]] = {} + counter = 1 for r in range(1, len(user_msg_templates) + 1): for user_msgs_combination in combinations(user_msg_templates, r): + key = f"prompt{counter}" messages_list = [system_message, user_init_msg, *user_msgs_combination, user_method_ctx_msg] - messages_lists.append(messages_list) - return messages_lists + messages_dict[key] = messages_list + counter += 1 + + # Save messages to a JSON file + output_file_path = os.path.join(output_path, "generated_messages.json") + if os.path.exists(output_file_path): + with open(output_file_path, "r") as file: + existing_data = json.load(file) + else: + existing_data = {} + + if class_name not in existing_data: + existing_data[class_name] = {} + + existing_data[class_name][method_info["method_name"]] = messages_dict + + with open(output_file_path, "w") as file: + json.dump(existing_data, file, indent=4) + + return messages_dict class CodellamaTestSuiteGenerator(TestSuiteGenerator): @@ -153,8 +179,8 @@ def save_output(self, test_template: str, output: str, dir: str, output_file_nam # Remove lines starting with "number. " (e.g., "1. public void test() {...}") output = re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) - # Look for @Before, @BeforeClass, or @BeforeAll first; fallback to @Test if none are found - markers = ["@Before", "@BeforeClass", "@BeforeAll", "@Test"] + # Look for @Before, @BeforeClass first; fallback to @Test if none are found + markers = ["@Before", "@BeforeClass", "@Test"] index = min((output.find(marker) for marker in markers if marker in output), default=-1) # Keep only the content starting from the first found annotation @@ -255,11 +281,14 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods scenario_infos_dict[class_name].append({ 'class_fields': class_fields if class_fields else [], 'constructor_codes': constructor_codes if constructor_codes else [], + 'method_name': method, 'method_code': method_code if method_code else "", 'left_changes_summary': left_changes_summary, 'right_changes_summary': right_changes_summary, 'test_template': ( "import org.junit.Test;\n" + "import org.junit.Before;\n" + "import org.junit.BeforeClass;\n" "import static org.junit.Assert.*;\n\n" f"public class {class_name.split('.')[-1]}_{method.split('(')[0]}Test {{\n" "#TEST_METHODS#\n" @@ -302,7 +331,7 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str save_json(imports_path, imports_dict) - def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int) -> None: + def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int, prompt_key: str) -> None: """Extracts individual tests from the generated test suite and saves them to separate files""" llm_outputs_path = os.path.join(output_path, "llm_outputs") counter = 0 @@ -316,30 +345,35 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: captured_text = self.extract_snippet(source_code, node.start_byte, node.end_byte) if "@Test" in captured_text: test_block.append({"snippet": captured_text}) - elif any(annotation in captured_text for annotation in ["@Before", "@BeforeClass", "@BeforeAll"]): + elif any(annotation in captured_text for annotation in ["@Before", "@BeforeClass"]): before_block.append({"snippet": captured_text}) return before_block, test_block + # Format: {i}{j}_{branch}_{class_name.split('.')[-1]}_{prompt_key}.txt + pattern = rf"^{i}\d+_(left|right)_{re.escape(class_name.split('.')[-1])}_{prompt_key}\.txt$" for file in os.listdir(llm_outputs_path): # Avoid processing the wrong files (from different classes) - pattern = rf"^{i}\d+_(left|right)_{re.escape(class_name.split('.')[-1])}\.txt$" if re.match(pattern, file): + logging.debug("Processing file: %s", file) source_code_path = os.path.join(llm_outputs_path, file) JAVA_LANGUAGE, source_code, tree = self.parse_code(source_code_path) + # Tree-sitter query to find method definitions with annotations query_text = """ (method_declaration - (modifiers - (marker_annotation))) @method_def + (modifiers [ + (annotation) + (marker_annotation) + ] + )) @method_def """ query = JAVA_LANGUAGE.query(query_text) captures = query.captures(tree.root_node) before_block, test_block = classify_annotations(captures, source_code) - for test in test_block: - method_name = f"{class_name.split('.')[-1]}Test_{i}_{counter}" + method_name = f"{class_name.split('.')[-1]}Test_{prompt_key}_{i}_{counter}" output_file_path = os.path.join(output_path, f"{method_name}.java") new_template = test_template.split("public class")[0] + f"public class {method_name} {{\n" @@ -349,13 +383,15 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: full_template += "".join(before['snippet'] for before in before_block) snippet = test['snippet'] - test_method_name = snippet.split('(')[0].split()[-1] - new_snippet = snippet.replace(test_method_name, f"test{i}{counter}") + public_index = snippet.find('public') + if public_index != -1: + test_method_name = snippet[public_index:].split('(')[0].split()[-1] + new_snippet = snippet.replace(test_method_name, f"test{i}{counter}") - with open(output_file_path, "w") as f: - f.write(full_template + new_snippet + "\n}") + with open(output_file_path, "w") as f: + f.write(full_template + new_snippet + "\n}") - counter += 1 + counter += 1 def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str, project_name: str) -> Dict[str, str]: """Finds the source code files for the given class name in the specified JAR path""" @@ -376,6 +412,7 @@ def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str, raise FileNotFoundError(f"Source code for class '{class_name}' not found in '{base_path}'") """ + input_jar = input_jar.split(":")[0] if not os.path.exists(input_jar): logging.error("The provided jar path '%s' does not exist", input_jar) raise FileNotFoundError(f"The provided path '{input_jar}' does not exist") @@ -509,26 +546,26 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), i=i, time_duration_path=time_duration_path, project_name=project_name) - def _process_prompts(self, messages_list: List[Dict[str, str]], test_template: str, output_path: str, branch: str, + def _process_prompts(self, messages_list: Dict[str, List[Dict[str, str]]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, time_duration_path: str, project_name: str, num_outputs: int = 1) -> None: for j in range(num_outputs): - for k, messages in enumerate(messages_list): - output_file_name = f"{i}{j}{k}_{branch}_{class_name.split('.')[-1]}" - self._process_single_prompt(messages, test_template, output_path, branch, class_name, imports, i, j, k, time_duration_path, project_name, output_file_name) + for prompt_key, messages in messages_list.items(): + output_file_name = f"{i}{j}_{branch}_{class_name.split('.')[-1]}_{prompt_key}" + self._process_single_prompt(messages, test_template, output_path, branch, class_name, imports, i, j, time_duration_path, project_name, output_file_name, prompt_key) def _process_single_prompt(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, - class_name: str, imports: List[str], i: int, j: int, k: int, time_duration_path: str, - project_name: str, output_file_name: str) -> None: + class_name: str, imports: List[str], i: int, j: int, time_duration_path: str, + project_name: str, output_file_name: str, prompt_key: str) -> None: try: - logging.debug("Processing output %d%d%d in branch \"%s\"", i, j, k, branch) + logging.debug("Processing output %d%d for prompt key '%s' in branch \"%s\"", i, j, prompt_key, branch) output = self.api.generate_output(messages) response = output.get("response", "Response not found.") total_duration = int(output.get("total_duration", self.api.timeout_seconds)) self.save_output(test_template, response, output_path, output_file_name) except Exception as e: - logging.error("Error while processing output %d%d%d in branch \"%s\": %s", i, j, k, branch, e) + logging.error("Error while processing output %d%d for prompt key '%s' in branch \"%s\": %s", i, j, prompt_key, branch, e) finally: self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) - self.extract_individual_tests(output_path, test_template, class_name, imports, i) + self.extract_individual_tests(output_path, test_template, class_name, imports, i, prompt_key) From 3f36b797935cb62a1a7c366da27f8a0522b00a8e Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Tue, 22 Apr 2025 16:14:40 -0300 Subject: [PATCH 42/69] =?UTF-8?q?conserto=20de=20l=C3=B3gica=20+=20sa?= =?UTF-8?q?=C3=ADda=20em=20arquivo=20json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_suite_executor.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index c1cb0023..089e29e1 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -1,6 +1,7 @@ import logging import re import subprocess +import json from os import path from typing import Dict, List from nimrod.test_suite_generation.test_suite import TestSuite @@ -11,6 +12,8 @@ from nimrod.tools.jacoco import Jacoco from nimrod.utils import generate_classpath +EXECUTION_LOG_FILE = "execution_results.json" + def is_failed_caused_by_compilation_problem(test_case_name: str, failed_test_message: str) -> bool: my_regex = re.escape(test_case_name) + r"[0-9A-Za-z0-9_\(\.\)\n \:]+(NoSuchMethodError|NoSuchFieldError|NoSuchClassError|NoClassDefFoundError|NoSuchAttributeError|tried to access method)" return re.search(my_regex, failed_test_message) != None @@ -34,9 +37,35 @@ def __init__(self, java: Java, jacoco: Jacoco) -> None: def execute_test_suite(self, test_suite: TestSuite, jar: str, number_of_executions: int = 3) -> Dict[str, TestCaseResult]: results: Dict[str, TestCaseResult] = dict() + # Load existing log if it exists + try: + with open(EXECUTION_LOG_FILE, "r") as log_file: + execution_log = json.load(log_file) + except (FileNotFoundError, json.JSONDecodeError): + execution_log = {} + for test_class in test_suite.test_classes_names: logging.debug("Test class: %s", test_class) + class_file_path = path.join(test_suite.path, f"classes/{test_class}.class") + + if not path.exists(class_file_path): + logging.warning("Class file %s does not exist; skipping execution", class_file_path) + continue + + if test_class not in execution_log: + execution_log[test_class] = [] + # Check if the current test_suite.path is already in the log + test_suite_entry = next((entry for entry in execution_log[test_class] if test_suite.path in entry), None) + if not test_suite_entry: + test_suite_entry = {test_suite.path: {"jar": {}}} + execution_log[test_class].append(test_suite_entry) + + # Ensure the JAR is tracked under the current test_suite.path + if jar not in test_suite_entry[test_suite.path]["jar"]: + test_suite_entry[test_suite.path]["jar"][jar] = [] + + # Append execution results for the current JAR for i in range(0, number_of_executions): logging.info("Starting execution %d of %s from suite %s", i + 1, test_class, test_suite.path) response = self._execute_junit(test_suite, jar, test_class) @@ -48,6 +77,14 @@ def execute_test_suite(self, test_suite: TestSuite, jar: str, number_of_executio elif not results.get(test_fqname): results[test_fqname] = test_case_result + test_suite_entry[test_suite.path]["jar"][jar].append({ + "execution_number": i + 1, + "result": {test_case: str(test_case_result) for test_case, test_case_result in response.items()} + }) + + with open(EXECUTION_LOG_FILE, "w") as log_file: + json.dump(execution_log, log_file, indent=4) + return results def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str, extra_params: List[str] = []) -> Dict[str, TestCaseResult]: @@ -91,8 +128,11 @@ def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResu if results: for i in range(0, test_run_count): test_case_name = 'test{number:0{width}d}'.format(width=len(str(test_run_count)), number=i) - if not results.get(test_case_name): + if not results.get(test_case_name) and test_run_count > 1: results[test_case_name] = TestCaseResult.PASS + if not results: + test_case_name = 'test0' + results[test_case_name] = TestCaseResult.NOT_EXECUTABLE return results def execute_test_suite_with_coverage(self, test_suite: TestSuite, target_jar: str, test_cases: List[str]) -> str: From 9195e6f4a87f1b3ce8195f7e74b80833eb7b4e52 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 17:17:23 -0300 Subject: [PATCH 43/69] small fix logging message --- nimrod/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimrod/__main__.py b/nimrod/__main__.py index 1d95e4d4..8991638d 100644 --- a/nimrod/__main__.py +++ b/nimrod/__main__.py @@ -98,7 +98,7 @@ def main(): if scenario.run_analysis: smat.run_tool_for_semmantic_conflict_detection(scenario) else: - logging.info(f"Skipping tool execution for project f{scenario.project_name}") + logging.info(f"Skipping tool execution for project {scenario.project_name}") if __name__ == '__main__': From e17eed83cf9f3f8ccd81ce9fcf7757b881191008 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 17:25:04 -0300 Subject: [PATCH 44/69] get seed from config --- .../codellama_test_suite_generator.py | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index 9fb736eb..ef970ee7 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -17,10 +17,11 @@ class Api: - def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model: str) -> None: + def __init__(self, api_url: str, timeout_seconds: int, temperature: float, seed: int, model: str) -> None: self.api_url = api_url self.timeout_seconds = timeout_seconds self.temperature = temperature + self.seed = seed self.model = model self.headers = {"Content-Type": "application/json"} self.payload = { @@ -30,7 +31,7 @@ def __init__(self, api_url: str, timeout_seconds: int, temperature: float, model "options": { "temperature": self.temperature, "num_ctx": 16384, - "seed": 123 + "seed": self.seed, }, } self.branch = None @@ -94,6 +95,9 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s left_changes_summary = method_info.get("left_changes_summary", "") right_changes_summary = method_info.get("right_changes_summary", "") + + ###################################################### + ## PROMPTS INCREMENTAL COMBINATIONS system_message = { "role": "system", "content": ( @@ -127,6 +131,14 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s messages_dict: Dict[str, List[Dict[str, str]]] = {} counter = 1 + + ## PROMPT SEM CONTEXTO + key = f"prompt{counter}" + messages_list = [system_message, user_init_msg, user_method_ctx_msg] + messages_dict[key] = messages_list + counter += 1 + + ## PROMPTS COM CONTEXTO for r in range(1, len(user_msg_templates) + 1): for user_msgs_combination in combinations(user_msg_templates, r): key = f"prompt{counter}" @@ -331,7 +343,7 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str save_json(imports_path, imports_dict) - def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int, prompt_key: str) -> None: + def extract_individual_tests(self, output_path: str, test_template: str, class_name: str, imports: List[str], i: int, j: int, prompt_key: str, branch: str) -> None: """Extracts individual tests from the generated test suite and saves them to separate files""" llm_outputs_path = os.path.join(output_path, "llm_outputs") counter = 0 @@ -351,7 +363,7 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: return before_block, test_block # Format: {i}{j}_{branch}_{class_name.split('.')[-1]}_{prompt_key}.txt - pattern = rf"^{i}\d+_(left|right)_{re.escape(class_name.split('.')[-1])}_{prompt_key}\.txt$" + pattern = rf"^{i}{j}_(left|right)_{re.escape(class_name.split('.')[-1])}_{prompt_key}\.txt$" for file in os.listdir(llm_outputs_path): # Avoid processing the wrong files (from different classes) if re.match(pattern, file): @@ -373,7 +385,7 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: before_block, test_block = classify_annotations(captures, source_code) for test in test_block: - method_name = f"{class_name.split('.')[-1]}Test_{prompt_key}_{i}_{counter}" + method_name = f"{class_name.split('.')[-1]}Test_{branch}_{prompt_key}_{j}_{i}_{counter}" output_file_path = os.path.join(output_path, f"{method_name}.java") new_template = test_template.split("public class")[0] + f"public class {method_name} {{\n" @@ -395,23 +407,6 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: def find_source_code_paths(self, input_jar: str, jar_type: str, class_name: str, project_name: str) -> Dict[str, str]: """Finds the source code files for the given class name in the specified JAR path""" - """ - base_path = os.path.join("/mnt/c/Users/natha/Downloads/tests", project_name) - if not os.path.exists(base_path): - raise FileNotFoundError(f"The base path '{base_path}' does not exist") - - # Converter class_name para path relativo dentro do diretório base - relative_path = class_name.replace(".", "/") + ".java" - - # Procurar o arquivo no diretório base - for root, _, files in os.walk(base_path): - if relative_path.split('/')[-1] in files: - full_path = os.path.join(root, relative_path.split('/')[-1]) - logging.debug("Source code found for class '%s' in '%s'", class_name, full_path) - return {key: full_path for key in ["base", "left", "right", "merge"]} - - raise FileNotFoundError(f"Source code for class '{class_name}' not found in '{base_path}'") - """ input_jar = input_jar.split(":")[0] if not os.path.exists(input_jar): logging.error("The provided jar path '%s' does not exist", input_jar) @@ -509,6 +504,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s api_url=codellama_params.get("api_url", "http://localhost:11434/api/chat"), timeout_seconds=codellama_params.get("timeout_seconds", 60), temperature=codellama_params.get("temperature", 0), + seed=codellama_params.get("seed", 42), model=codellama_params.get("model", "codellama:70b") ) @@ -568,4 +564,4 @@ def _process_single_prompt(self, messages: List[Dict[str, str]], test_template: finally: self.record_output_duration(time_duration_path, output_path, class_name, output_file_name, total_duration, project_name) - self.extract_individual_tests(output_path, test_template, class_name, imports, i, prompt_key) + self.extract_individual_tests(output_path, test_template, class_name, imports, i, j, prompt_key, branch) From 4e491870a873ded74501caf808e289cdaa0f9b00 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 19:30:02 -0300 Subject: [PATCH 45/69] remove version from requirements --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index dfc7cfe6..1b4a09df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -beautifulsoup4==4.12.3 -setuptools==70.0.0 -tree_sitter==0.22.3 -tree_sitter_java==0.21.0 -types-requests==2.32.0.20241016 \ No newline at end of file +beautifulsoup4 +setuptools +tree_sitter +tree_sitter_java +types-requests \ No newline at end of file From 94003b045593e2b43c10b1cda8115f3a29ee23d2 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 20:13:28 -0300 Subject: [PATCH 46/69] typing issues --- nimrod/core/merge_scenario_under_analysis.py | 4 +- .../semantic_conflicts_output_generator.py | 78 +++++++++++-------- .../test_suites_output_generator.py | 4 +- .../codellama_test_suite_generator.py | 32 ++++++-- .../evosuite_test_suite_generator.py | 15 ++-- .../randoop_test_suite_generator.py | 12 ++- 6 files changed, 93 insertions(+), 52 deletions(-) diff --git a/nimrod/core/merge_scenario_under_analysis.py b/nimrod/core/merge_scenario_under_analysis.py index e2cec34a..86fb9a56 100644 --- a/nimrod/core/merge_scenario_under_analysis.py +++ b/nimrod/core/merge_scenario_under_analysis.py @@ -1,8 +1,8 @@ -from typing import List, Dict +from typing import List, Dict, Union class MergeScenarioUnderAnalysis: - def __init__(self, project_name: str, run_analysis: bool, scenario_commits: "ScenarioInformation", targets: "Dict[str, List[str]]", scenario_jars: "ScenarioInformation", jar_type: str): + def __init__(self, project_name: str, run_analysis: bool, scenario_commits: "ScenarioInformation", targets: "Dict[str, Union[List[Dict[str, str]], List[str]]]", scenario_jars: "ScenarioInformation", jar_type: str): self.project_name = project_name self.run_analysis = run_analysis self.scenario_commits = scenario_commits diff --git a/nimrod/output_generation/semantic_conflicts_output_generator.py b/nimrod/output_generation/semantic_conflicts_output_generator.py index 12c582c0..dacb8325 100644 --- a/nimrod/output_generation/semantic_conflicts_output_generator.py +++ b/nimrod/output_generation/semantic_conflicts_output_generator.py @@ -1,8 +1,9 @@ -from typing import Dict, List, TypedDict +from typing import Dict, List, TypedDict, Union from nimrod.output_generation.output_generator import OutputGenerator, OutputGeneratorContext from nimrod.test_suites_execution.main import TestSuitesExecution from os import path from bs4 import BeautifulSoup +import logging class SemanticConflictsOutput(TypedDict): @@ -12,7 +13,7 @@ class SemanticConflictsOutput(TypedDict): test_case_name: str test_case_results: Dict[str, str] test_suite_path: str - scenario_targets: Dict[str, List[str]] + scenario_targets: Dict[str, Union[List[Dict[str, str]], List[str]]] exercised_targets: Dict[str, List[str]] @@ -26,40 +27,51 @@ def _generate_report_data(self, context: OutputGeneratorContext) -> List[Semanti for semantic_conflict in context.semantic_conflicts: # We need to detect which targets from the input were exercised in this conflict. - coverage_report_root = self._test_suites_execution.execute_test_suite_with_coverage( - test_suite=semantic_conflict.detected_in.test_suite, - target_jar=context.scenario.scenario_jars.merge, - test_cases=[semantic_conflict.detected_in.name] - ) - - exercised_targets = self._extract_exercised_targets_from_coverage_report( - coverage_report_root=coverage_report_root, - targets=context.scenario.targets - ) - - report_data.append({ - "project_name": context.scenario.project_name, - "scenario_commits": context.scenario.scenario_commits.__dict__, - "criteria": semantic_conflict._satisfying_criteria.__class__.__name__, - "test_case_name": semantic_conflict.detected_in.name, - "test_case_results": { - "base": semantic_conflict.detected_in.base, - "left": semantic_conflict.detected_in.left, - "right": semantic_conflict.detected_in.right, - "merge": semantic_conflict.detected_in.merge - }, - "test_suite_path": semantic_conflict.detected_in.test_suite.path, - "scenario_targets": context.scenario.targets, - "exercised_targets": exercised_targets - }) + exercised_targets: Dict[str, List[str]] = dict() + try: + coverage_report_root = self._test_suites_execution.execute_test_suite_with_coverage( + test_suite=semantic_conflict.detected_in.test_suite, + target_jar=context.scenario.scenario_jars.merge, + test_cases=[semantic_conflict.detected_in.name] + ) + + exercised_targets = self._extract_exercised_targets_from_coverage_report( + coverage_report_root=coverage_report_root, + targets=context.scenario.targets + ) + + except Exception as e: + # If we cannot execute the test suite with coverage, we log the error and continue. + logging.error(f"Error executing test suite with coverage for semantic conflict: {e}") + + finally: + report_data.append({ + "project_name": context.scenario.project_name, + "scenario_commits": context.scenario.scenario_commits.__dict__, + "criteria": semantic_conflict._satisfying_criteria.__class__.__name__, + "test_case_name": semantic_conflict.detected_in.name, + "test_case_results": { + "base": semantic_conflict.detected_in.base, + "left": semantic_conflict.detected_in.left, + "right": semantic_conflict.detected_in.right, + "merge": semantic_conflict.detected_in.merge + }, + "test_suite_path": semantic_conflict.detected_in.test_suite.path, + "scenario_targets": context.scenario.targets, + "exercised_targets": exercised_targets + }) return report_data - def _extract_exercised_targets_from_coverage_report(self, coverage_report_root: str, targets: Dict[str, List[str]]): + def _extract_exercised_targets_from_coverage_report(self, coverage_report_root: str, targets: Dict[str, Union[List[Dict[str, str]], List[str]]]): exercised_targets: Dict[str, List[str]] = dict() for class_name in targets.keys(): - for method_name in targets[class_name]: + for method_item in targets[class_name]: + if isinstance(method_item, dict): + method_name = method_item.get("method", "") + else: + method_name = method_item if self._was_target_exercised(coverage_report_root, class_name, method_name): exercised_targets[class_name] = exercised_targets.get( class_name, []) + [method_name] @@ -80,7 +92,11 @@ def _was_target_exercised(self, coverage_report_root: str, fqcn: str, method_sig # We itereate in each method row for method_row in method_report_rows: if method_row.get_text().find(method_name) != -1: - if method_row.select_one('td:nth-last-child(2)').get_text() == '0': + tag = method_row.select_one('td:nth-last-child(2)') + if tag is None: + continue + # If the second last column is 0, it means the method was not executed + if tag.get_text() == "0": return True return False diff --git a/nimrod/output_generation/test_suites_output_generator.py b/nimrod/output_generation/test_suites_output_generator.py index b9a9039c..2ebb1ed0 100644 --- a/nimrod/output_generation/test_suites_output_generator.py +++ b/nimrod/output_generation/test_suites_output_generator.py @@ -1,4 +1,4 @@ -from typing import List, TypedDict +from typing import List, TypedDict, Dict, Union from nimrod.dynamic_analysis.behavior_change import BehaviorChange from nimrod.dynamic_analysis.semantic_conflict import SemanticConflict from nimrod.output_generation.output_generator import OutputGenerator, OutputGeneratorContext @@ -7,7 +7,7 @@ class TestSuitesOutput(TypedDict): project_name: str - targets: dict[str, list[str]] + targets: Dict[str, Union[List[Dict[str, str]], List[str]]] generator_name: str path: str detected_semantic_conflicts: bool diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index ef970ee7..f325d259 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -34,7 +34,7 @@ def __init__(self, api_url: str, timeout_seconds: int, temperature: float, seed: "seed": self.seed, }, } - self.branch = None + self.branch: Optional[str] = None def set_branch(self, branch: str) -> None: """Sets the branch to be used in the API requests.""" @@ -66,7 +66,7 @@ def set_payload_messages(self, messages: List[Dict[str, str]]) -> None: """Sets the messages in the payload.""" self.payload["messages"] = messages - def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Union[str, int]]: + def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: """Generates output by sending messages to the API.""" try: self.set_payload_messages(messages) @@ -89,8 +89,8 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s self.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] method_name = method_info.get("method_name", "") - class_fields = method_info.get("class_fields", []) - constructor_codes = method_info.get("constructor_codes", []) + class_fields: Union[str, List[str]] = method_info.get("class_fields", []) + constructor_codes: Union[str, List[str]] = method_info.get("constructor_codes", []) method_code = method_info.get("method_code", "") left_changes_summary = method_info.get("left_changes_summary", "") right_changes_summary = method_info.get("right_changes_summary", "") @@ -272,7 +272,7 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ logging.error(f"An error occurred while extracting class info for '{full_class_name}': {e}") raise e - def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods: List[str], source_code_path: str) -> None: + def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods: Union[List[str], List[Dict[str, str]]], source_code_path: str) -> None: """Stores relevant scenario information (for each class and method) in a JSON file""" if os.path.exists(scenario_infos_path): scenario_infos_dict = load_json(scenario_infos_path) @@ -282,11 +282,27 @@ def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods if class_name not in scenario_infos_dict: scenario_infos_dict[class_name] = [] + """ + method_item -> + { + "method": "methodname()", + "leftChangesSummary": "Left does...", + "rightChangesSummary": "Right does..." + } + + method_item -> "methodname()" + """ for method_item in methods: - method = method_item.get("method", method_item) - left_changes_summary = method_item.get("leftChangesSummary", "") - right_changes_summary = method_item.get("rightChangesSummary", "") + if not isinstance(method_item, dict): + method = method_item + left_changes_summary = "" + right_changes_summary = "" + else: + method = method_item.get("method", "") + left_changes_summary = method_item.get("leftChangesSummary", "") + right_changes_summary = method_item.get("rightChangesSummary", "") try: + method = re.sub(r'\|', ',', method) logging.debug("Saving scenario information for method '%s' in class '%s'", method, class_name) class_fields, constructor_codes, method_code = self.extract_class_info(source_code_path, method, class_name) diff --git a/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py b/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py index 39eb9343..28895137 100644 --- a/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/evosuite_test_suite_generator.py @@ -1,6 +1,6 @@ import logging import os -from typing import List +from typing import List, Dict, Union from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import \ @@ -69,10 +69,15 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: def _compile_test_suite(self, input_jar: str, output_path: str, extra_class_path: List[str] = []) -> str: return super()._compile_test_suite(input_jar, output_path, [EVOSUITE_RUNTIME] + extra_class_path) - def _create_method_list(self, methods: "List[str]"): - rectified_methods = [self._convert_method_signature( - method) for method in methods] - return (":").join(rectified_methods) + def _create_method_list(self, methods: "Union[List[Dict[str, str]], List[str]]"): + rectified_methods = [] + for method_item in methods: + if isinstance(method_item, dict): + method_str = method_item.get("method", "") + else: + method_str = method_item + rectified_methods.append(self._convert_method_signature(method_str)) + return ":".join(rectified_methods) def _convert_method_signature(self, meth_signature: str) -> str: method_return = "" diff --git a/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py b/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py index 775d14b1..0bd84855 100644 --- a/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/randoop_test_suite_generator.py @@ -1,5 +1,5 @@ import os -from typing import Dict, List +from typing import Dict, List, Union from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import \ @@ -38,7 +38,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s self._java.exec_java(output_path, self._java.get_env(), 3000, *tuple(params)) - def _generate_target_classes_file(self, output_path: str, targets: "Dict[str, List[str]]"): + def _generate_target_classes_file(self, output_path: str, targets: "Dict[str, Union[List[Dict[str, str]], List[str]]]"): filename = os.path.join(output_path, self.TARGET_CLASS_LIST_FILENAME) with open(filename, 'w') as f: @@ -48,12 +48,16 @@ def _generate_target_classes_file(self, output_path: str, targets: "Dict[str, Li return filename - def _generate_target_methods_file(self, output_path: str, targets: "Dict[str, List[str]]"): + def _generate_target_methods_file(self, output_path: str, targets: "Dict[str, Union[List[Dict[str, str]], List[str]]]"): filename = os.path.join(output_path, self.TARGET_METHODS_LIST_FILENAME) with open(filename, 'w') as f: for fqcn, methods in targets.items(): - for method in methods: + for method_item in methods: + if not isinstance(method_item, dict): + method = method_item + else: + method = method_item.get("method", "") method_signature = fqcn + "." + method f.write(method_signature) From 6bc007c871e95b1efa4b8a71d1d6d7f9ab6feaa9 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 20:14:23 -0300 Subject: [PATCH 47/69] change logic - special codellama --- .../test_suite_executor.py | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 089e29e1..4a79e9da 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -102,20 +102,32 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str command = self._java.exec_java(test_suite.path, self._java.get_env(), TIMEOUT, *params) output = command.decode('unicode_escape') + if test_suite.generator_name == "CODELLAMA": + #HSaslThriftClientTest_right_prompt1_0_39.java + #test_class_num = "39" + #HSaslThriftClientTest_right_prompt1_1_39.java + #test_class_num = "139" + parts = test_class.replace(".java", "").split("_") + test_class_num = parts[-2] + parts[-1] + return self._parse_test_results_from_output(output, test_class_num) return self._parse_test_results_from_output(output) except subprocess.CalledProcessError as error: output = error.output.decode('unicode_escape') return self._parse_test_results_from_output(output) - def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResult]: + def _parse_test_results_from_output(self, output: str, test_class_num: str = None) -> Dict[str, TestCaseResult]: results: Dict[str, TestCaseResult] = dict() success_match = re.search(r'OK \((?P\d+) tests?\)', output) if success_match: - number_of_tests = int(success_match.group('number_of_tests')) - for i in range(0, number_of_tests): - test_case_name = 'test{number:0{width}d}'.format(width=len(str(number_of_tests)), number=i) + if test_class_num: + test_case_name = f'test{test_class_num}' results[test_case_name] = TestCaseResult.PASS + else: + number_of_tests = int(success_match.group('number_of_tests')) + for i in range(0, number_of_tests): + test_case_name = 'test{number:0{width}d}'.format(width=len(str(number_of_tests)), number=i) + results[test_case_name] = TestCaseResult.PASS else: failed_tests = re.findall(r'(?Ptest\d+)\([A-Za-z0-9_.]+\)', output) for failed_test in failed_tests: @@ -131,7 +143,10 @@ def _parse_test_results_from_output(self, output: str) -> Dict[str, TestCaseResu if not results.get(test_case_name) and test_run_count > 1: results[test_case_name] = TestCaseResult.PASS if not results: - test_case_name = 'test0' + if test_class_num: + test_case_name = f'test{test_class_num}' + else: + test_case_name = 'test0' results[test_case_name] = TestCaseResult.NOT_EXECUTABLE return results From a19fb601da1f6188e15e3cfb695b4e1af81842c4 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 20:16:18 -0300 Subject: [PATCH 48/69] update config params --- nimrod/tests/env-config.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nimrod/tests/env-config.json b/nimrod/tests/env-config.json index 7578cf3b..27ca9d3c 100644 --- a/nimrod/tests/env-config.json +++ b/nimrod/tests/env-config.json @@ -7,5 +7,16 @@ "input_path": "", "tests_dst": "", "path_output_csv": "", - "logger_level": "DEBUG" + "logger_level": "DEBUG", + "test_suite_generators":["codellama", "evosuite", "randoop"], + "test_suite_generation_search_time_available":"45", + "api_params": { + "codellama": { + "model": "codellama:70b", + "api_url": "http://ip/api/chat", + "temperature": 0.7, + "seed": 42, + "timeout_seconds": 300 + } + } } From 32da8cae99b89c52db7dc2c85deb9e2953e7ae00 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 20:56:22 -0300 Subject: [PATCH 49/69] dealing with paths - evosuite --- nimrod/test_suites_execution/test_suite_executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 4a79e9da..c62d49d0 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -46,7 +46,11 @@ def execute_test_suite(self, test_suite: TestSuite, jar: str, number_of_executio for test_class in test_suite.test_classes_names: logging.debug("Test class: %s", test_class) - class_file_path = path.join(test_suite.path, f"classes/{test_class}.class") + if test_suite.generator_name == "EVOSUITE": + class_file_path = path.join(test_suite.path, f"classes/{test_class.replace('.', '/')}.class") + + else: + class_file_path = path.join(test_suite.path, f"classes/{test_class}.class") if not path.exists(class_file_path): logging.warning("Class file %s does not exist; skipping execution", class_file_path) From 2c3237df68302aa593a0272589dd5944e7827508 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 21:02:30 -0300 Subject: [PATCH 50/69] remove unused globals --- nimrod/report_metrics/coverage/coverage_report.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nimrod/report_metrics/coverage/coverage_report.py b/nimrod/report_metrics/coverage/coverage_report.py index 37aad142..07a26273 100644 --- a/nimrod/report_metrics/coverage/coverage_report.py +++ b/nimrod/report_metrics/coverage/coverage_report.py @@ -94,9 +94,6 @@ def get_valid_test_suite(self, toolSuites, first_entry, last_entry): return None def retornaDadosParaAnalise(self, evo, path_suite, suite_merge, jacoco, classeTarget, listaPacoteMetodoClasse, targets: "dict[str, list[str]]"): - global tagAClasseTarget - global tagSpanMetodoTarget - print("Classe Target ", classeTarget) listaJar = evo.project_dep.mergeDir.split( From 5f38ba8d3c13f3fd56cbc594276a23fa100652b3 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 19 Jun 2025 21:06:20 -0300 Subject: [PATCH 51/69] remove implicit optional --- nimrod/test_suites_execution/test_suite_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index c62d49d0..5172f4f6 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -3,7 +3,7 @@ import subprocess import json from os import path -from typing import Dict, List +from typing import Dict, List, Optional from nimrod.test_suite_generation.test_suite import TestSuite from nimrod.test_suites_execution.test_case_result import TestCaseResult from nimrod.tests.utils import get_base_output_path @@ -119,7 +119,7 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str output = error.output.decode('unicode_escape') return self._parse_test_results_from_output(output) - def _parse_test_results_from_output(self, output: str, test_class_num: str = None) -> Dict[str, TestCaseResult]: + def _parse_test_results_from_output(self, output: str, test_class_num: Optional[str] = None) -> Dict[str, TestCaseResult]: results: Dict[str, TestCaseResult] = dict() success_match = re.search(r'OK \((?P\d+) tests?\)', output) From cfb8b621dfd3ced85b031d8f57d52367d7740562 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 30 Oct 2025 09:59:04 -0300 Subject: [PATCH 52/69] Update nimrod/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- nimrod/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nimrod/utils.py b/nimrod/utils.py index b51388d8..b6cf35c1 100644 --- a/nimrod/utils.py +++ b/nimrod/utils.py @@ -47,5 +47,9 @@ def load_json(file_path): def save_json(file_path, content): """Saves a dictionary as a JSON file""" - with open(file_path, "w") as file: - json.dump(content, file, indent=4) + try: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "w") as file: + json.dump(content, file, indent=4) + except (OSError, IOError) as e: + print(f"Error saving JSON to {file_path}: {e}") From 433eec212aaef65959efe434b65defe1f250c8c9 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 30 Oct 2025 10:04:24 -0300 Subject: [PATCH 53/69] Update nimrod/test_suites_execution/test_suite_executor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- nimrod/test_suites_execution/test_suite_executor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 5172f4f6..8e7e3708 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -112,7 +112,11 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str #HSaslThriftClientTest_right_prompt1_1_39.java #test_class_num = "139" parts = test_class.replace(".java", "").split("_") - test_class_num = parts[-2] + parts[-1] + if len(parts) >= 2: + test_class_num = parts[-2] + parts[-1] + else: + logging.warning(f"Unexpected test_class format: '{test_class}'. Unable to extract test_class_num, defaulting to '0'.") + test_class_num = "0" return self._parse_test_results_from_output(output, test_class_num) return self._parse_test_results_from_output(output) except subprocess.CalledProcessError as error: From dda0c871ec4393d40cc486a85e6b1131c864a008 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 30 Oct 2025 10:05:36 -0300 Subject: [PATCH 54/69] Update nimrod/test_suite_generation/generators/codellama_test_suite_generator.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../generators/codellama_test_suite_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py index f325d259..23245ea9 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py @@ -269,7 +269,7 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ return class_fields, class_constructors, class_method except Exception as e: - logging.error(f"An error occurred while extracting class info for '{full_class_name}': {e}") + logging.error("An error occurred while extracting class info for '%s': %s", full_class_name, e) raise e def save_scenario_infos(self, scenario_infos_path: str, class_name: str, methods: Union[List[str], List[Dict[str, str]]], source_code_path: str) -> None: From 5e6c389bc392cddaee3da6e13009a73bf6064b78 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 30 Oct 2025 10:07:51 -0300 Subject: [PATCH 55/69] Update nimrod/output_generation/semantic_conflicts_output_generator.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- nimrod/output_generation/semantic_conflicts_output_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nimrod/output_generation/semantic_conflicts_output_generator.py b/nimrod/output_generation/semantic_conflicts_output_generator.py index dacb8325..73a742fa 100644 --- a/nimrod/output_generation/semantic_conflicts_output_generator.py +++ b/nimrod/output_generation/semantic_conflicts_output_generator.py @@ -42,7 +42,7 @@ def _generate_report_data(self, context: OutputGeneratorContext) -> List[Semanti except Exception as e: # If we cannot execute the test suite with coverage, we log the error and continue. - logging.error(f"Error executing test suite with coverage for semantic conflict: {e}") + logging.error("Error executing test suite with coverage for semantic conflict: %s", e) finally: report_data.append({ From 3fb1a94c5c2316e258d429af3a9381ae82a38be2 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Sun, 16 Nov 2025 19:40:21 -0300 Subject: [PATCH 56/69] refactor: rename codellama to ollama --- nimrod/__main__.py | 8 +++---- nimrod/setup_tools/tools.py | 2 +- ...ator.py => ollama_test_suite_generator.py} | 22 +++++++++---------- .../test_suite_executor.py | 2 +- nimrod/tests/env-config.json | 4 ++-- nimrod/tools/{codellama.py => ollama.py} | 6 ++--- 6 files changed, 22 insertions(+), 22 deletions(-) rename nimrod/test_suite_generation/generators/{codellama_test_suite_generator.py => ollama_test_suite_generator.py} (97%) rename nimrod/tools/{codellama.py => ollama.py} (77%) diff --git a/nimrod/__main__.py b/nimrod/__main__.py index 8991638d..8c610493 100644 --- a/nimrod/__main__.py +++ b/nimrod/__main__.py @@ -16,7 +16,7 @@ from nimrod.test_suite_generation.generators.randoop_test_suite_generator import RandoopTestSuiteGenerator from nimrod.test_suite_generation.generators.evosuite_differential_test_suite_generator import EvosuiteDifferentialTestSuiteGenerator from nimrod.test_suite_generation.generators.evosuite_test_suite_generator import EvosuiteTestSuiteGenerator -from nimrod.test_suite_generation.generators.codellama_test_suite_generator import CodellamaTestSuiteGenerator +from nimrod.test_suite_generation.generators.ollama_test_suite_generator import OllamaTestSuiteGenerator from nimrod.test_suite_generation.generators.project_test_suite_generator import ProjectTestSuiteGenerator from nimrod.test_suites_execution.main import TestSuitesExecution, TestSuiteExecutor from nimrod.tools.bin import MOD_RANDOOP, RANDOOP @@ -27,7 +27,7 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]: config_generators = config.get( - 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'codellama', 'project']) + 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'ollama', 'project']) generators: List[TestSuiteGenerator] = list() if 'randoop' in config_generators: @@ -39,8 +39,8 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator generators.append(EvosuiteTestSuiteGenerator(Java())) if 'evosuite-differential' in config_generators: generators.append(EvosuiteDifferentialTestSuiteGenerator(Java())) - if 'codellama' in config_generators: - generators.append(CodellamaTestSuiteGenerator(Java())) + if 'ollama' in config_generators: + generators.append(OllamaTestSuiteGenerator(Java())) if 'project' in config_generators: generators.append(ProjectTestSuiteGenerator(Java())) diff --git a/nimrod/setup_tools/tools.py b/nimrod/setup_tools/tools.py index c0985757..57b3dcf7 100644 --- a/nimrod/setup_tools/tools.py +++ b/nimrod/setup_tools/tools.py @@ -5,4 +5,4 @@ class Tools(Enum): RANDOOP_MOD='RANDOOP-MODIFIED' EVOSUITE='EVOSUITE' DIFF_EVOSUITE='DIFFERENTIAL-EVOSUITE' - CODELLAMA='CODELLAMA' \ No newline at end of file + OLLAMA='OLLAMA' \ No newline at end of file diff --git a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py similarity index 97% rename from nimrod/test_suite_generation/generators/codellama_test_suite_generator.py rename to nimrod/test_suite_generation/generators/ollama_test_suite_generator.py index 23245ea9..fbd7ea41 100644 --- a/nimrod/test_suite_generation/generators/codellama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py @@ -165,10 +165,10 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s return messages_dict -class CodellamaTestSuiteGenerator(TestSuiteGenerator): +class OllamaTestSuiteGenerator(TestSuiteGenerator): def get_generator_tool_name(self) -> str: - return "CODELLAMA" + return "OLLAMA" def _get_test_suite_class_paths(self, path: str) -> List[str]: paths: List[str] = [] @@ -512,16 +512,16 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s if not api_params: raise ValueError("The 'api_params' section is missing from the configuration file") - if not api_params.get("codellama"): - raise ValueError("The 'codellama' section is missing from the 'api_params' configuration") + if not api_params.get("ollama"): + raise ValueError("The 'ollama' section is missing from the 'api_params' configuration") - codellama_params = api_params.get("codellama", {}) + ollama_params = api_params.get("ollama", {}) self.api = Api( - api_url=codellama_params.get("api_url", "http://localhost:11434/api/chat"), - timeout_seconds=codellama_params.get("timeout_seconds", 60), - temperature=codellama_params.get("temperature", 0), - seed=codellama_params.get("seed", 42), - model=codellama_params.get("model", "codellama:70b") + api_url=ollama_params.get("api_url", "http://localhost:11434/api/chat"), + timeout_seconds=ollama_params.get("timeout_seconds", 60), + temperature=ollama_params.get("temperature", 0), + seed=ollama_params.get("seed", 42), + model=ollama_params.get("model", "codellama:70b") ) # Define paths for storing scenario information (for prompt generation), @@ -532,7 +532,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s time_duration_path = os.path.join( os.path.dirname( os.path.dirname( - os.path.dirname(output_path))), "reports", "codellama_time_duration.json") + os.path.dirname(output_path))), "reports", "ollama_time_duration.json") project_name = scenario.project_name targets = scenario.targets diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 8e7e3708..5740c2b7 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -106,7 +106,7 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str command = self._java.exec_java(test_suite.path, self._java.get_env(), TIMEOUT, *params) output = command.decode('unicode_escape') - if test_suite.generator_name == "CODELLAMA": + if test_suite.generator_name == "OLLAMA": #HSaslThriftClientTest_right_prompt1_0_39.java #test_class_num = "39" #HSaslThriftClientTest_right_prompt1_1_39.java diff --git a/nimrod/tests/env-config.json b/nimrod/tests/env-config.json index 27ca9d3c..6d8efdde 100644 --- a/nimrod/tests/env-config.json +++ b/nimrod/tests/env-config.json @@ -8,10 +8,10 @@ "tests_dst": "", "path_output_csv": "", "logger_level": "DEBUG", - "test_suite_generators":["codellama", "evosuite", "randoop"], + "test_suite_generators":["ollama", "evosuite", "randoop"], "test_suite_generation_search_time_available":"45", "api_params": { - "codellama": { + "ollama": { "model": "codellama:70b", "api_url": "http://ip/api/chat", "temperature": 0.7, diff --git a/nimrod/tools/codellama.py b/nimrod/tools/ollama.py similarity index 77% rename from nimrod/tools/codellama.py rename to nimrod/tools/ollama.py index c1aa1462..939ef04e 100644 --- a/nimrod/tools/codellama.py +++ b/nimrod/tools/ollama.py @@ -4,10 +4,10 @@ from nimrod.utils import get_class_files -class Codellama(SuiteGenerator): +class Ollama(SuiteGenerator): def _get_tool_name(self): - return "codellama" + return "ollama" def _test_classes(self): classes = [] @@ -19,4 +19,4 @@ def _test_classes(self): return classes def _get_suite_dir(self): - return os.path.join(self.suite_dir, 'codellama-tests') + return os.path.join(self.suite_dir, 'ollama-tests') From 0c46b5f180aa7667258195f6e289de39d75b86c9 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Sun, 16 Nov 2025 19:43:10 -0300 Subject: [PATCH 57/69] docs: add docstrings for data loading and writing methods --- nimrod/output_generation/output_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nimrod/output_generation/output_generator.py b/nimrod/output_generation/output_generator.py index b3eb8792..b3cf03ce 100644 --- a/nimrod/output_generation/output_generator.py +++ b/nimrod/output_generation/output_generator.py @@ -41,6 +41,7 @@ def write_report(self, context: OutputGeneratorContext) -> None: logging.info(f"Finished generation of {self._report_name} report") def _load_existing_data(self, file_path: str): + """Loads data stored from previous runs so new data is appended instead of overwriting.""" if not path.exists(file_path): return [] try: @@ -50,5 +51,6 @@ def _load_existing_data(self, file_path: str): return [] def _write_json(self, file_path: str, data) -> None: + """Writes data to a JSON file with indentation for readability.""" with open(file_path, "w") as write_file: json.dump(data, write_file, indent=4) From 421002276c6f3eb7c80d6c58058ce16ce4fa8bf0 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Sun, 16 Nov 2025 19:47:21 -0300 Subject: [PATCH 58/69] Update nimrod/test_suite_generation/generators/ollama_test_suite_generator.py Co-authored-by: Heitor Carvalho --- .../generators/ollama_test_suite_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py index fbd7ea41..b2b40c56 100644 --- a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py @@ -71,7 +71,6 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: try: self.set_payload_messages(messages) response = self.post(self.payload) - #logging.debug("Response: %s", response) return { "response": response.get("message", {}).get("content", "Response not found."), "total_duration": response.get("total_duration", self.timeout_seconds), From f6dc6bcd6170297336a40d86f797a754e1471689 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Sun, 16 Nov 2025 19:50:20 -0300 Subject: [PATCH 59/69] Update nimrod/test_suites_execution/test_suite_executor.py Co-authored-by: Heitor Carvalho --- nimrod/test_suites_execution/test_suite_executor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index 5740c2b7..d47732a0 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -107,10 +107,6 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str output = command.decode('unicode_escape') if test_suite.generator_name == "OLLAMA": - #HSaslThriftClientTest_right_prompt1_0_39.java - #test_class_num = "39" - #HSaslThriftClientTest_right_prompt1_1_39.java - #test_class_num = "139" parts = test_class.replace(".java", "").split("_") if len(parts) >= 2: test_class_num = parts[-2] + parts[-1] From cca64e7cab6dd8bf6c74618a180a5c16323cbdbb Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Sun, 16 Nov 2025 19:56:15 -0300 Subject: [PATCH 60/69] fix(tests): update compile timeout and standardize assertion methods --- nimrod/tests/test_utils.py | 2 +- nimrod/tests/tools/test_maven.py | 4 ++-- nimrod/tests/tools/test_mujava.py | 2 +- nimrod/tools/evosuite.py | 2 +- nimrod/tools/jacoco.py | 2 +- nimrod/tools/junit.py | 4 ++-- pytest.ini | 6 ++++++ 7 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 pytest.ini diff --git a/nimrod/tests/test_utils.py b/nimrod/tests/test_utils.py index 4c89c24f..40449164 100644 --- a/nimrod/tests/test_utils.py +++ b/nimrod/tests/test_utils.py @@ -22,7 +22,7 @@ def setUp(self): self.java = Java(self.java_home) self.maven = Maven(self.java, self.maven_home) - self.maven.compile(calculator_project_dir(), 10) + self.maven.compile(calculator_project_dir(), 20) def test_get_files(self): classes = get_files(calculator_target_dir()) diff --git a/nimrod/tests/tools/test_maven.py b/nimrod/tests/tools/test_maven.py index 6e31155c..6ded82db 100644 --- a/nimrod/tests/tools/test_maven.py +++ b/nimrod/tests/tools/test_maven.py @@ -81,8 +81,8 @@ def test_extract_results(self): '/a/b/c/target/classes\n[INFO] 0asdjhaskdjf Compiling') results = Maven.extract_results(output) - self.assertEquals(6, results.source_files) - self.assertEquals('/a/b/c/target/classes', results.classes_dir) + self.assertEqual(6, results.source_files) + self.assertEqual('/a/b/c/target/classes', results.classes_dir) @staticmethod def _clear_environment(): diff --git a/nimrod/tests/tools/test_mujava.py b/nimrod/tests/tools/test_mujava.py index 81b54bc0..3829662b 100644 --- a/nimrod/tests/tools/test_mujava.py +++ b/nimrod/tests/tools/test_mujava.py @@ -42,7 +42,7 @@ def test_read_log_without_log_dir(self): mujava = MuJava(self.java, calculator_mutants_dir()) mutants = mujava.read_log() - self.assertEquals(3, len(mutants)) + self.assertEqual(3, len(mutants)) def test_not_found_log(self): mujava = MuJava(self.java, calculator_mutants_dir()) diff --git a/nimrod/tools/evosuite.py b/nimrod/tools/evosuite.py index 7bed4412..0ca96e6a 100644 --- a/nimrod/tools/evosuite.py +++ b/nimrod/tools/evosuite.py @@ -97,7 +97,7 @@ def generate_differential(self, mutant_classpath, make_dir=True): def get_format_evosuite_method_name(self): method_name = "" try: - pattern = re.compile("\.[a-zA-Z0-9\-\_]*\([\s\S]*") + pattern = re.compile(r"\.[a-zA-Z0-9\-\_]*\([\s\S]*") result = pattern.search(self.sut_method) method_name = result.group(0)[1:] except Exception as e: diff --git a/nimrod/tools/jacoco.py b/nimrod/tools/jacoco.py index f85cd64b..d0017e33 100644 --- a/nimrod/tools/jacoco.py +++ b/nimrod/tools/jacoco.py @@ -48,7 +48,7 @@ def dealingWithDuplicatedFilesOnJars(self, jarFile, message_error): def parseDuplicatedFile(self, message_error): "Exception in thread \"main\" java.util.zip.ZipException: duplicate entry: META-INF/LICENSE.txt" - x = re.search("duplicate entry\: .*", message_error, re.IGNORECASE) + x = re.search(r"duplicate entry: .*", message_error, re.IGNORECASE) if x: fileName = str(message_error.split("duplicate entry: ")[1]).split("\n")[0] if ("$" in fileName): diff --git a/nimrod/tools/junit.py b/nimrod/tools/junit.py index 0683ed5d..bb7a8385 100644 --- a/nimrod/tools/junit.py +++ b/nimrod/tools/junit.py @@ -131,9 +131,9 @@ def _extract_test_id(output): list_failed_tests = [] list_failed_tests = re.findall(r'test[0-9]+\([A-Za-z0-9_.]+\)', output) - number_executed_tests = int(re.findall('Tests run: \d+', output)[0].split("Tests run: ")[-1]) + number_executed_tests = int(re.findall(r'Tests run: \d+', output)[0].split("Tests run: ")[-1]) for test in list_failed_tests: - i = re.findall('\d+', test) + i = re.findall(r'\d+', test) test_case = re.findall(r'.+?(?=\()', test)[0] file = str(re.findall(r'\(.+?(?=\))', test)[0]).split(".")[-1] #re.findall(r'\(.+?(?=\))', test)[0][1:].to_s.split(".")[-1] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..200ef1ee --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[tool:pytest] +python_classes = *Test *Tests +python_functions = test_* +addopts = --tb=short +filterwarnings = + ignore::pytest.PytestCollectionWarning \ No newline at end of file From 5a054358b633cb636e31630f310b494bb7556bf1 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 10:57:41 -0300 Subject: [PATCH 61/69] feat: add PromptManager class and prompt templates --- .../generators/prompt_manager.py | 158 ++++++++++++++++++ .../generators/prompt_templates.json | 82 +++++++++ 2 files changed, 240 insertions(+) create mode 100644 nimrod/test_suite_generation/generators/prompt_manager.py create mode 100644 nimrod/test_suite_generation/generators/prompt_templates.json diff --git a/nimrod/test_suite_generation/generators/prompt_manager.py b/nimrod/test_suite_generation/generators/prompt_manager.py new file mode 100644 index 00000000..f0913bb6 --- /dev/null +++ b/nimrod/test_suite_generation/generators/prompt_manager.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +import json +import os +from typing import Dict, List, Any, Optional + + +class PromptManager: + def __init__(self, config_path: Optional[str] = None): + if config_path is None: + current_dir = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(current_dir, "prompt_templates.json") + self.config_path = config_path + self.config = self._load_config() + + def _load_config(self) -> Dict[str, Any]: + try: + with open(self.config_path, 'r', encoding='utf-8') as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError, IOError): + return {"prompt_templates": {"zero_shot": {}}} + + def _format_template(self, template: Dict[str, str], **kwargs) -> Dict[str, str]: + formatted = template.copy() + if "content" in formatted: + formatted["content"] = formatted["content"].format(**kwargs) + return formatted + + def _prepare_context_data(self, method_info: Dict[str, Any]) -> Dict[str, str]: + class_fields = method_info.get("class_fields", []) + class_fields_str = "\n".join(class_fields) if class_fields else "" + + constructors = method_info.get("constructor_codes", []) + constructors_str = "\n".join(constructors) if constructors else "" + + return { + "class_fields": class_fields_str, + "constructors": constructors_str, + "method_code": method_info.get("method_code", ""), + "left_changes_summary": method_info.get("left_changes_summary", ""), + "right_changes_summary": method_info.get("right_changes_summary", "") + } + + def generate_messages_for_template(self, template_name: str, method_info: Dict[str, Any], + class_name: str, branch: str, context_keys: Optional[List[str]] = None) -> List[Dict[str, str]]: + template_config = self.config.get("prompt_templates", {}).get(template_name, {}) + + context_data = self._prepare_context_data(method_info) + context_data.update({"class_name": class_name, "branch": branch, "method_name": method_info.get("method_name", "")}) + + messages = [] + + if template_name == "zero_shot": + if "system_message" in template_config: + messages.append(self._format_template(template_config["system_message"], **context_data)) + + if "user_init_message" in template_config: + messages.append(self._format_template(template_config["user_init_message"], **context_data)) + + if context_keys: + context_templates = template_config.get("context_templates", {}) + for key in context_keys: + if key in context_templates: + messages.append(self._format_template(context_templates[key], **context_data)) + + if "method_context_message" in template_config: + messages.append(self._format_template(template_config["method_context_message"], **context_data)) + + elif template_name == "one_shot": + if "system_message" in template_config: + messages.append(self._format_template(template_config["system_message"], **context_data)) + + if "user_init_message" in template_config: + messages.append(self._format_template(template_config["user_init_message"], **context_data)) + + if context_keys: + example_context = template_config.get("example_context", {}) + for key in context_keys: + if key in example_context: + messages.append(example_context[key]) + + if "example_method_message" in template_config: + messages.append(template_config["example_method_message"]) + + if "example_response" in template_config: + messages.append(template_config["example_response"]) + + if "user_init_message" in template_config: + messages.append(self._format_template(template_config["user_init_message"], **context_data)) + + if context_keys: + context_templates = template_config.get("context_templates", {}) + for key in context_keys: + if key in context_templates: + messages.append(self._format_template(context_templates[key], **context_data)) + + if "method_context_message" in template_config: + messages.append(self._format_template(template_config["method_context_message"], **context_data)) + + return messages + + def generate_all_combinations(self, method_info: Dict[str, Any], class_name: str, + branch: str, template_name: str = "zero_shot") -> Dict[str, List[Dict[str, str]]]: + """Generates the 8 specific combinations for zero-shot or one-shot""" + if template_name == "zero_shot": + combinations = [ + [], # prompt1 + ["changes_summary"], # prompt2 + ["class_fields"], # prompt3 + ["constructors"], # prompt4 + ["changes_summary", "class_fields"], # prompt5 + ["changes_summary", "constructors"], # prompt6 + ["class_fields", "constructors"], # prompt7 + ["changes_summary", "class_fields", "constructors"] # prompt8 + ] + elif template_name == "one_shot": + combinations = [ + [], # prompt1: sem contexto + ["changes_summary"], # prompt2 + ["class_fields"], # prompt3 + ["constructors"], # prompt4 + ["changes_summary", "class_fields"], # prompt5 + ["changes_summary", "constructors"], # prompt6 + ["class_fields", "constructors"], # prompt7 + ["changes_summary", "class_fields", "constructors"] # prompt8 + ] + else: + return {} + + messages_dict = {} + for i, context_keys in enumerate(combinations, 1): + messages_dict[f"prompt{i}"] = self.generate_messages_for_template( + template_name, method_info, class_name, branch, context_keys + ) + + return messages_dict + + def save_generated_messages(self, messages_dict: Dict[str, List[Dict[str, str]]], + output_path: str, class_name: str, method_name: str) -> None: + output_file_path = os.path.join(output_path, "generated_messages.json") + + try: + if os.path.exists(output_file_path): + with open(output_file_path, "r", encoding='utf-8') as file: + existing_data = json.load(file) + else: + existing_data = {} + except (FileNotFoundError, json.JSONDecodeError, IOError): + existing_data = {} + + if class_name not in existing_data: + existing_data[class_name] = {} + + existing_data[class_name][method_name] = messages_dict + + os.makedirs(output_path, exist_ok=True) + with open(output_file_path, "w", encoding='utf-8') as file: + json.dump(existing_data, file, indent=4, ensure_ascii=False) \ No newline at end of file diff --git a/nimrod/test_suite_generation/generators/prompt_templates.json b/nimrod/test_suite_generation/generators/prompt_templates.json new file mode 100644 index 00000000..d895e03d --- /dev/null +++ b/nimrod/test_suite_generation/generators/prompt_templates.json @@ -0,0 +1,82 @@ +{ + "prompt_templates": { + "zero_shot": { + "system_message": { + "role": "system", + "content": "You are a senior Java developer with expertise in JUnit testing.\nYour task is to provide JUnit tests for the given method in the class under test, considering the changes introduced in the left and right branches.\nYou have to answer with the test code only, inside code blocks (```).\nThe tests should start with @Test." + }, + "user_init_message": { + "role": "user", + "content": "Here is the context of the method under test in the class {class_name} on the {branch} branch:" + }, + "context_templates": { + "changes_summary": { + "role": "user", + "content": "Left {left_changes_summary}.\nRight {right_changes_summary}" + }, + "class_fields": { + "role": "user", + "content": "Class fields:\n{class_fields}" + }, + "constructors": { + "role": "user", + "content": "Constructors:\n{constructors}" + } + }, + "method_context_message": { + "role": "user", + "content": "Target Method Under Test:\n{method_code}\n\nNow generate JUnit tests for the method under test, considering the given context. Remember to create meaningful assertions.\nWrite all tests inside code blocks (```), and start each test with @Test." + } + }, + "one_shot": { + "system_message": { + "role": "system", + "content": "You are a senior Java developer with expertise in JUnit testing.Your only task is to generate JUnit test methods based on the provided Java class details, using Java syntax.Follow these guidelines:1. Provide only the JUnit test code, written in Java syntax, and nothing else.2. Fully implement the test methods, including the actual test logic (assertEquals, assertTrue, etc.), starting with @Test.3. Exclude any setup/teardown code or content outside the test method itself.4. You have to answer with the test code only, inside code blocks (```).5. Use comment blocks /* */ or // for extra text, such as comments, titles, explanations, or any additional details within the code.6. Ensure that the generated output is completely functional as code, compiles successfully, and runs without errors." + }, + "user_init_message": { + "role": "user", + "content": "Below, you will find additional information regarding the Class Under Test:" + }, + "example_context": { + "changes_summary": { + "role": "user", + "content": "Left wanted to extend the method cleanText() to also remove duplicated whitespace in the text by adding the method call normalizeWhitespace().\nRight wanted to clean the text by removing consecutive duplicated words." + }, + "class_fields": { + "role": "user", + "content": "Class fields:\npublic String text;\n" + }, + "constructors": { + "role": "user", + "content": "Constructors:\npublic DFPBaseSample(String text) {\n this.text = text;\n }\n" + } + }, + "example_method_message": { + "role": "user", + "content": "Target Method Under Test:\npublic void cleanText() {\n DFPBaseSample inst = new DFPBaseSample(text);\n inst.normalizeWhiteSpace();\n inst.removeComments();\n this.text = inst.text;\n }\n\nTests for the method 'cleanText()' in the class 'DFPBaseSample':" + }, + "example_response": { + "role": "assistant", + "content": "```@Test\npublic void test00() {\n DFPBaseSample sample = new DFPBaseSample(\"This is a sample text\");\n sample.cleanText();\n assertEquals(\"This is a sample text\", sample.getText());\n}\n\n@Test\npublic void test01() {\n DFPBaseSample sample1 = new DFPBaseSample(\"Hello World\");\n DFPBaseSample sample2 = sample1;\n sample1.cleanText();\n sample2.normalizeWhiteSpace();\n sample2.removeComments();\n assertEquals(sample1.getText(), sample2.getText());\n}```" + }, + "context_templates": { + "changes_summary": { + "role": "user", + "content": "{left_changes_summary}\n{right_changes_summary}" + }, + "class_fields": { + "role": "user", + "content": "Class fields:\n{class_fields}" + }, + "constructors": { + "role": "user", + "content": "Constructors:\n{constructors}" + } + }, + "method_context_message": { + "role": "user", + "content": "Target Method Under Test:\n{method_code}\n\nTests for the method '{method_name}' in the class '{class_name}':" + } + } + } +} \ No newline at end of file From 5784994a5a231c4593be6b3787b34b5c42be3a52 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 10:57:56 -0300 Subject: [PATCH 62/69] fix: update paths for compilation and execution results to use reports directory --- .../generators/test_suite_generator.py | 5 ++++- .../test_suites_execution/test_suite_executor.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/nimrod/test_suite_generation/generators/test_suite_generator.py b/nimrod/test_suite_generation/generators/test_suite_generator.py index 391eb9d5..d71c9fa2 100644 --- a/nimrod/test_suite_generation/generators/test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/test_suite_generator.py @@ -69,7 +69,10 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: def _update_compilation_results(self, test_suite_path: str, java_file: str, output: str) -> None: """Updates the compilation results file with the output of the compilation of a test suite class.""" - COMPILATION_LOG_FILE = "compilation_results.json" + reports_dir = path.join(path.dirname(get_base_output_path()), "reports") + COMPILATION_LOG_FILE = path.join(reports_dir, "compilation_results.json") + + makedirs(reports_dir, exist_ok=True) if path.exists(COMPILATION_LOG_FILE): with open(COMPILATION_LOG_FILE, "r", encoding="utf-8") as f: diff --git a/nimrod/test_suites_execution/test_suite_executor.py b/nimrod/test_suites_execution/test_suite_executor.py index d47732a0..c1e40466 100644 --- a/nimrod/test_suites_execution/test_suite_executor.py +++ b/nimrod/test_suites_execution/test_suite_executor.py @@ -2,7 +2,7 @@ import re import subprocess import json -from os import path +from os import path, makedirs from typing import Dict, List, Optional from nimrod.test_suite_generation.test_suite import TestSuite from nimrod.test_suites_execution.test_case_result import TestCaseResult @@ -12,15 +12,17 @@ from nimrod.tools.jacoco import Jacoco from nimrod.utils import generate_classpath -EXECUTION_LOG_FILE = "execution_results.json" +reports_dir = path.join(path.dirname(get_base_output_path()), "reports") +makedirs(reports_dir, exist_ok=True) +EXECUTION_LOG_FILE = path.join(reports_dir, "execution_results.json") def is_failed_caused_by_compilation_problem(test_case_name: str, failed_test_message: str) -> bool: my_regex = re.escape(test_case_name) + r"[0-9A-Za-z0-9_\(\.\)\n \:]+(NoSuchMethodError|NoSuchFieldError|NoSuchClassError|NoClassDefFoundError|NoSuchAttributeError|tried to access method)" - return re.search(my_regex, failed_test_message) != None + return re.search(my_regex, failed_test_message) is not None def is_failed_caused_by_error(test_case_name: str, failed_test_message: str) -> bool: my_regex = re.escape(test_case_name) + r"[0-9A-Za-z0-9_(.)]RegressionTest[0-9A-Za-z0-9_(.)\n]+Exception" - return re.search(my_regex, failed_test_message) != None + return re.search(my_regex, failed_test_message) is not None def get_result_for_test_case(failed_test: str, output: str) -> TestCaseResult: if is_failed_caused_by_compilation_problem(failed_test, output): @@ -106,7 +108,8 @@ def _execute_junit(self, test_suite: TestSuite, target_jar: str, test_class: str command = self._java.exec_java(test_suite.path, self._java.get_env(), TIMEOUT, *params) output = command.decode('unicode_escape') - if test_suite.generator_name == "OLLAMA": + # Special handling for LLM-generated test classes + if test_suite.generator_name not in ["RANDOOP", "EVOSUITE", "RANDOOP_MODIFIED", "EVOSUITE_DIFFERENTIAL", "PROJECT_TEST"]: parts = test_class.replace(".java", "").split("_") if len(parts) >= 2: test_class_num = parts[-2] + parts[-1] @@ -189,5 +192,5 @@ def _execute_junit_5(self, test_suite: TestSuite, target_jar: str, test_targets: ['org.junit.platform.console.ConsoleLauncher'] + test_targets return self._java.exec_java(test_suite.path, self._java.get_env(), TIMEOUT, *params) - except subprocess.CalledProcessError as error: + except subprocess.CalledProcessError: return None From 7f435b5c07a922c6112208daf3d3b0a2768ff4a7 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 10:58:48 -0300 Subject: [PATCH 63/69] feat: enhance OllamaTestSuiteGenerator to support configurable model parameters and prompt templates --- nimrod/__main__.py | 16 +- .../generators/ollama_test_suite_generator.py | 231 ++++++++++-------- nimrod/tests/env-config.json | 3 +- 3 files changed, 142 insertions(+), 108 deletions(-) diff --git a/nimrod/__main__.py b/nimrod/__main__.py index 8c610493..d5604f12 100644 --- a/nimrod/__main__.py +++ b/nimrod/__main__.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, List +from typing import Dict, List, Any from nimrod.dynamic_analysis.behavior_change_checker import BehaviorChangeChecker from nimrod.dynamic_analysis.criteria.first_semantic_conflict_criteria import FirstSemanticConflictCriteria from nimrod.dynamic_analysis.criteria.second_semantic_conflict_criteria import SecondSemanticConflictCriteria @@ -25,7 +25,7 @@ from nimrod.input_parsing.input_parser import CsvInputParser, JsonInputParser -def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]: +def get_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator]: config_generators = config.get( 'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'ollama', 'project']) generators: List[TestSuiteGenerator] = list() @@ -40,14 +40,20 @@ def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator if 'evosuite-differential' in config_generators: generators.append(EvosuiteDifferentialTestSuiteGenerator(Java())) if 'ollama' in config_generators: - generators.append(OllamaTestSuiteGenerator(Java())) + # Create one generator instance for each configured model + api_params = config.get('api_params', {}) + if api_params: + for model_key, model_config in api_params.items(): + generators.append(OllamaTestSuiteGenerator(Java(), model_key, model_config)) + else: + generators.append(OllamaTestSuiteGenerator(Java())) if 'project' in config_generators: generators.append(ProjectTestSuiteGenerator(Java())) return generators -def get_output_generators(config: Dict[str, str]) -> List[OutputGenerator]: +def get_output_generators(config: Dict[str, Any]) -> List[OutputGenerator]: config_generators = config.get( 'output_generators', ['behavior_changes', 'semantic_conflicts', 'test_suites']) generators: List[OutputGenerator] = list() @@ -63,7 +69,7 @@ def get_output_generators(config: Dict[str, str]) -> List[OutputGenerator]: return generators -def parse_scenarios_from_input(config: Dict[str, str]) -> List[MergeScenarioUnderAnalysis]: +def parse_scenarios_from_input(config: Dict[str, Any]) -> List[MergeScenarioUnderAnalysis]: json_input = config.get('input_path', "") csv_input_path = config.get('path_hash_csv', "") diff --git a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py index b2b40c56..e7d589a4 100644 --- a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py @@ -1,16 +1,16 @@ import json import logging import os -import requests +import requests # type: ignore from typing import List, Dict, Union, Any, Optional -from itertools import combinations import re import tree_sitter_java as tsjava -from tree_sitter import Language, Parser +from tree_sitter import Language, Parser, QueryCursor from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator +from nimrod.test_suite_generation.generators.prompt_manager import PromptManager from nimrod.tests.utils import get_config from nimrod.utils import load_json, save_json @@ -73,101 +73,119 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: response = self.post(self.payload) return { "response": response.get("message", {}).get("content", "Response not found."), - "total_duration": response.get("total_duration", self.timeout_seconds), + "total_duration": response.get("total_duration", self.timeout_seconds * 1_000_000_000), } except Exception as e: logging.error(f"Error generating output: {e}") - return {"error": "Output generation error"} + return {"error": "Output generation error", "total_duration": self.timeout_seconds * 1_000_000_000} + +class OllamaTestSuiteGenerator(TestSuiteGenerator): + + def __init__(self, java_tool, model_key: str = "codellama", model_config: Optional[Dict[str, Any]] = None): + super().__init__(java_tool) + self.model_key = model_key + self.model_config = model_config or {} + self.api: Optional[Api] = None + self.prompt_manager = PromptManager() + + # Loads global configurations + global_config = get_config() + + # Prompt configurations (priority: model_config > global_config > default) + self.prompt_template = ( + self.model_config.get("prompt_template") or + global_config.get("prompt_template") or + "zero_shot" + ) + + logging.info(f"Initialized {self.model_key} with prompt template: {self.prompt_template}") + def generate_messages_list(self, method_info: Dict[str, str], full_class_name: str, branch: str, output_path: str) -> Dict[str, List[Dict[str, str]]]: """ - Generates the messages for the API requests. - Each list of messages contains different information about the method under test. + Generates messages for API requests using the configurable prompt system. + Supports different templates (zero-shot, one-shot) and context combinations. """ - self.set_branch(branch) # Set the branch + api = self._get_api_instance() + api.set_branch(branch) # Set the branch class_name = full_class_name.split('.')[-1] method_name = method_info.get("method_name", "") - class_fields: Union[str, List[str]] = method_info.get("class_fields", []) - constructor_codes: Union[str, List[str]] = method_info.get("constructor_codes", []) - method_code = method_info.get("method_code", "") - left_changes_summary = method_info.get("left_changes_summary", "") - right_changes_summary = method_info.get("right_changes_summary", "") - - - ###################################################### - ## PROMPTS INCREMENTAL COMBINATIONS - system_message = { - "role": "system", - "content": ( - "You are a senior Java developer with expertise in JUnit testing.\n" - "Your task is to provide JUnit tests for the given method in the class under test, " - "considering the changes introduced in the left and right branches.\n" - "You have to answer with the test code only, inside code blocks (```).\n" - "The tests should start with @Test." - ), - } - - user_init_msg = { - "role": "user", - "content": f"""Here is the context of the method under test in the class {class_name} on the {branch} branch:""", - } + + logging.debug(f"Generating messages for {class_name}.{method_name} using template: {self.prompt_template}") + + # Generates messages using the PromptManager + messages_dict = self.prompt_manager.generate_all_combinations( + method_info=method_info, + class_name=class_name, + branch=branch, + template_name=self.prompt_template + ) + + # Saves generated messages + self.prompt_manager.save_generated_messages( + messages_dict=messages_dict, + output_path=output_path, + class_name=class_name, + method_name=method_name + ) + + logging.info(f"Generated {len(messages_dict)} prompt variations for {class_name}.{method_name}") - user_msg_templates = [ - {"role": "user", "content": f"{left_changes_summary}\n{right_changes_summary}"}, - {"role": "user", "content": f"Class fields:\n" + "\n".join(class_fields)}, - {"role": "user", "content": f"Constructors:\n" + "\n".join(constructor_codes)}, - ] - - user_method_ctx_msg = { - "role": "user", - "content": ( - f"Target Method Under Test:\n{method_code}\n\n" - "Now generate JUnit tests for the method under test, considering the given context. Remember to create meaningful assertions.\n" - "Write all tests inside code blocks (```), and start each test with @Test." - ), - } + return messages_dict + + def _ensure_api_initialized(self) -> Api: + """Initializes the API if it has not been initialized yet and returns the instance.""" + if self.api is not None: + return self.api + + config = get_config() + api_params = config.get("api_params", {}) + if not api_params: + raise ValueError("The 'api_params' section is missing from the configuration file") - messages_dict: Dict[str, List[Dict[str, str]]] = {} - counter = 1 - - ## PROMPT SEM CONTEXTO - key = f"prompt{counter}" - messages_list = [system_message, user_init_msg, user_method_ctx_msg] - messages_dict[key] = messages_list - counter += 1 - - ## PROMPTS COM CONTEXTO - for r in range(1, len(user_msg_templates) + 1): - for user_msgs_combination in combinations(user_msg_templates, r): - key = f"prompt{counter}" - messages_list = [system_message, user_init_msg, *user_msgs_combination, user_method_ctx_msg] - messages_dict[key] = messages_list - counter += 1 - - # Save messages to a JSON file - output_file_path = os.path.join(output_path, "generated_messages.json") - if os.path.exists(output_file_path): - with open(output_file_path, "r") as file: - existing_data = json.load(file) + # Use the specific model configuration for this generator instance + if self.model_config: + model_params = self.model_config else: - existing_data = {} + if not api_params.get(self.model_key): + raise ValueError(f"The '{self.model_key}' section is missing from the 'api_params' configuration") + model_params = api_params.get(self.model_key, {}) - if class_name not in existing_data: - existing_data[class_name] = {} - - existing_data[class_name][method_info["method_name"]] = messages_dict + self.api = Api( + api_url=model_params.get("api_url", "http://localhost:11434/api/chat"), + timeout_seconds=model_params.get("timeout_seconds", 60), + temperature=model_params.get("temperature", 0), + seed=model_params.get("seed", 42), + model=model_params.get("model", "codellama:70b") + ) + return self.api - with open(output_file_path, "w") as file: - json.dump(existing_data, file, indent=4) + def _get_api_instance(self) -> Api: + """Returns a valid API instance, initializing it if necessary.""" + return self._ensure_api_initialized() - return messages_dict + def get_generator_tool_name(self) -> str: + self._get_api_instance() + config_suffix = self._generate_config_suffix() + return f"{self.model_key.upper()}{config_suffix}" + + def _generate_config_suffix(self) -> str: + """Generates a suffix with configuration information for folder identification""" + if not self.api: + return "" + # Prompt format: ZS (zero-shot) or 1S (one-shot) + prompt_code = "ZS" if self.prompt_template == "zero_shot" else "1S" -class OllamaTestSuiteGenerator(TestSuiteGenerator): + # Temperature: T00, T05, T07, etc. (always with 2 digits) + temp_value = int(self.api.temperature * 100) # 0.7 -> 70, 0.05 -> 5, 0 -> 0 + temp_code = f"T{temp_value:02d}" # Formats with 2 digits: T00, T05, T07 - def get_generator_tool_name(self) -> str: - return "OLLAMA" + # Seed: S123, S42, etc. + seed_code = f"S{self.api.seed}" + + return f"_{prompt_code}_{temp_code}_{seed_code}" def _get_test_suite_class_paths(self, path: str) -> List[str]: paths: List[str] = [] @@ -246,7 +264,13 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_ ) """ query = JAVA_LANGUAGE.query(query_text) - captures = query.captures(tree.root_node) + cursor = QueryCursor(query) + captures_dict = cursor.captures(tree.root_node) + + captures = [] + for capture_name, nodes in captures_dict.items(): + for node in nodes: + captures.append((node, capture_name)) if not captures: raise Exception(f"No captures found for the class '{class_name}' in '{source_code_path}'") @@ -337,7 +361,13 @@ def save_imports(self, class_name: str, source_code_path: str, imports_path: str (package_declaration) @package """ query = JAVA_LANGUAGE.query(query_text) - captures = query.captures(tree.root_node) + cursor = QueryCursor(query) + captures_dict = cursor.captures(tree.root_node) + + captures = [] + for capture_name, nodes in captures_dict.items(): + for node in nodes: + captures.append((node, capture_name)) if os.path.exists(imports_path): imports_dict = load_json(imports_path) @@ -396,7 +426,13 @@ def classify_annotations(captures: List[tuple], source_code: str) -> tuple: )) @method_def """ query = JAVA_LANGUAGE.query(query_text) - captures = query.captures(tree.root_node) + cursor = QueryCursor(query) + captures_dict = cursor.captures(tree.root_node) + + captures = [] + for capture_name, nodes in captures_dict.items(): + for node in nodes: + captures.append((node, capture_name)) before_block, test_block = classify_annotations(captures, source_code) for test in test_block: @@ -506,32 +542,21 @@ def record_output_duration(self, time_duration_path: str, output_path: str, clas raise def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, scenario: MergeScenarioUnderAnalysis, use_determinism: bool) -> None: - config = get_config() - api_params = config.get("api_params", {}) - if not api_params: - raise ValueError("The 'api_params' section is missing from the configuration file") - - if not api_params.get("ollama"): - raise ValueError("The 'ollama' section is missing from the 'api_params' configuration") - - ollama_params = api_params.get("ollama", {}) - self.api = Api( - api_url=ollama_params.get("api_url", "http://localhost:11434/api/chat"), - timeout_seconds=ollama_params.get("timeout_seconds", 60), - temperature=ollama_params.get("temperature", 0), - seed=ollama_params.get("seed", 42), - model=ollama_params.get("model", "codellama:70b") - ) + self._get_api_instance() # Define paths for storing scenario information (for prompt generation), # importing data (to be extracted from source code), and recording time duration (for each output) scenario_infos_path = os.path.join(output_path, "scenario_infos.json") imports_path = os.path.join(output_path, "imports.json") # Save time duration data in the 'reports' folder, located next to the 'projects' folder + # Generates config suffix for the duration file + config_suffix = self._generate_config_suffix().replace("_", "") if hasattr(self, '_generate_config_suffix') else "" + duration_filename = f"{self.model_key}{config_suffix}_time_duration.json" + time_duration_path = os.path.join( os.path.dirname( os.path.dirname( - os.path.dirname(output_path))), "reports", "ollama_time_duration.json") + os.path.dirname(output_path))), "reports", duration_filename) project_name = scenario.project_name targets = scenario.targets @@ -551,7 +576,7 @@ def _execute_tool_for_tests_generation(self, input_jar: str, output_path: str, s for class_name, scenario_infos_list in scenario_infos_dict.items(): logging.debug("Generating tests for target methods in class '%s'", class_name) for i, method_info in enumerate(scenario_infos_list): - messages_list = self.api.generate_messages_list(method_info, class_name, branch, output_path) + messages_list = self.generate_messages_list(method_info, class_name, branch, output_path) test_template = method_info.get("test_template", "") self._process_prompts(messages_list=messages_list, test_template=test_template, output_path=output_path, branch=branch, class_name=class_name, imports=imports_dict.get(class_name, []), @@ -568,11 +593,13 @@ def _process_prompts(self, messages_list: Dict[str, List[Dict[str, str]]], test_ def _process_single_prompt(self, messages: List[Dict[str, str]], test_template: str, output_path: str, branch: str, class_name: str, imports: List[str], i: int, j: int, time_duration_path: str, project_name: str, output_file_name: str, prompt_key: str) -> None: + api = self._get_api_instance() + total_duration = api.timeout_seconds * 1_000_000_000 # Initialize with timeout value in nanoseconds try: logging.debug("Processing output %d%d for prompt key '%s' in branch \"%s\"", i, j, prompt_key, branch) - output = self.api.generate_output(messages) + output = api.generate_output(messages) response = output.get("response", "Response not found.") - total_duration = int(output.get("total_duration", self.api.timeout_seconds)) + total_duration = int(output.get("total_duration", api.timeout_seconds * 1_000_000_000)) self.save_output(test_template, response, output_path, output_file_name) except Exception as e: logging.error("Error while processing output %d%d for prompt key '%s' in branch \"%s\": %s", i, j, prompt_key, branch, e) diff --git a/nimrod/tests/env-config.json b/nimrod/tests/env-config.json index 6d8efdde..9934a146 100644 --- a/nimrod/tests/env-config.json +++ b/nimrod/tests/env-config.json @@ -10,8 +10,9 @@ "logger_level": "DEBUG", "test_suite_generators":["ollama", "evosuite", "randoop"], "test_suite_generation_search_time_available":"45", + "prompt_template": "zero_shot", "api_params": { - "ollama": { + "codellama-70b": { "model": "codellama:70b", "api_url": "http://ip/api/chat", "temperature": 0.7, From e99d5d0c2da4198c7e10009bdacbe93abd6b31b4 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 11:17:02 -0300 Subject: [PATCH 64/69] feat: implement LLMOutputProcessor for output sanitization in OllamaTestSuiteGenerator --- .../generators/llm_output_processor.py | 108 ++++++++++++++++++ .../generators/ollama_test_suite_generator.py | 31 ++--- 2 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 nimrod/test_suite_generation/generators/llm_output_processor.py diff --git a/nimrod/test_suite_generation/generators/llm_output_processor.py b/nimrod/test_suite_generation/generators/llm_output_processor.py new file mode 100644 index 00000000..d49caafa --- /dev/null +++ b/nimrod/test_suite_generation/generators/llm_output_processor.py @@ -0,0 +1,108 @@ +import re +from typing import List, Optional +from abc import ABC, abstractmethod + + +class OutputSanitizationRule(ABC): + """Abstract base class for output sanitization rules.""" + + @abstractmethod + def apply(self, output: str) -> str: + """Apply the sanitization rule to the output.""" + pass + + +class RemoveThinkTagsRule(OutputSanitizationRule): + """Removes content between tags.""" + + def apply(self, output: str) -> str: + return re.sub(r'.*?', '', output, flags=re.DOTALL) + + +class ExtractCodeBlocksRule(OutputSanitizationRule): + """Extracts content from code blocks (``` markers).""" + + def apply(self, output: str) -> str: + matches = re.findall(r'```(?:\w+)?\n?(.*?)```', output, flags=re.DOTALL) + return '\n'.join(matches).strip() + + +class RemoveNumberedLinesRule(OutputSanitizationRule): + """Removes lines starting with 'number. ' pattern.""" + + def apply(self, output: str) -> str: + return re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) + + +class ExtractFromAnnotationsRule(OutputSanitizationRule): + """Keeps only content starting from the first annotation marker.""" + + def __init__(self, markers: Optional[List[str]] = None): + self.markers = markers or ["@Before", "@BeforeClass", "@Test"] + + def apply(self, output: str) -> str: + index = min( + (output.find(marker) for marker in self.markers if marker in output), + default=-1 + ) + return output[index:] if index != -1 else output + + +class LLMOutputProcessor: + """ + Processes and sanitizes LLM outputs using configurable rules. + + This class provides a flexible framework for cleaning LLM outputs + by applying a series of sanitization rules in sequence. + """ + + def __init__(self): + self._rules: List[OutputSanitizationRule] = [] + self._load_default_rules() + + def _load_default_rules(self) -> None: + """Load the default set of sanitization rules.""" + self._rules = [ + RemoveThinkTagsRule(), + ExtractCodeBlocksRule(), + RemoveNumberedLinesRule(), + ExtractFromAnnotationsRule() + ] + + def add_rule(self, rule: OutputSanitizationRule) -> None: + """Add a custom sanitization rule.""" + self._rules.append(rule) + + def remove_rule(self, rule_type: type) -> None: + """Remove all rules of the specified type.""" + self._rules = [rule for rule in self._rules if not isinstance(rule, rule_type)] + + def clear_rules(self) -> None: + """Remove all sanitization rules.""" + self._rules.clear() + + def process(self, output: str) -> str: + """ + Process the LLM output by applying all sanitization rules in sequence. + + Args: + output: The raw output from the LLM model + + Returns: + The processed and sanitized output + """ + processed_output = output + + for rule in self._rules: + try: + processed_output = rule.apply(processed_output) + except Exception as e: + # Log the error but continue processing with other rules + import logging + logging.warning(f"Error applying rule {rule.__class__.__name__}: {e}") + + return processed_output + + def get_active_rules(self) -> List[str]: + """Get the names of currently active rules.""" + return [rule.__class__.__name__ for rule in self._rules] diff --git a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py index e7d589a4..43009207 100644 --- a/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py +++ b/nimrod/test_suite_generation/generators/ollama_test_suite_generator.py @@ -11,6 +11,7 @@ from nimrod.core.merge_scenario_under_analysis import MergeScenarioUnderAnalysis from nimrod.test_suite_generation.generators.test_suite_generator import TestSuiteGenerator from nimrod.test_suite_generation.generators.prompt_manager import PromptManager +from nimrod.test_suite_generation.generators.llm_output_processor import LLMOutputProcessor from nimrod.tests.utils import get_config from nimrod.utils import load_json, save_json @@ -82,12 +83,13 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]: class OllamaTestSuiteGenerator(TestSuiteGenerator): - def __init__(self, java_tool, model_key: str = "codellama", model_config: Optional[Dict[str, Any]] = None): + def __init__(self, java_tool, model_key: str = "codellama", model_config: Dict[str, Any] = {}): super().__init__(java_tool) self.model_key = model_key - self.model_config = model_config or {} + self.model_config = model_config self.api: Optional[Api] = None self.prompt_manager = PromptManager() + self.output_processor = LLMOutputProcessor() # Loads global configurations global_config = get_config() @@ -198,26 +200,15 @@ def _get_test_suite_class_names(self, test_suite_path: str) -> List[str]: def save_output(self, test_template: str, output: str, dir: str, output_file_name: str) -> None: """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template.""" - # Remove content between tags - output = re.sub(r'.*?', '', output, flags=re.DOTALL) - - # Extract only the content inside ``` blocks (excluding the ``` markers) - matches = re.findall(r'```(?:\w+)?\n?(.*?)```', output, flags=re.DOTALL) - output = '\n'.join(matches).strip() - - # Remove lines starting with "number. " (e.g., "1. public void test() {...}") - output = re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE) - - # Look for @Before, @BeforeClass first; fallback to @Test if none are found - markers = ["@Before", "@BeforeClass", "@Test"] - index = min((output.find(marker) for marker in markers if marker in output), default=-1) - - # Keep only the content starting from the first found annotation - output = output[index:] if index != -1 else output - + # Process and sanitize the LLM output using the configured processor + processed_output = self.output_processor.process(output) + + # Replace the placeholder in the template with the processed content + filled_template = test_template.replace("#TEST_METHODS#", processed_output) + + # Save to file llm_outputs_dir = os.path.join(dir, "llm_outputs") output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt") - filled_template = test_template.replace("#TEST_METHODS#", output) os.makedirs(llm_outputs_dir, exist_ok=True) with open(output_file_path, "w") as file: From d7b02da879e0dca87770a7752ba685153653f513 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 11:28:34 -0300 Subject: [PATCH 65/69] fix: update requirements.txt for specific tree_sitter versions and improve LLMOutputProcessor constructor type hint --- .github/workflows/main.yml | 2 +- .../test_suite_generation/generators/llm_output_processor.py | 2 +- requirements.txt | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f1db392..40dcf96b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,8 +19,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest mypy if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install flake8 pytest mypy - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/nimrod/test_suite_generation/generators/llm_output_processor.py b/nimrod/test_suite_generation/generators/llm_output_processor.py index d49caafa..9209ec2e 100644 --- a/nimrod/test_suite_generation/generators/llm_output_processor.py +++ b/nimrod/test_suite_generation/generators/llm_output_processor.py @@ -56,7 +56,7 @@ class LLMOutputProcessor: by applying a series of sanitization rules in sequence. """ - def __init__(self): + def __init__(self) -> None: self._rules: List[OutputSanitizationRule] = [] self._load_default_rules() diff --git a/requirements.txt b/requirements.txt index 1b4a09df..14a71772 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ beautifulsoup4 setuptools -tree_sitter -tree_sitter_java +tree_sitter==0.25.1 +tree_sitter_java==0.23.5 +requests types-requests \ No newline at end of file From bba8cf620ca7efe42d9eb7e10940ef6478407e01 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 11:33:59 -0300 Subject: [PATCH 66/69] chore: update GitHub Actions workflow to use Python 3.10 and Java Temurin --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40dcf96b..18524dd6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,11 +11,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 + - uses: actions/checkout@v6 + - name: Set up Python 3.10 + uses: actions/setup-python@v6 with: - python-version: 3.9 + python-version: '3.10' - name: Install dependencies run: | python -m pip install --upgrade pip @@ -30,9 +30,9 @@ jobs: - name: Lint with mypy run: mypy nimrod/test_suite_generation/ nimrod/test_suites_execution/ nimrod/dynamic_analysis/ nimrod/core nimrod/output_generation nimrod/__main__.py nimrod/smat.py --ignore-missing-imports - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v5 with: - distribution: 'adopt' + distribution: 'temurin' java-version: '8' - name: Setup Maven uses: stCarolas/setup-maven@v4.1 From ecbad7c3d5e21829ced71e187562ef64540efdb6 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 11:43:11 -0300 Subject: [PATCH 67/69] fix: update Maven setup action version and adjust env-config.json creation for JAVA_HOME --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 18524dd6..90907c7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,13 +35,12 @@ jobs: distribution: 'temurin' java-version: '8' - name: Setup Maven - uses: stCarolas/setup-maven@v4.1 + uses: stCarolas/setup-maven@v5 - name: Creating env-config.json run: | repo_name=$(basename $GITHUB_REPOSITORY) cd /home/runner/work/$repo_name/$repo_name/nimrod/tests/ - java_path="/opt/hostedtoolcache/Java_Adopt_jdk/$(ls /opt/hostedtoolcache/Java_Adopt_jdk)/x64" - contents="$(jq --arg java_path "$java_path" '.java_home=$java_path | .maven_home = "/opt/hostedtoolcache/maven/3.5.4/x64"' env-config.json)" + contents="$(jq --arg java_path "$JAVA_HOME" '.java_home=$java_path | .maven_home = "/opt/hostedtoolcache/maven/3.5.4/x64"' env-config.json)" echo "${contents}" > env-config.json cd /home/runner/work/$repo_name/$repo_name - name: Test with pytest From 65bf1932711f69c43127b35c690d36e33e9fb6c9 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 11:48:07 -0300 Subject: [PATCH 68/69] fix: update env-config.json creation to use dynamic MAVEN_HOME path --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 90907c7d..7c3757b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,11 +36,13 @@ jobs: java-version: '8' - name: Setup Maven uses: stCarolas/setup-maven@v5 + with: + maven-version: 3.8.2 - name: Creating env-config.json run: | repo_name=$(basename $GITHUB_REPOSITORY) cd /home/runner/work/$repo_name/$repo_name/nimrod/tests/ - contents="$(jq --arg java_path "$JAVA_HOME" '.java_home=$java_path | .maven_home = "/opt/hostedtoolcache/maven/3.5.4/x64"' env-config.json)" + contents="$(jq --arg java_path "$JAVA_HOME" --arg maven_path "${MAVEN_HOME:-/opt/hostedtoolcache/maven/3.8.2/x64}" '.java_home=$java_path | .maven_home=$maven_path' env-config.json)" echo "${contents}" > env-config.json cd /home/runner/work/$repo_name/$repo_name - name: Test with pytest From 74a3cc0ed5b79b423ae7d1817ce028ebf14c2a11 Mon Sep 17 00:00:00 2001 From: Nathalia Barbosa Date: Thu, 18 Dec 2025 15:20:58 -0300 Subject: [PATCH 69/69] feat: add Docker support with Dockerfile and docker-compose.yml for consistent environment setup --- .github/workflows/main.yml | 2 +- docker-compose.yml | 17 ++++++ docker/Dockerfile | 44 +++++++++++++++ docker/README.md | 109 +++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile create mode 100644 docker/README.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c3757b5..8f8629a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,4 +47,4 @@ jobs: cd /home/runner/work/$repo_name/$repo_name - name: Test with pytest run: | - pytest -k 'not test_general_behavior_study_semantic_conflict' + pytest -k 'not test_general_behavior_study_semantic_conflict' --color=yes diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9ed02675 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + smat: + build: + context: . + dockerfile: docker/Dockerfile + args: + USER_ID: ${USER_ID} + GROUP_ID: ${GROUP_ID} + image: smat-ubuntu + container_name: smat_container + volumes: + # Mount the current directory to /app in the container + - .:/app + # Mount the dataset directory to /data/dataset in the container (read-only) + - /path/to/your/mergedataset/:/data/dataset:ro + stdin_open: true + tty: true \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..bcecacd9 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,44 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +ARG USER_ID +ARG GROUP_ID + +# 1. Install system dependencies, Python, and Java in a single layer +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + python3.11 \ + python3-pip \ + python3.11-dev \ + openjdk-8-jdk \ + maven \ + git \ + curl \ + jq \ + build-essential \ + ca-certificates && \ + + groupadd -g ${GROUP_ID} appuser && \ + useradd -m -u ${USER_ID} -g ${GROUP_ID} appuser && \ + + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + + apt-get clean && rm -rf /var/lib/apt/lists/* + +# 2. Set Environment Variables for Java and Maven +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +ENV MAVEN_HOME=/usr/share/maven +ENV PATH="$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH" + +# 3. Setup working directory and data mount point +WORKDIR /app +RUN mkdir -p /data/dataset && chown appuser:appuser /data/dataset + +# 4. Install Python dependencies +COPY --chown=appuser:appuser requirements.txt . +RUN python3 -m pip install --no-cache-dir --upgrade pip && \ + python3 -m pip install --no-cache-dir -r requirements.txt ruff mypy pytest + +USER appuser +CMD ["/bin/bash", "-i"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..e530943a --- /dev/null +++ b/docker/README.md @@ -0,0 +1,109 @@ +# Running SMAT with Docker Compose + +This guide explains how to run SMAT in a containerized environment using **Docker Compose**. This ensures a consistent environment with Python 3.11, Java 8, and Maven, regardless of your host operating system. + +## 1. Prerequisites + +Install Docker on your system: + +* **Docker Desktop** (Recommended for Windows and Mac): [Download here](https://docs.docker.com/desktop/) +* **Docker Engine** (For Linux): [Installation Guide](https://docs.docker.com/engine/install/) + +--- + +## 2. Configuration + +Before running the container, you need to configure three files to ensure the paths match the Docker environment. + +### A. Docker Compose (Dataset Path) + +Open `docker-compose.yml` and point the dataset volume to your local path: + +```yaml +volumes: + - .:/app + # Replace the path below with the path to your dataset on your host machine + - /path/to/your/mergedataset/:/data/dataset:ro + +``` + +*Note: The `:ro` flag ensures your dataset is read-only for safety.* + +### B. SMAT Input Config (`input-smat.json`) + +The `input-smat.json` should be in the **root directory** of the project, so that the container can see it. Internally, the scenario jars must point to the `/data/dataset/` path, for example: + +```json +{ + ... + "scenarioJars": { + "base": "/data/dataset/antlr4/69ff2669eec265e25721dbc27cb00f6c381d0b41/...", + ... + }, + ... +} +``` + +### C. Environment Config (`nimrod/tests/env-config.json`) + +Point the `input_smat` path to the location inside the container. If it is on the root folder: + +```json +"input_path": "/app/input-smat.json", +``` + +--- + +## 3. Running the Container + +Navigate to the project root and run the following command according to your OS: + +### Linux & macOS (Terminal) + +The following command passes your user and group IDs to avoid permission issues with generated files: + +```bash +USER_ID=$(id -u) GROUP_ID=$(id -g) docker compose run --rm --build smat +``` + +### Windows (PowerShell) + +In PowerShell, the variables are handled differently: + +```powershell +$env:USER_ID=1000; $env:GROUP_ID=1000; docker compose run --rm --build smat +``` + +*Note: On Windows, the default UID/GID 1000 is usually sufficient for Docker Desktop.* + +--- + +## 4. Usage Inside the Container + +Once the command finishes, you will be inside the Ubuntu shell at `/app`. You can run tests or start an analysis: + +```bash +# Check if the dataset is visible +ls /data/dataset + +# Run SMAT analysis +python3 -m nimrod + +# Run tests +pytest -k 'not test_general_behavior_study_semantic_conflict' +``` + +### Command Breakdown: + +* `run`: Starts a one-off container for interactive use. +* `--rm`: Automatically removes the container upon exit to keep your system clean. +* `--build`: Forces a rebuild of the image if you modified the Dockerfile or requirements. +* `smat`: The service name defined in `docker-compose.yml`. + +--- + +## Troubleshooting + +* **Dataset Not Found**: Ensure the path on the left side of the colon in `docker-compose.yml` is an absolute path to your local folder. +* **Permission Denied**: On Linux, double-check that `USER_ID` and `GROUP_ID` match the output of the `id` command on your host terminal. +* **File Changes**: Since we use volumes, any code change made on your host machine will be instantly reflected inside the container.