From 6ce8c8a1f8616ced4151a55dac1ddae18af1e93e Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 10:40:00 +0800 Subject: [PATCH 01/11] feat: Reset app --- android_world/task_evals/single/browser.py | 8 ++++---- android_world/task_evals/task_eval.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/android_world/task_evals/single/browser.py b/android_world/task_evals/single/browser.py index 6418291ce..0360cc851 100644 --- a/android_world/task_evals/single/browser.py +++ b/android_world/task_evals/single/browser.py @@ -60,10 +60,10 @@ def initialize_task(self, env: interface.AsyncEnv): adb_utils.get_adb_activity('chrome') ) - adb_utils.clear_app_data( - chrome_activity, - env.controller, - ) + # adb_utils.clear_app_data( + # chrome_activity, + # env.controller, + # ) adb_utils.grant_permissions( chrome_activity, 'android.permission.POST_NOTIFICATIONS', diff --git a/android_world/task_evals/task_eval.py b/android_world/task_evals/task_eval.py index 070ce949d..5776965ac 100644 --- a/android_world/task_evals/task_eval.py +++ b/android_world/task_evals/task_eval.py @@ -23,7 +23,15 @@ from android_world.env import interface from android_world.utils import app_snapshot from android_world.utils import datetime_utils +from android_world.env.setup_device import setup, apps +RESET_APPS = { + "audio recorder": apps.AudioRecorder, + "camera": apps.CameraApp, + "chrome": apps.ChromeApp, + "markor": apps.MarkorApp, + "simple calendar pro": apps.SimpleCalendarProApp, +} class TaskEval(abc.ABC): """Interface for a task and its evaluation. @@ -122,6 +130,11 @@ def _initialize_apps(self, env: interface.AsyncEnv) -> None: except RuntimeError as error: logging.warning("Skipping app snapshot loading : %s", error) + if app_name in RESET_APPS: + logging.info("Reset app for %s", app_name) + setup.setup_app(RESET_APPS[app_name], env) + + @classmethod def set_device_time(cls, env: interface.AsyncEnv) -> None: """Sets the device time.""" From 9efb3a54fa391b6a6ab0f6e275684c1dbe81144c Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 10:40:29 +0800 Subject: [PATCH 02/11] fix: Fix bug --- android_world/utils/file_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/android_world/utils/file_utils.py b/android_world/utils/file_utils.py index 6bf60bc4f..bc1a5ef00 100644 --- a/android_world/utils/file_utils.py +++ b/android_world/utils/file_utils.py @@ -361,7 +361,11 @@ def tmp_directory_from_device( yield tmp_directory finally: - shutil.rmtree(tmp_directory) + try: + shutil.rmtree(tmp_directory) + except Exception as e: + logging.error("Failed to delete temporary directory: %s with error %s", + tmp_directory, e) @contextlib.contextmanager From dec759603c99afe23ab32ed578537064d7f39dad Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 10:40:55 +0800 Subject: [PATCH 03/11] feat: Reset app --- android_world/env/setup_device/apps.py | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/android_world/env/setup_device/apps.py b/android_world/env/setup_device/apps.py index e28395497..d9eeef0b3 100644 --- a/android_world/env/setup_device/apps.py +++ b/android_world/env/setup_device/apps.py @@ -30,6 +30,8 @@ from android_world.task_evals.information_retrieval import joplin_app_utils from android_world.utils import file_utils import requests +from android_world.env import actuation +from android_world.env import json_action APP_DATA = file_utils.convert_to_posix_path(os.path.dirname(__file__), @@ -156,8 +158,15 @@ def setup(cls, env: interface.AsyncEnv) -> None: try: controller = tools.AndroidToolController(env=env.controller) time.sleep(2.0) - # Welcome screen. - controller.click_element("Accept & continue") + try: + # Welcome screen. + controller.click_element("Accept & continue") + except ValueError as e: + # Use without an account + logging.warn("Can't find `Accept & continue`, try to clock `Use without an account`.") + time.sleep(10) + action = json_action.JSONAction(action_type='click', x=540, y=2096) + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) # Turn on sync? controller.click_element("No thanks") @@ -239,18 +248,31 @@ def setup(cls, env: interface.AsyncEnv) -> None: adb_utils.launch_app(cls.app_name, env.controller) try: controller = tools.AndroidToolController(env=env.controller) + time.sleep(10) + action = json_action.JSONAction(action_type='click', x=984, y=2243) + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") - time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("DONE") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("DONE") + # time.sleep(2.0) + controller.click_element("OK") time.sleep(2.0) controller.click_element("Allow access to manage all files") From 48bbf3dcb7ede6e67b0a6d2b7568b0fa3de9ffcb Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 15:20:48 +0800 Subject: [PATCH 04/11] some fix --- android_world/suite_utils.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/android_world/suite_utils.py b/android_world/suite_utils.py index aed9a7bd3..20dbd9da4 100644 --- a/android_world/suite_utils.py +++ b/android_world/suite_utils.py @@ -163,7 +163,10 @@ def _get_instance_seed(name: str, i: int) -> int: suite = _filter_tasks(suite, task_registry, tasks) # Sort suite alphabetically by task name. - return Suite(sorted(suite.items())) + if tasks is None: + return Suite(sorted(suite.items())) + else: + return Suite(list(suite.items())) def _suggest_keyword( @@ -207,10 +210,12 @@ def _filter_tasks( + _suggest_keyword(name, list(task_registry.keys())) ) - # Filter. - for name, instances in suite.items(): - if name in tasks: - subset[name] = instances + # # Filter. + # for name, instances in suite.items(): + # if name in tasks: + # subset[name] = instances + for name in tasks: + subset[name] = suite[name] return subset @@ -255,8 +260,7 @@ def _run_task( else: agent_successful = task_successful if interaction_results.done else 0.0 print( - f'{"Task Successful ✅" if agent_successful > 0.5 else "Task Failed ❌"};' - f' {task.goal}' + f'{"Task Successful √√√√√√√√√√√" if agent_successful > 0.5 else "Task Failed xxxxxxxxxx"}; {task.goal}' ) if demo_mode: @@ -411,6 +415,8 @@ def _run_task_suite( _update_scoreboard(correct, total, env.controller) print() + process_episodes_fn(episodes_metadata, print_summary=True) + return full_episode_data if return_full_episode_data else episodes_metadata From c81351d5bf0ac177f0f6dffcb5ba8d57cf905058 Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 15:24:20 +0800 Subject: [PATCH 05/11] feat: Change max step --- android_world/suite_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/android_world/suite_utils.py b/android_world/suite_utils.py index 20dbd9da4..39edd00ac 100644 --- a/android_world/suite_utils.py +++ b/android_world/suite_utils.py @@ -499,7 +499,8 @@ def _allocate_step_budget(task_complexity: float) -> int: """ if task_complexity is None: raise ValueError('Task complexity must be provided.') - return int(10 * (task_complexity)) + max_step_rate = int(os.getenv('ANDROID_MAX_STEP', '10')) + return int(max_step_rate * (task_complexity)) def _display_message( From bb56a1fd20cec8f96475464793f4f54ab1f20f36 Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 15:31:44 +0800 Subject: [PATCH 06/11] feat: specify adb port --- android_world/env/android_world_controller.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/android_world/env/android_world_controller.py b/android_world/env/android_world_controller.py index 96de38d5d..0a8f643e4 100644 --- a/android_world/env/android_world_controller.py +++ b/android_world/env/android_world_controller.py @@ -316,7 +316,10 @@ def get_controller( adb_port=console_port + 1, grpc_port=grpc_port, ), - adb_controller=config_classes.AdbControllerConfig(adb_path=adb_path), + adb_controller=config_classes.AdbControllerConfig( + adb_path=adb_path, + adb_server_port=int(os.environ.get('ANDROID_ADB_SERVER_PORT', '5037')) + ), ), ) android_env_instance = loader.load(config) From 95eb7f07674fe788ec336f7bf7ff2ebc8ff5cd05 Mon Sep 17 00:00:00 2001 From: Ning Li Date: Wed, 23 Apr 2025 18:48:28 +0800 Subject: [PATCH 07/11] fix --- .../task_evals/utils/user_data_generation.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/android_world/task_evals/utils/user_data_generation.py b/android_world/task_evals/utils/user_data_generation.py index 5d801a142..e0593b02a 100644 --- a/android_world/task_evals/utils/user_data_generation.py +++ b/android_world/task_evals/utils/user_data_generation.py @@ -466,10 +466,15 @@ def clear_internal_storage(env: interface.AsyncEnv) -> None: ), env.controller, ) - adb_utils.issue_generic_request( - ["shell", "rm", f"{device_constants.EMULATOR_DATA}/*"], - env.controller, - ) + try: + adb_utils.issue_generic_request( + ["shell", "rm", f"{device_constants.EMULATOR_DATA}/*"], + env.controller, + ) + except Exception as e: + logging.warning( + "Failed to clear internal storage directories: %s", str(e) + ) def _clear_external_downloads(env: interface.AsyncEnv) -> None: From 9a207d1a378bacbef0dbf3b81b79c63369e11f7e Mon Sep 17 00:00:00 2001 From: Ning Li Date: Fri, 25 Apr 2025 11:18:08 +0800 Subject: [PATCH 08/11] fix: clear internal storage --- android_world/task_evals/utils/user_data_generation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android_world/task_evals/utils/user_data_generation.py b/android_world/task_evals/utils/user_data_generation.py index e0593b02a..2871e8f83 100644 --- a/android_world/task_evals/utils/user_data_generation.py +++ b/android_world/task_evals/utils/user_data_generation.py @@ -468,12 +468,12 @@ def clear_internal_storage(env: interface.AsyncEnv) -> None: ) try: adb_utils.issue_generic_request( - ["shell", "rm", f"{device_constants.EMULATOR_DATA}/*"], + ["shell", "find", f"{device_constants.EMULATOR_DATA}", "-maxdepth 1 -type f -exec rm -f {} \;"], env.controller, ) except Exception as e: logging.warning( - "Failed to clear internal storage directories: %s", str(e) + "Failed to clear internal storage files: %s", str(e) ) From de8a4df9423cb7838c744a8e977b791c1a144588 Mon Sep 17 00:00:00 2001 From: Ning Li <2212363918@qq.com> Date: Thu, 4 Sep 2025 11:04:43 +0000 Subject: [PATCH 09/11] fix vlc loading issue --- android_world/task_evals/single/vlc.py | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/android_world/task_evals/single/vlc.py b/android_world/task_evals/single/vlc.py index 2a82bc54e..cd9aeeb07 100644 --- a/android_world/task_evals/single/vlc.py +++ b/android_world/task_evals/single/vlc.py @@ -70,6 +70,35 @@ def _get_playlist_file_info( sqlite_schema_utils.PlaylistInfo, ) +def _fix_loading_issue(env: interface.AsyncEnv): + try: + import time + from absl import logging + from android_world.env import adb_utils + from android_world.env import actuation + from android_world.env import json_action + adb_utils.launch_app("vlc", env.controller) + pages = ["Playlists", "Browse"] + for page in pages: + wait_time = 0 + should_wait = True + while should_wait and wait_time < 300: + should_wait = False + actuation.find_and_click_element(page, env.controller) + ui_elements = env.controller.get_ui_elements() + for i, element in enumerate(ui_elements): + for attr in [element.text, element.content_description]: + if attr is not None: + if "Loading" in attr: + should_wait = True + wait_time += 10 + time.sleep(10) + if wait_time >= 300: + logging.warning(f"Failed to skip loading screen in {page} page for VLC app.") + action = json_action.JSONAction(action_type='navigate_home') + actuation.execute_adb_action(action, [], (0, 0), env.controller) + except: + logging.warning("Failed to execute `fix_loading_issue` for VLC app.") class _VLC(task_eval.TaskEval): @@ -136,6 +165,7 @@ def initialize_task(self, env: interface.AsyncEnv): super().initialize_task(env) _clear_playlist_dbs(env) self.setup_files(env) + _fix_loading_issue(env) def tear_down(self, env: interface.AsyncEnv): super().tear_down(env) @@ -216,6 +246,7 @@ def initialize_task(self, env: interface.AsyncEnv): super().initialize_task(env) self.task1.initialize_task(env) self.task2.setup_files(env) # Don't want to clear db. + _fix_loading_issue(env) def tear_down(self, env: interface.AsyncEnv): super().tear_down(env) From 062f06920a27c1652e9998b6cfe6c2f73cc57f8f Mon Sep 17 00:00:00 2001 From: Ning Li <2212363918@qq.com> Date: Thu, 4 Sep 2025 13:20:01 +0000 Subject: [PATCH 10/11] fix vlc loading issue --- android_world/task_evals/single/vlc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android_world/task_evals/single/vlc.py b/android_world/task_evals/single/vlc.py index cd9aeeb07..1ae300c00 100644 --- a/android_world/task_evals/single/vlc.py +++ b/android_world/task_evals/single/vlc.py @@ -82,9 +82,9 @@ def _fix_loading_issue(env: interface.AsyncEnv): for page in pages: wait_time = 0 should_wait = True + actuation.find_and_click_element(page, env.controller) while should_wait and wait_time < 300: should_wait = False - actuation.find_and_click_element(page, env.controller) ui_elements = env.controller.get_ui_elements() for i, element in enumerate(ui_elements): for attr in [element.text, element.content_description]: From f08359b005fc8e98e5dd4478c88a33841d38292d Mon Sep 17 00:00:00 2001 From: Ning Li <2212363918@qq.com> Date: Sat, 6 Sep 2025 08:02:38 +0000 Subject: [PATCH 11/11] feat: Reset app --- android_world/task_evals/task_eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/android_world/task_evals/task_eval.py b/android_world/task_evals/task_eval.py index 5776965ac..20e30c5e4 100644 --- a/android_world/task_evals/task_eval.py +++ b/android_world/task_evals/task_eval.py @@ -31,6 +31,7 @@ "chrome": apps.ChromeApp, "markor": apps.MarkorApp, "simple calendar pro": apps.SimpleCalendarProApp, + "tasks": apps.TasksApp, } class TaskEval(abc.ABC):