diff --git a/android_world/env/android_world_controller.py b/android_world/env/android_world_controller.py index 1f4e3b570..2a8082572 100644 --- a/android_world/env/android_world_controller.py +++ b/android_world/env/android_world_controller.py @@ -321,7 +321,10 @@ def get_controller( adb_port=console_port + 1, grpc_port=grpc_port, ), - adb_controller=config_classes.AdbControllerConfig(adb_path=adb_path), + adb_controller=config_classes.AdbControllerConfig( + adb_path=adb_path, + adb_server_port=int(os.environ.get('ANDROID_ADB_SERVER_PORT', '5037')) + ), ), ) android_env_instance = loader.load(config) diff --git a/android_world/env/setup_device/apps.py b/android_world/env/setup_device/apps.py index 694f2be60..de35bb620 100644 --- a/android_world/env/setup_device/apps.py +++ b/android_world/env/setup_device/apps.py @@ -30,6 +30,8 @@ from android_world.task_evals.information_retrieval import joplin_app_utils from android_world.utils import file_utils import requests +from android_world.env import actuation +from android_world.env import json_action APP_DATA = file_utils.convert_to_posix_path(os.path.dirname(__file__), @@ -162,8 +164,15 @@ def setup(cls, env: interface.AsyncEnv) -> None: try: controller = tools.AndroidToolController(env=env.controller) time.sleep(2.0) - # Welcome screen. - controller.click_element("Accept & continue") + try: + # Welcome screen. + controller.click_element("Accept & continue") + except ValueError as e: + # Use without an account + logging.warn("Can't find `Accept & continue`, try to clock `Use without an account`.") + time.sleep(10) + action = json_action.JSONAction(action_type='click', x=540, y=2096) + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) # Turn on sync? controller.click_element("No thanks") @@ -245,18 +254,31 @@ def setup(cls, env: interface.AsyncEnv) -> None: adb_utils.launch_app(cls.app_name, env.controller) try: controller = tools.AndroidToolController(env=env.controller) + time.sleep(10) + action = json_action.JSONAction(action_type='click', x=984, y=2243) + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") - time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("NEXT") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) - controller.click_element("DONE") + actuation.execute_adb_action(action, [], (0, 0), env.controller) time.sleep(2.0) + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("NEXT") + # time.sleep(2.0) + # controller.click_element("DONE") + # time.sleep(2.0) + controller.click_element("OK") time.sleep(2.0) controller.click_element("Allow access to manage all files") diff --git a/android_world/suite_utils.py b/android_world/suite_utils.py index 3b91646b4..c0cef2328 100644 --- a/android_world/suite_utils.py +++ b/android_world/suite_utils.py @@ -169,7 +169,10 @@ def _get_instance_seed(name: str, i: int) -> int: suite = _filter_tasks(suite, task_registry, tasks) # Sort suite alphabetically by task name. - return Suite(sorted(suite.items())) + if tasks is None: + return Suite(sorted(suite.items())) + else: + return Suite(list(suite.items())) def _suggest_keyword( @@ -213,10 +216,12 @@ def _filter_tasks( + _suggest_keyword(name, list(task_registry.keys())) ) - # Filter. - for name, instances in suite.items(): - if name in tasks: - subset[name] = instances + # # Filter. + # for name, instances in suite.items(): + # if name in tasks: + # subset[name] = instances + for name in tasks: + subset[name] = suite[name] return subset @@ -261,7 +266,7 @@ def _run_task( agent_successful = task_successful if interaction_results.done else 0.0 _log_and_print( '%s; %s', - 'Task Successful ✅' if agent_successful > 0.5 else 'Task Failed ❌', + 'Task Successful √√√√√√√√√√√' if agent_successful > 0.5 else 'Task Failed xxxxxxxxxx', f' {task.goal}', ) @@ -417,6 +422,8 @@ def _run_task_suite( _update_scoreboard(correct, total, env.controller) print() + process_episodes_fn(episodes_metadata, print_summary=True) + return full_episode_data if return_full_episode_data else episodes_metadata @@ -499,7 +506,8 @@ def _allocate_step_budget(task_complexity: float) -> int: """ if task_complexity is None: raise ValueError('Task complexity must be provided.') - return int(10 * (task_complexity)) + max_step_rate = int(os.getenv('ANDROID_MAX_STEP', '10')) + return int(max_step_rate * (task_complexity)) def _display_message( diff --git a/android_world/task_evals/single/browser.py b/android_world/task_evals/single/browser.py index bf6d413fc..61fd6d667 100644 --- a/android_world/task_evals/single/browser.py +++ b/android_world/task_evals/single/browser.py @@ -60,10 +60,10 @@ def initialize_task(self, env: interface.AsyncEnv): adb_utils.get_adb_activity('chrome') ) - adb_utils.clear_app_data( - chrome_activity, - env.controller, - ) + # adb_utils.clear_app_data( + # chrome_activity, + # env.controller, + # ) adb_utils.grant_permissions( chrome_activity, 'android.permission.POST_NOTIFICATIONS', diff --git a/android_world/task_evals/single/vlc.py b/android_world/task_evals/single/vlc.py index c20ce4bf1..818af2553 100644 --- a/android_world/task_evals/single/vlc.py +++ b/android_world/task_evals/single/vlc.py @@ -70,6 +70,35 @@ def _get_playlist_file_info( sqlite_schema_utils.PlaylistInfo, ) +def _fix_loading_issue(env: interface.AsyncEnv): + try: + import time + from absl import logging + from android_world.env import adb_utils + from android_world.env import actuation + from android_world.env import json_action + adb_utils.launch_app("vlc", env.controller) + pages = ["Playlists", "Browse"] + for page in pages: + wait_time = 0 + should_wait = True + actuation.find_and_click_element(page, env.controller) + while should_wait and wait_time < 300: + should_wait = False + ui_elements = env.controller.get_ui_elements() + for i, element in enumerate(ui_elements): + for attr in [element.text, element.content_description]: + if attr is not None: + if "Loading" in attr: + should_wait = True + wait_time += 10 + time.sleep(10) + if wait_time >= 300: + logging.warning(f"Failed to skip loading screen in {page} page for VLC app.") + action = json_action.JSONAction(action_type='navigate_home') + actuation.execute_adb_action(action, [], (0, 0), env.controller) + except: + logging.warning("Failed to execute `fix_loading_issue` for VLC app.") class _VLC(task_eval.TaskEval): @@ -136,6 +165,7 @@ def initialize_task(self, env: interface.AsyncEnv): super().initialize_task(env) _clear_playlist_dbs(env) self.setup_files(env) + _fix_loading_issue(env) def tear_down(self, env: interface.AsyncEnv): super().tear_down(env) @@ -216,6 +246,7 @@ def initialize_task(self, env: interface.AsyncEnv): super().initialize_task(env) self.task1.initialize_task(env) self.task2.setup_files(env) # Don't want to clear db. + _fix_loading_issue(env) def tear_down(self, env: interface.AsyncEnv): super().tear_down(env) diff --git a/android_world/task_evals/task_eval.py b/android_world/task_evals/task_eval.py index fd80d41ec..c08b28cfc 100644 --- a/android_world/task_evals/task_eval.py +++ b/android_world/task_evals/task_eval.py @@ -25,7 +25,16 @@ from android_world.env.setup_device import setup from android_world.utils import app_snapshot from android_world.utils import datetime_utils +from android_world.env.setup_device import setup, apps +RESET_APPS = { + "audio recorder": apps.AudioRecorder, + "camera": apps.CameraApp, + "chrome": apps.ChromeApp, + "markor": apps.MarkorApp, + "simple calendar pro": apps.SimpleCalendarProApp, + "tasks": apps.TasksApp, +} class TaskEval(abc.ABC): """Interface for a task and its evaluation. @@ -124,6 +133,10 @@ def _initialize_apps(self, env: interface.AsyncEnv) -> None: except RuntimeError as error: logging.warning("Skipping app snapshot loading : %s", error) + if app_name in RESET_APPS: + logging.info("Reset app for %s", app_name) + setup.setup_app(RESET_APPS[app_name], env) + def install_apps_if_not_installed(self, env: interface.AsyncEnv) -> None: for app_name in self.app_names: setup.install_app_if_not_installed(app_name, env)