Add TalkBack Utils #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

noidsirius wants to merge 7 commits into main from talkback_lib

.travis.yml

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -20,19 +20,19 @@ jobs:
  
        - language: android

          dist: trusty

          jdk:

            - openjdk8

            - openjdk11

          android:

            components:

              - platform-tools

              - build-tools-29.0.3

              - android-30

              - android-31

              - extra

          before_install:

            - yes | sdkmanager "platform-tools"

            - yes | sdkmanager "build-tools;29.0.3"

            - yes | sdkmanager "platforms;android-30"

    #      before_install:

     #       - yes | sdkmanager "platform-tools"

    #        - yes | sdkmanager "build-tools;29.0.3"

            - yes | sdkmanager "platforms;android-31"

          script:

            - ./gradlew build check jacocoTestReport

            - ./gradlew :app:assembleDebug :UseCaseExecutor:check jacocoTestReport

          env:

              - CODECOV_TOKEN=d9615b91-c123-4003-ac18-2df188e88470

          after_success:

UseCaseExecutor/build.gradle

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -15,11 +15,12 @@ jacoco {
  
    tasks.withType(Test) {

        jacoco.includeNoLocationClasses = true

        jacoco.excludes = ['jdk.internal.*']

    }

    android {

        compileSdkVersion 30

        compileSdkVersion 31

        buildToolsVersion "29.0.3"

        defaultConfig {

    @@ -52,12 +53,13 @@ android {
  
    }

    dependencies {

        implementation fileTree(dir: "libs", include: ["*.jar"])

        implementation 'com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework:3.1'

        implementation 'androidx.appcompat:appcompat:1.2.0'

    //    implementation fileTree(dir: "libs", include: ["*.aar"])

        implementation project(path: ':utils')

        implementation 'com.google.android.apps.common.testing.accessibility.framework:accessibility-test-framework:4.0.0'

        implementation 'androidx.appcompat:appcompat:1.3.1'

        testImplementation 'junit:junit:4.12'

        testImplementation "org.mockito:mockito-core:4.6.1"

        implementation 'com.google.guava:guava:24.1-jre'

        implementation 'com.google.guava:guava:31.0-jre'

        implementation (group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1') {

            exclude group: 'junit', module: 'junit'

        }

UseCaseExecutor/src/main/java/dev/navids/latte/MessageReceiver.java

-Original file line number
+Diff line change
@@ Expand Up / @@ -7,6 +7,14 @@ @@
     import android.util.Xml;
     import android.view.accessibility.AccessibilityNodeInfo;
+    import androidx.core.view.accessibility.AccessibilityNodeInfoCompat;
+    import com.google.android.accessibility.utils.AccessibilityNodeInfoUtils;
+    import com.google.android.accessibility.utils.Filter;
+    import com.google.android.accessibility.utils.traversal.OrderedTraversalController;
+    import com.google.android.accessibility.utils.traversal.OrderedTraversalStrategy;
+    import com.google.android.accessibility.utils.traversal.TraversalStrategy;
+    import com.google.android.accessibility.utils.traversal.TraversalStrategyUtils;
     import com.google.android.apps.common.testing.accessibility.framework.AccessibilityCheckPreset;
     import com.google.android.apps.common.testing.accessibility.framework.AccessibilityCheckResult;
     import com.google.android.apps.common.testing.accessibility.framework.AccessibilityCheckResultUtils;
@@ Expand Down Expand Up / @@ -57,6 +65,27 @@ public MessageReceiver() { @@
             // ----------- General ----------------
             messageEventMap.put("is_live", (extra) -> Utils.createFile(String.format(Config.v().IS_LIVE_FILE_PATH_PATTERN, extra), "I'm alive " + extra));
             messageEventMap.put("log", (extra) -> Utils.getAllA11yNodeInfo(true));
+            messageEventMap.put("dummy", (extra) -> {
+               Log.i(LatteService.TAG, "I'm in dummy message!");
+               // Logs the ordered list of focusable nodes in TalkBack
+                AccessibilityNodeInfo nodeInfo = LatteService.getInstance().getRootInActiveWindow();
+                if (nodeInfo != null) {
+                    AccessibilityNodeInfoCompat nodeInfoCompat = AccessibilityNodeInfoCompat.wrap(nodeInfo);
+                    OrderedTraversalStrategy orderedTraversalStrategy = new OrderedTraversalStrategy(nodeInfoCompat);
+                    Filter<AccessibilityNodeInfoCompat> focusNodeFilter =
+                            AccessibilityNodeInfoUtils.FILTER_SHOULD_FOCUS;
+                    AccessibilityNodeInfoCompat firstNode = TraversalStrategyUtils.findInitialFocusInNodeTree(orderedTraversalStrategy,nodeInfoCompat, TraversalStrategy.SEARCH_FOCUS_FORWARD, focusNodeFilter);
+                    Log.i(LatteService.TAG, "First Node: " + firstNode);
+                    AccessibilityNodeInfoCompat iterNode = TraversalStrategyUtils.searchFocus(orderedTraversalStrategy, firstNode, TraversalStrategy.SEARCH_FOCUS_FORWARD, focusNodeFilter);
+                    Log.i(LatteService.TAG, "Iteration:");
+                    while(iterNode != null){
+                        Log.i(LatteService.TAG, "\t" + iterNode);
+                        iterNode = TraversalStrategyUtils.searchFocus(orderedTraversalStrategy, iterNode, TraversalStrategy.SEARCH_FOCUS_FORWARD, focusNodeFilter);
+                    }
+                    Log.i(LatteService.TAG, "After Iteration");
+                }
+            });
             messageEventMap.put("invisible_nodes", (extra) -> LatteService.considerInvisibleNodes = (extra.equals("true")));
             messageEventMap.put("report_a11y_issues", (extra) -> {
                 Context context2 = LatteService.getInstance().getApplicationContext();
@@ Expand Down @@

UseCaseExecutor/src/main/java/dev/navids/latte/Utils.java

-Original file line number
+Diff line change
@@ Expand Up @@
         private static void dumpNodeRec(AccessibilityNodeInfo node, XmlSerializer serializer, int index,
                                         int width, int height) throws IOException {
+            if (node == null)
+                return;
             boolean supportsWebAction =
                     node.getActionList().contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_NEXT_HTML_ELEMENT) ||
                             node.getActionList().contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_PREVIOUS_HTML_ELEMENT);
@@ Expand Down @@

app/build.gradle

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -1,7 +1,7 @@
  
    apply plugin: 'com.android.application'

    android {

        compileSdkVersion 30

        compileSdkVersion 31

        buildToolsVersion "29.0.3"

        defaultConfig {

    @@ -28,8 +28,10 @@ android {
  
    dependencies {

        implementation fileTree(dir: "libs", include: ["*.jar"])

        implementation 'androidx.appcompat:appcompat:1.2.0'

        implementation 'androidx.appcompat:appcompat:1.3.1'

        implementation project(path: ':UseCaseExecutor')

        implementation 'com.google.android.material:material:1.6.1'

        implementation 'androidx.constraintlayout:constraintlayout:2.1.4'

        testImplementation 'junit:junit:4.12'

        androidTestImplementation 'androidx.test.ext:junit:1.1.2'

        androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0'

app/src/main/AndroidManifest.xml

-Original file line number
+Diff line change
@@ Expand Up / @@ -9,15 +9,28 @@ @@
             android:roundIcon="@mipmap/ic_launcher_round"
             android:supportsRtl="true"
             android:theme="@style/AppTheme">
+            <activity
+                android:name=".MainActivity"
+                android:exported="true"
+                >
+                <intent-filter>
+                    <action android:name="android.intent.action.MAIN" />
+                    <category android:name="android.intent.category.LAUNCHER" />
+                </intent-filter>
+            </activity>
             <service
                 android:name=".MyLatteService"
-                android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
-                android:label="@string/accessibility_service_label"
                 android:enabled="true"
-                android:exported="true">
+                android:exported="true"
+                android:label="@string/accessibility_service_label"
+                android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
                 <intent-filter>
                     <action android:name="android.accessibilityservice.AccessibilityService" />
                 </intent-filter>
                 <meta-data
                     android:name="android.accessibilityservice"
                     android:resource="@xml/accessibility_service_config" />
@@ Expand Down @@

app/src/main/java/dev/navids/latte/app/MainActivity.java

-Original file line number
+Diff line change
@@ -0,0 +1,14 @@
+    package dev.navids.latte.app;
+    import androidx.appcompat.app.AppCompatActivity;
+    import android.os.Bundle;
+    public class MainActivity extends AppCompatActivity {
+        @Override
+        protected void onCreate(Bundle savedInstanceState) {
+            super.onCreate(savedInstanceState);
+            setContentView(R.layout.activity_main);
+        }
+    }

app/src/main/res/layout/activity_main.xml

-Original file line number
+Diff line change
@@ -0,0 +1,9 @@
+    <?xml version="1.0" encoding="utf-8"?>
+    <androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+        xmlns:app="http://schemas.android.com/apk/res-auto"
+        xmlns:tools="http://schemas.android.com/tools"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        tools:context=".MainActivity">
+    </androidx.constraintlayout.widget.ConstraintLayout>

gradle/wrapper/gradle-wrapper.properties

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME @@
     distributionPath=wrapper/dists
     zipStoreBase=GRADLE_USER_HOME
     zipStorePath=wrapper/dists
-    distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-all.zip
+    distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-all.zip

py_src/A11yPuppetry/replayer.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -34,6 +34,9 @@ @@
         'com.dictionary': '/Users/navid/StudioProjects/Latte/BM_APKs/ase_apks/com.dictionary.apk',
         'com.yelp.android': '/Users/navid/StudioProjects/Latte/BM_APKs/ase_apks/com.yelp.android.apk',
         'com.offerup': '/Users/navid/StudioProjects/Latte/BM_APKs/ase_apks/com.offerup.apk',
+        'com.espn.score_center': '/Users/navid/StudioProjects/Latte/BM_APKs/ase_apks/com.espn.score_center.apk',
+        'com.expedia.bookings': '/Users/navid/StudioProjects/Latte/BM_APKs/ase_apks/com.expedia.bookings.apk',
+        'com.dd.doordash': '/Users/navid/StudioProjects/Latte/Setup/com.dd.doordash.apk',
         'com.different.toonme': '/Users/navid/StudioProjects/Latte/BM_APKs/topplay_apks/com.different.toonme.apk', # Didn't work
         'com.zhiliaoapp.musically': '/Users/navid/StudioProjects/Latte/BM_APKs/topplay_apks/com.zhiliaoapp.musically.apk',
         'com.squareup.cash': '/Users/navid/StudioProjects/Latte/BM_APKs/topplay_apks/com.squareup.cash.apk',
@@ Expand Down @@

py_src/genymotion_utils.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -9,7 +9,7 @@
  
    logger = logging.getLogger(__name__)

    BASE_RECIPE_NAME = 'AP-Base-2'

    BASE_RECIPE_NAME = 'AP-Base-3'

    async def send_gmsaas_command(command: str) -> Optional[dict]:

    @@ -159,7 +159,7 @@ async def setup_ap_instance(instance_name: str, app_paths : List[str] = None) ->
  
        if app_paths is None:

            app_paths = []

        instance = await create_instance(instance_name=instance_name)

        if not instance.is_online():

        if instance is None or not instance.is_online():

            logger.error(f"Instance {instance_name} could not be created")

            return False

        if not await instance.connect_adb():

py_src/task/create_a11y_puppetry_video.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -24,7 +24,7 @@ async def execute(self): @@
             create_gif(source_images=images,
                        target_gif=record_manager.recorder_path.joinpath("video.gif"),
                        image_to_nodes=image_to_nodes,
-                       outline=(220, 20, 60),
+                       outline= (220, 20, 60),
                        duration=500)

settings.gradle

-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
     include ':UseCaseExecutor'
     include ':app'
-    rootProject.name = "Latte"
+    rootProject.name = "Latte"
+    include ':utils'

utils/.gitignore

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		/build

utils/b_build.gradle

-Original file line number
+Diff line change
@@ -0,0 +1,35 @@
+    plugins {
+        id 'com.android.library'
+    }
+    android {
+        compileSdkVersion 31
+        defaultConfig {
+            minSdkVersion 26
+            targetSdkVersion 31
+            testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+            consumerProguardFiles "consumer-rules.pro"
+        }
+        buildTypes {
+            release {
+                minifyEnabled false
+                proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+            }
+        }
+        compileOptions {
+            sourceCompatibility JavaVersion.VERSION_1_8
+            targetCompatibility JavaVersion.VERSION_1_8
+        }
+    }
+    dependencies {
+        implementation 'androidx.appcompat:appcompat:1.5.0'
+        implementation 'com.google.android.material:material:1.6.1'
+        testImplementation 'junit:junit:4.13.2'
+        androidTestImplementation 'androidx.test.ext:junit:1.1.3'
+        androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
+    }

utils/build.gradle

-Original file line number
+Diff line change
@@ -0,0 +1,53 @@
+    apply plugin: 'com.android.library'
+    ext {
+        talkbackApplicationId = "com.android.talkback"
+        talkbackMainPermission = "com.android.talkback.permission.TALKBACK"
+    }
+    android {
+        compileSdkVersion 'android-31'
+        compileOptions {
+            sourceCompatibility JavaVersion.VERSION_1_8
+            targetCompatibility JavaVersion.VERSION_1_8
+        }
+        defaultConfig {
+            vectorDrawables.useSupportLibrary = true
+            minSdkVersion 26
+        }
+    }
+    dependencies {
+        // Google common
+        implementation 'com.google.guava:guava:31.0-jre'
+        implementation 'com.google.android.gms:play-services-mlkit-text-recognition:17.0.1'
+        implementation 'com.google.android.material:material:1.4.0'
+        // Support library
+        implementation 'androidx.annotation:annotation:1.2.0'
+        implementation 'androidx.appcompat:appcompat:1.3.1'
+        implementation 'androidx.collection:collection:1.1.0'
+        implementation 'androidx.core:core:1.6.0-alpha03'
+        implementation 'androidx.fragment:fragment:1.4.0-alpha08'
+        implementation 'androidx.localbroadcastmanager:localbroadcastmanager:1.0.0'
+        implementation 'androidx.preference:preference:1.1.1'
+        implementation 'androidx.recyclerview:recyclerview:1.2.1'
+        implementation 'androidx.viewpager2:viewpager2:1.0.0'
+        // Nullable
+    //    implementation 'org.checkerframework:checker-qual:2.8.1'
+        // Auto-value
+        implementation 'com.google.auto.value:auto-value-annotations:1.8.2'
+        annotationProcessor 'com.google.auto.value:auto-value:1.8.2'
+        implementation 'javax.annotation:javax.annotation-api:1.3.2'
+    //    implementation 'androidx.wear:wear:1.2.0-rc01'
+    //    implementation 'com.google.android.support:wearable:2.8.1'
+    }
+    android {
+        defaultConfig {
+            buildConfigField("String", "TALKBACK_APPLICATION_ID", '"' + talkbackApplicationId + '"')
+        }
+    }

utils/consumer-rules.pro

Empty file.

utils/proguard-rules.pro

-Original file line number
+Diff line change
@@ -0,0 +1,21 @@
+    # Add project specific ProGuard rules here.
+    # You can control the set of applied configuration files using the
+    # proguardFiles setting in build.gradle.
+    #
+    # For more details, see
+    #   http://developer.android.com/guide/developing/tools/proguard.html
+    # If your project uses WebView with JS, uncomment the following
+    # and specify the fully qualified class name to the JavaScript interface
+    # class:
+    #-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+    #   public *;
+    #}
+    # Uncomment this to preserve the line number information for
+    # debugging stack traces.
+    #-keepattributes SourceFile,LineNumberTable
+    # If you keep the line number information, uncomment this to
+    # hide the original source file name.
+    #-renamesourcefileattribute SourceFile

utils/src/main/AndroidManifest.xml

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		<?xml version="1.0" encoding="utf-8"?>
		<manifest package="com.google.android.accessibility.utils" />

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add TalkBack Utils #34

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing

Uh oh!

Add TalkBack Utils #34

Are you sure you want to change the base?

Uh oh!

Add TalkBack Utils #34

Uh oh!

Diff view

Diff view

Uh oh!

There are no files selected for viewing

Uh oh!