gui: add a script to process ButterKnife annotations

AppAnalysis/preprocess/IntelliDroidPreprocessAPK.sh

@@ -6,7 +6,7 @@ if [ "$(uname)" == "Darwin" ]; then
     DARE=$(dirname $0)/dare-1.1.0-macos/dare
 fi
 
-APKTOOL=$(dirname $0)/apktool-2.0.0rc4/apktool
+APKTOOL=$(dirname $0)/apktool-2.3.1/apktool
 
 preprocessAPK() {
     # Expand the APK file path (only works in Linux)

AppAnalysis/preprocess/gui-annotation-extract/.gitignore

@@ -0,0 +1 @@
+bin

AppAnalysis/preprocess/gui-annotation-extract/README.md

@@ -0,0 +1,14 @@
+Extracts GUI method runtime annotations to help with GUI method analysis.
+
+Usage: ./guiAnnotationExtract.sh <apk> <output.json>
+
+Manually copy the output json into the preprocessed APK folder as guiAnnotation.json for now: this needs Java 8 due to dexlib2
+but the rest of IntelliDroid doesn't like Java 8. Will rebuild dexlib2 with Java 7 later.
+
+Implemented using [dexlib2](https://github.com/JesusFreke/smali/tree/master/dexlib2).
+
+Currently supports [ButterKnife](https://github.com/JakeWharton/butterknife)'s OnClick attributes.
+
+This is required since Dare doesn't preserve annotations. Other converters such as Dex2jar do preserve annotations, but cannot handle malformed Dex files that Dare can easily handle.
+
+Note: only run this on non-malicious apps from trusted sources! While there shouldn't be any security issues in this code, it has not been audited for them.

AppAnalysis/preprocess/gui-annotation-extract/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+cd "$(dirname $0)"
+rm -r bin || true
+mkdir bin
+find src -name "*.java" | xargs -- javac -classpath "src:lib/*" -d bin

AppAnalysis/preprocess/gui-annotation-extract/guiAnnotationExtract.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+$(dirname $0)/build.sh
+exec java -cp "$(dirname $0)/bin:$(dirname $0)/lib/*" intellidroid.appanalysis.guiannotations.GUIAnnotationExtract $@

AppAnalysis/preprocess/gui-annotation-extract/src/intellidroid/appanalysis/guiannotations/GUIAnnotationExtract.java

@@ -0,0 +1,85 @@
+package intellidroid.appanalysis.guiannotations;
+
+import java.io.*;
+import java.util.*;
+import org.json.*;
+import org.jf.dexlib2.*;
+import org.jf.dexlib2.dexbacked.*;
+import org.jf.dexlib2.iface.*;
+import org.jf.dexlib2.iface.value.*;
+
+/**
+ * A utility to extract GUI-related annotations from an APK to a JSON file.
+ * Currently supports extracting ButterKnife.OnClick annotations from methods.
+ */
+public class GUIAnnotationExtract {
+
+	public static String getBytecodeSignature(DexBackedMethod dexMethod) {
+		StringBuilder builder = new StringBuilder();
+		builder.append(dexMethod.getName()).append("(");
+		for (String s: dexMethod.getParameterTypes()) {
+			builder.append(s);
+		}
+		builder.append(")").append(dexMethod.getReturnType());
+		return builder.toString();
+	}
+
+	public static void processDexFile(DexBackedDexFile dexFile, JSONArray outArray) throws Exception {
+		for (DexBackedClassDef dexClass : dexFile.getClasses()) {
+			for (DexBackedMethod dexMethod: dexClass.getMethods()) {
+				for (Annotation annotation: dexMethod.getAnnotations()) {
+					String annotationType = annotation.getType();
+					// we only handle onClick for now.
+					boolean found = annotationType.equals("Lbutterknife/OnClick;");
+					if (found) {
+						List<? extends EncodedValue> viewIds = null;
+						for (AnnotationElement element: annotation.getElements()) {
+							if (element.getName().equals("value")) {
+								viewIds = ((ArrayEncodedValue)element.getValue()).getValue();
+								break;
+							}
+						}
+						JSONArray viewIdsArray = new JSONArray();
+						if (viewIds == null) {
+							System.out.println("ViewHolder-based binding not handled: " +
+								dexClass.getType() + "->" + dexMethod.getName());
+						} else {
+							for (EncodedValue value: viewIds) {
+								viewIdsArray.put(((IntEncodedValue)value).getValue());
+							}
+						}
+						JSONObject outObj = new JSONObject();
+						String className = dexClass.getType();
+						outObj.put("className", className.substring(0, className.length() - 1));
+						outObj.put("method", getBytecodeSignature(dexMethod));
+						outObj.put("annotation", annotationType);
+						outObj.put("viewIds", viewIdsArray);
+						outArray.put(outObj);
+					}
+				}
+			}
+		}
+	}
+
+	public static void doExtract(File[] inputFile, File outputFile) throws Exception {
+		JSONArray outArray = new JSONArray();
+		for (File inFile : inputFile) {
+			MultiDexContainer<? extends DexBackedDexFile> dexFiles = DexFileFactory.loadDexContainer(inFile, Opcodes.getDefault());
+			for (String dexName: dexFiles.getDexEntryNames()) {
+				DexBackedDexFile dexFile = dexFiles.getEntry(dexName);
+				processDexFile(dexFile, outArray);
+			}
+		}
+		PrintStream outStream = new PrintStream(outputFile);
+		outStream.println(outArray.toString(4));
+	}
+
+	public static void main(String[] args) throws Exception {
+		File[] inFiles = new File[args.length - 1];
+		for (int i = 0; i < args.length - 1; i++) {
+			inFiles[i] = new File(args[i]);
+		}
+		File outFile = new File(args[args.length - 1]);
+		doExtract(inFiles, outFile);
+	}
+}

AppAnalysis/src/intellidroid/appanalysis/EntrypointAnalysis.java

@@ -316,7 +316,8 @@ private Map<IMethod, MethodReference> getActivityEntrypoints() {
             for (IMethod activityMethod : activityClass.getDeclaredMethods()) {
                 if (_activityLifecycleMethods.contains(activityMethod.getSelector())) {
                     activityEntrypoints.put(activityMethod, MethodReference.findOrCreate(_activityClass, activityMethod.getSelector()));
-                } else if (uiDefinedHandlers.contains(activityMethod.getSelector().getName().toString())) {
+                } else if (uiDefinedHandlers.contains(activityMethod.getSelector().getName().toString()) ||
+                    annotationHandlers.contains(activityMethod.getSelector().toString())) {
                     activityEntrypoints.put(activityMethod, AndroidMethods.getOnClickMethod());
                 }
             }

AppAnalysis/src/intellidroid/appanalysis/UIActivityMapping.java

@@ -19,8 +19,10 @@
 import org.w3c.dom.Element;
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.FileReader;
 import org.apache.commons.io.FilenameUtils;
 import org.xml.sax.SAXParseException;
+import com.google.gson.Gson;
 
 class UIActivityMapping {
     private final boolean DEBUG = false;
@@ -31,6 +33,8 @@ class UIActivityMapping {
 
     private Map<String, Set<String>> _handlerLayoutMap = new HashMap<String, Set<String>>();
     private Map<IMethod, Set<TypeReference>> _handlerActivityMap = new HashMap<IMethod, Set<TypeReference>>();
+    private UIAnnotationJsonEntry[] _annotationEntries;
+    private Map<String, Set<String>> _annotationListeners = new HashMap<String, Set<String>>();
 
     public UIActivityMapping(IClassHierarchy cha) {
         if (DEBUG) {
@@ -43,6 +47,10 @@ public UIActivityMapping(IClassHierarchy cha) {
             getLayoutIDs();
             getLayoutHandlers();
             //mapHandlerToElements();
+            File annotationsFile = new File(IntelliDroidAppAnalysis.Config.AppDirectory + "/guiAnnotations.json");
+            if (annotationsFile.exists()) {
+                loadUIAnnotations(annotationsFile);
+            }
         } catch (Exception e) {
             System.err.println("Exception: " + e.toString());
             e.printStackTrace();
@@ -589,5 +597,49 @@ private boolean isUICallbackClass(TypeReference callbackClass) {
 
         return false;
     }
+
+    private static final class UIAnnotationJsonEntry {
+        String className;
+        String method;
+        String annotation;
+        int[] viewIds;
+    }
+
+    private void loadUIAnnotations(File annotationFile) throws Exception {
+        // load extracted annotations
+        // we can't process annotations internally since Dare doesn't copy them, like Dex2jar or other dex converters.
+        // so we have a gui-annotations-extract script to pull them directly from the .apk.
+        // For now, this is used for ButterKnife.
+
+        // Note: ButterKnife annotations are fragile: obfuscators remove/mangle them (so e.g. Microsoft Outlook loses the IDs)
+        // So long-term, we should process the ViewBinder/ViewBinding classes instead.
+        // this is good enough for non-ProGuarded apps like Kickstarter.
+        FileReader reader = null;
+        UIAnnotationJsonEntry[] entries = null;
+        try {
+            reader = new FileReader(annotationFile);
+            entries = new Gson().fromJson(reader, UIAnnotationJsonEntry[].class);
+        } finally {
+            if (reader != null) reader.close();
+        }
+        for (UIAnnotationJsonEntry entry: entries) {
+            if (entry.viewIds.length == 0) {
+                continue; // no IDs. See above.
+            }
+            String className = entry.className;
+            Set<String> myset = _annotationListeners.get(className);
+            if (myset == null) {
+                myset = new HashSet<String>();
+                _annotationListeners.put(className, myset);
+            }
+            myset.add(entry.method);
+        }
+        _annotationEntries = entries;
+    }
+    public Set<String> getAnnotationDefinedHandlers(String className) {
+        Set<String> retval = _annotationListeners.get(className);
+        if (retval != null) return retval;
+        return Collections.emptySet();
+    }
 }