From 2b746ab45aa5b4fc636ce4a6306836c4f28f88ab Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Wed, 15 Oct 2014 15:51:31 +0100 Subject: [PATCH 1/7] pycharm ignores etc. --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 1811c16..935ff68 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,10 @@ nosetests.xml # git *.orig + +# pycharm +.idea* + +# rmv stuff +geist/data/* +test_repo/* From 0b5c01d85b533b13a308652d494374e1412ef8b3 Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Tue, 21 Oct 2014 09:19:33 +0100 Subject: [PATCH 2/7] first commit of android backend --- geist/backends/android.py | 105 ++++++++++++++++++++++++++++++++++++++ prereq.py | 49 ++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 geist/backends/android.py create mode 100644 prereq.py diff --git a/geist/backends/android.py b/geist/backends/android.py new file mode 100644 index 0000000..2927661 --- /dev/null +++ b/geist/backends/android.py @@ -0,0 +1,105 @@ +from __future__ import division, absolute_import, print_function + +import androidwebui.device_store as device_store +import androidwebui.actions as actions +import numpy as np +import struct +import subprocess +import zlib +from ..finders import Location, LocationList +from . import logger + + +class AndroidToNumpyReader(object): + _HEADER = struct.Struct('III') + + def __init__(self, fb, decompress=False): + self.h = 0 + self.w = 0 + if decompress: + wbits = zlib.MAX_WBITS | 16 + self._fb = zlib.decompress(fb, wbits) + else: + self._fb = fb + self._np = self._read_dump() + self._read_w_h_from_header() + + def _read_dump(self): + return np.fromstring(self._fb, np.uint8) + + def _read_w_h_from_header(self): + self.w, self.h = AndroidToNumpyReader._HEADER.unpack_from(self._np)[:2] + logger.debug('read header, w:%s, h%s' % (self.w, self.h)) + + def get_rect(self): + return 0, 0, self.w, self.h + + def get_image(self): + return self._np[12:].reshape((self.h, self.w, 4))[:, :, :3] + + +class GeistAndroidBackend(object): + def __init__(self, **kwargs): + self.device_id = kwargs.get('device_id', None) + logger.debug('New android backend with device %s' % self.device_id) + self._devices = device_store.DevicesStore() + self._devices.update() + logger.debug('%d devices found' % len(self._devices.get_all_devices())) + + def get_device(self, device_id=None): + if device_id: + device = self._devices.get_by_device_id(device_id) + elif self.device_id: + device = self._devices.get_by_device_id(self.device_id) + else: + device = self._devices.get_first_device() + return device if device else None + + def capture_locations(self, device_id=None): + _stream = self.do_screencap(device_id) + if not _stream: + logger.warning('no device found') + # use stream/buffer when we go back to actions.image + # _buffer = ''.join(list(_stream)) + # _image = AndroidToNumpyReader(_buffer, True).get_image() + _image = AndroidToNumpyReader(_stream, True).get_image() + + h, w = _image.shape[:2] + return LocationList([Location(0, 0, w, h, image=_image)]) + + def do_screencap(self, device_id=None, quality=1): + device = self.get_device(device_id) + if not device: + return None + # actions.image returns fbcompressed image, just use raw image for now + # return actions.image(device, quality) + process = subprocess.Popen( + r'adb -s {adb_id} shell "screencap | gzip -1" | sed "s/\r$//"'.format( + adb_id=device.adb_id + ), + shell=True, + stdout=subprocess.PIPE + ) + return process.stdout.read() + + def click(self, px, py, device_id=None): + device = self.get_device(device_id) + if not device: + return None + return actions.clickimage(device, px, py) + + def swipe(self): + pass + # TODO implement swipe + # return actions.swipeimage(self.get_device(), json_data, time) + + def press_button(self): + pass + # TODO implement button + # return actions.button(self.get_device(), button) + + def getrotation(self, device_id=None): + device = self.get_device(device_id) + if not device: + return None + return actions.getrotation(device) diff --git a/prereq.py b/prereq.py new file mode 100644 index 0000000..4324906 --- /dev/null +++ b/prereq.py @@ -0,0 +1,49 @@ +try: + get_ipython().magic(u'pylab') +except (NameError,): + pass +from geist import ( + GUI, + DirectoryRepo, + ApproxTemplateFinder, + ExactTemplateFinder, + left_of, + right_of, + below, + row_aligned, + column_aligned, + TemplateFinderFromRepo, + LocationOperatorFinder, + LocationFinderFilter, + MultipleFinderFinder, + BinaryRegionFinder, + text_finder_filter_from_path, + max_horizontal_separation, + ThresholdTemplateFinder, + ColourRegionFinder + +) +from geist.backends.android import GeistAndroidBackend +from geist.vision import grey_scale +from geist.colour import rgb_to_hsv, hsv +from geist.keyboard import KeyUp, KeyDown, KeyDownUp +from PIL import Image +from scipy.ndimage import binary_erosion, binary_dilation +from geist.pyplot import Viewer + +repo = DirectoryRepo('test_repo') +backend = GeistAndroidBackend() +gui = GUI(backend) +V = Viewer(gui, repo) +S = V.save +C = V.show_capture +F = V.show_found +R = V.show_repo + +top = lambda locations: sorted(locations, key=lambda loc: loc.y)[0] +bottom = lambda locations: sorted(locations, key=lambda loc: loc.y)[-1] +left = lambda locations: sorted(locations, key=lambda loc: loc.x)[0] +right = lambda locations: sorted(locations, key=lambda loc: loc.x)[-1] + +approx_finder = TemplateFinderFromRepo(repo, ApproxTemplateFinder) +exact_finder = TemplateFinderFromRepo(repo, ExactTemplateFinder) From fb2ffba677075a85913d6791dcd9212ea6c4d9f2 Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Tue, 21 Oct 2014 13:47:26 +0100 Subject: [PATCH 3/7] android backend part 2: buttons and swipes --- geist/backends/android.py | 50 ++++++++++++++++++++++++----- prereq.py | 67 ++++++++++++++++++++++++++++----------- 2 files changed, 90 insertions(+), 27 deletions(-) diff --git a/geist/backends/android.py b/geist/backends/android.py index 2927661..fdd2e01 100644 --- a/geist/backends/android.py +++ b/geist/backends/android.py @@ -7,9 +7,22 @@ import subprocess import zlib from ..finders import Location, LocationList +from ._common import BackendActionBuilder from . import logger +class _ActionsTransaction(object): + def __init__(self, backend): + self._actions_builder = BackendActionBuilder(backend) + + def __enter__(self): + return self._actions_builder + + def __exit__(self, *args): + self._actions_builder.execute() + return False + + class AndroidToNumpyReader(object): _HEADER = struct.Struct('III') @@ -55,6 +68,9 @@ def get_device(self, device_id=None): device = self._devices.get_first_device() return device if device else None + def actions_transaction(self): + return _ActionsTransaction(self) + def capture_locations(self, device_id=None): _stream = self.do_screencap(device_id) if not _stream: @@ -88,18 +104,36 @@ def click(self, px, py, device_id=None): return None return actions.clickimage(device, px, py) - def swipe(self): - pass - # TODO implement swipe - # return actions.swipeimage(self.get_device(), json_data, time) + def swipe(self, json_data, device_id=None, time=None): + device = self.get_device(device_id) + if not device: + return None + return actions.swipeimage(device, json_data, time) - def press_button(self): - pass - # TODO implement button - # return actions.button(self.get_device(), button) + def press_button(self, button, device_id=None): + device = self.get_device(device_id) + if not device: + return None + return actions.button(device, button) def getrotation(self, device_id=None): device = self.get_device(device_id) if not device: return None return actions.getrotation(device) + + def button_down(self, button_num): + pass + + def button_up(self, button_num): + pass + + def move(self, point): + x, y = point + self.click(x, y) + + def close(self): + pass + + def __del__(self): + self.close() diff --git a/prereq.py b/prereq.py index 4324906..29c67c1 100644 --- a/prereq.py +++ b/prereq.py @@ -7,29 +7,18 @@ DirectoryRepo, ApproxTemplateFinder, ExactTemplateFinder, - left_of, - right_of, - below, - row_aligned, - column_aligned, TemplateFinderFromRepo, - LocationOperatorFinder, - LocationFinderFilter, - MultipleFinderFinder, - BinaryRegionFinder, - text_finder_filter_from_path, - max_horizontal_separation, - ThresholdTemplateFinder, - ColourRegionFinder - ) from geist.backends.android import GeistAndroidBackend -from geist.vision import grey_scale -from geist.colour import rgb_to_hsv, hsv -from geist.keyboard import KeyUp, KeyDown, KeyDownUp -from PIL import Image -from scipy.ndimage import binary_erosion, binary_dilation from geist.pyplot import Viewer +import json + + +def go_home(): + max_presses = 50 + while max_presses and not gui.find_all(approx_finder.centrescreen): + backend.press_button("home") + max_presses -= 1 repo = DirectoryRepo('test_repo') backend = GeistAndroidBackend() @@ -47,3 +36,43 @@ approx_finder = TemplateFinderFromRepo(repo, ApproxTemplateFinder) exact_finder = TemplateFinderFromRepo(repo, ExactTemplateFinder) + +left_swipe = [ + {"startx": 110, "starty": 1066, "endx": 676, "endy": 1129}, +] +backend.swipe(json.dumps(left_swipe)) +backend.swipe(json.dumps(left_swipe)) +gui.click(approx_finder.kidspaint) + +write_geist = [ + {"startx": 268, "starty": 476, "endx": 266, "endy": 474}, + {"startx": 178, "starty": 344, "endx": 170, "endy": 308}, + {"startx": 172, "starty": 306, "endx": 138, "endy": 282}, + {"startx": 124, "starty": 282, "endx": 66, "endy": 302}, + {"startx": 62, "starty": 304, "endx": 68, "endy": 380}, + {"startx": 62, "starty": 376, "endx": 56, "endy": 470}, + {"startx": 58, "starty": 472, "endx": 110, "endy": 564}, + {"startx": 110, "starty": 564, "endx": 142, "endy": 556}, + {"startx": 148, "starty": 556, "endx": 180, "endy": 550}, + {"startx": 180, "starty": 550, "endx": 178, "endy": 492}, + {"startx": 180, "starty": 490, "endx": 176, "endy": 460}, + {"startx": 180, "starty": 452, "endx": 132, "endy": 466}, + {"startx": 234, "starty": 502, "endx": 278, "endy": 490}, + {"startx": 280, "starty": 488, "endx": 266, "endy": 456}, + {"startx": 262, "starty": 456, "endx": 242, "endy": 448}, + {"startx": 240, "starty": 446, "endx": 222, "endy": 474}, + {"startx": 220, "starty": 476, "endx": 260, "endy": 542}, + {"startx": 260, "starty": 536, "endx": 290, "endy": 524}, + {"startx": 324, "starty": 442, "endx": 330, "endy": 548}, + {"startx": 308, "starty": 376, "endx": 308, "endy": 376}, + {"startx": 406, "starty": 454, "endx": 394, "endy": 436}, + {"startx": 388, "starty": 432, "endx": 352, "endy": 444}, + {"startx": 350, "starty": 442, "endx": 368, "endy": 502}, + {"startx": 370, "starty": 494, "endx": 404, "endy": 492}, + {"startx": 406, "starty": 494, "endx": 406, "endy": 530}, + {"startx": 404, "starty": 534, "endx": 372, "endy": 546}, + {"startx": 366, "starty": 546, "endx": 346, "endy": 520}, + {"startx": 460, "starty": 238, "endx": 476, "endy": 594}, + {"startx": 412, "starty": 346, "endx": 490, "endy": 322}, +] +backend.swipe(json.dumps(write_geist)) From a58b698db89c3769b43de4bc3c91b20c13b8a89a Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Wed, 22 Oct 2014 14:58:56 +0100 Subject: [PATCH 4/7] code review changes --- geist/backends/android.py | 5 ++++- prereq.py | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/geist/backends/android.py b/geist/backends/android.py index fdd2e01..42c9c09 100644 --- a/geist/backends/android.py +++ b/geist/backends/android.py @@ -2,13 +2,16 @@ import androidwebui.device_store as device_store import androidwebui.actions as actions +import logging import numpy as np import struct import subprocess import zlib from ..finders import Location, LocationList from ._common import BackendActionBuilder -from . import logger + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class _ActionsTransaction(object): diff --git a/prereq.py b/prereq.py index 29c67c1..495ea31 100644 --- a/prereq.py +++ b/prereq.py @@ -20,6 +20,15 @@ def go_home(): backend.press_button("home") max_presses -= 1 + +def write_geist(): + backend.swipe(json.dumps(left_swipe)) + backend.swipe(json.dumps(left_swipe)) + gui.click(approx_finder.drawingapp) + backend.click(660, 50) + backend.swipe(json.dumps(geist_text)) + backend.click(560, 50) + repo = DirectoryRepo('test_repo') backend = GeistAndroidBackend() gui = GUI(backend) @@ -38,13 +47,13 @@ def go_home(): exact_finder = TemplateFinderFromRepo(repo, ExactTemplateFinder) left_swipe = [ - {"startx": 110, "starty": 1066, "endx": 676, "endy": 1129}, + {"startx": 100, "starty": 1100, "endx": 700, "endy": 1100}, +] +right_swipe = [ + {"startx": 700, "starty": 1100, "endx": 100, "endy": 1100}, ] -backend.swipe(json.dumps(left_swipe)) -backend.swipe(json.dumps(left_swipe)) -gui.click(approx_finder.kidspaint) -write_geist = [ +geist_text = [ {"startx": 268, "starty": 476, "endx": 266, "endy": 474}, {"startx": 178, "starty": 344, "endx": 170, "endy": 308}, {"startx": 172, "starty": 306, "endx": 138, "endy": 282}, @@ -75,4 +84,3 @@ def go_home(): {"startx": 460, "starty": 238, "endx": 476, "endy": 594}, {"startx": 412, "starty": 346, "endx": 490, "endy": 322}, ] -backend.swipe(json.dumps(write_geist)) From c5880c572f65a91e1e2e51c07ce94feddfab4d01 Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Wed, 22 Oct 2014 14:58:56 +0100 Subject: [PATCH 5/7] code review changes --- geist/backends/android.py | 5 ++++- prereq.py | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/geist/backends/android.py b/geist/backends/android.py index fdd2e01..42c9c09 100644 --- a/geist/backends/android.py +++ b/geist/backends/android.py @@ -2,13 +2,16 @@ import androidwebui.device_store as device_store import androidwebui.actions as actions +import logging import numpy as np import struct import subprocess import zlib from ..finders import Location, LocationList from ._common import BackendActionBuilder -from . import logger + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class _ActionsTransaction(object): diff --git a/prereq.py b/prereq.py index 29c67c1..495ea31 100644 --- a/prereq.py +++ b/prereq.py @@ -20,6 +20,15 @@ def go_home(): backend.press_button("home") max_presses -= 1 + +def write_geist(): + backend.swipe(json.dumps(left_swipe)) + backend.swipe(json.dumps(left_swipe)) + gui.click(approx_finder.drawingapp) + backend.click(660, 50) + backend.swipe(json.dumps(geist_text)) + backend.click(560, 50) + repo = DirectoryRepo('test_repo') backend = GeistAndroidBackend() gui = GUI(backend) @@ -38,13 +47,13 @@ def go_home(): exact_finder = TemplateFinderFromRepo(repo, ExactTemplateFinder) left_swipe = [ - {"startx": 110, "starty": 1066, "endx": 676, "endy": 1129}, + {"startx": 100, "starty": 1100, "endx": 700, "endy": 1100}, +] +right_swipe = [ + {"startx": 700, "starty": 1100, "endx": 100, "endy": 1100}, ] -backend.swipe(json.dumps(left_swipe)) -backend.swipe(json.dumps(left_swipe)) -gui.click(approx_finder.kidspaint) -write_geist = [ +geist_text = [ {"startx": 268, "starty": 476, "endx": 266, "endy": 474}, {"startx": 178, "starty": 344, "endx": 170, "endy": 308}, {"startx": 172, "starty": 306, "endx": 138, "endy": 282}, @@ -75,4 +84,3 @@ def go_home(): {"startx": 460, "starty": 238, "endx": 476, "endy": 594}, {"startx": 412, "starty": 346, "endx": 490, "endy": 322}, ] -backend.swipe(json.dumps(write_geist)) From f8d2a58aa0e88da3c52791b6cb3ff1309938e1e1 Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Wed, 22 Oct 2014 15:08:16 +0100 Subject: [PATCH 6/7] corrected logger setup --- geist/backends/android.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geist/backends/android.py b/geist/backends/android.py index 42c9c09..665f3c4 100644 --- a/geist/backends/android.py +++ b/geist/backends/android.py @@ -11,7 +11,6 @@ from ._common import BackendActionBuilder logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) class _ActionsTransaction(object): From 8af225d559f8c326783778ab52d85ba506748f9b Mon Sep 17 00:00:00 2001 From: Jon Morris Date: Wed, 22 Oct 2014 15:29:32 +0100 Subject: [PATCH 7/7] made prereq.py more generic --- prereq.py | 53 ++--------------------------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/prereq.py b/prereq.py index 495ea31..838a140 100644 --- a/prereq.py +++ b/prereq.py @@ -11,23 +11,6 @@ ) from geist.backends.android import GeistAndroidBackend from geist.pyplot import Viewer -import json - - -def go_home(): - max_presses = 50 - while max_presses and not gui.find_all(approx_finder.centrescreen): - backend.press_button("home") - max_presses -= 1 - - -def write_geist(): - backend.swipe(json.dumps(left_swipe)) - backend.swipe(json.dumps(left_swipe)) - gui.click(approx_finder.drawingapp) - backend.click(660, 50) - backend.swipe(json.dumps(geist_text)) - backend.click(560, 50) repo = DirectoryRepo('test_repo') backend = GeistAndroidBackend() @@ -47,40 +30,8 @@ def write_geist(): exact_finder = TemplateFinderFromRepo(repo, ExactTemplateFinder) left_swipe = [ - {"startx": 100, "starty": 1100, "endx": 700, "endy": 1100}, + {"startx": 100, "starty": 500, "endx": 700, "endy": 500}, ] right_swipe = [ - {"startx": 700, "starty": 1100, "endx": 100, "endy": 1100}, -] - -geist_text = [ - {"startx": 268, "starty": 476, "endx": 266, "endy": 474}, - {"startx": 178, "starty": 344, "endx": 170, "endy": 308}, - {"startx": 172, "starty": 306, "endx": 138, "endy": 282}, - {"startx": 124, "starty": 282, "endx": 66, "endy": 302}, - {"startx": 62, "starty": 304, "endx": 68, "endy": 380}, - {"startx": 62, "starty": 376, "endx": 56, "endy": 470}, - {"startx": 58, "starty": 472, "endx": 110, "endy": 564}, - {"startx": 110, "starty": 564, "endx": 142, "endy": 556}, - {"startx": 148, "starty": 556, "endx": 180, "endy": 550}, - {"startx": 180, "starty": 550, "endx": 178, "endy": 492}, - {"startx": 180, "starty": 490, "endx": 176, "endy": 460}, - {"startx": 180, "starty": 452, "endx": 132, "endy": 466}, - {"startx": 234, "starty": 502, "endx": 278, "endy": 490}, - {"startx": 280, "starty": 488, "endx": 266, "endy": 456}, - {"startx": 262, "starty": 456, "endx": 242, "endy": 448}, - {"startx": 240, "starty": 446, "endx": 222, "endy": 474}, - {"startx": 220, "starty": 476, "endx": 260, "endy": 542}, - {"startx": 260, "starty": 536, "endx": 290, "endy": 524}, - {"startx": 324, "starty": 442, "endx": 330, "endy": 548}, - {"startx": 308, "starty": 376, "endx": 308, "endy": 376}, - {"startx": 406, "starty": 454, "endx": 394, "endy": 436}, - {"startx": 388, "starty": 432, "endx": 352, "endy": 444}, - {"startx": 350, "starty": 442, "endx": 368, "endy": 502}, - {"startx": 370, "starty": 494, "endx": 404, "endy": 492}, - {"startx": 406, "starty": 494, "endx": 406, "endy": 530}, - {"startx": 404, "starty": 534, "endx": 372, "endy": 546}, - {"startx": 366, "starty": 546, "endx": 346, "endy": 520}, - {"startx": 460, "starty": 238, "endx": 476, "endy": 594}, - {"startx": 412, "starty": 346, "endx": 490, "endy": 322}, + {"startx": 700, "starty": 500, "endx": 100, "endy": 500}, ]