Add arcpro runner

README.md

@@ -7,9 +7,9 @@ Master branch [![Build Status](https://travis-ci.com/mapaction/mapactionpy_contr
 
 Installing
 ==========
-To install the latest stable release via PyPi:
+To install the occamlabsarcpro version via PyPi:
 ```
-python -m pip install mapactionpy_controller
+python -m pip install git+https://github.com/mapaction/mapactionpy_controller.git@occamlabsarcpro
 ```
 
 To install a specific version for testing, see the relevant command line from here:
@@ -18,6 +18,21 @@ https://pypi.org/project/mapactionpy-controller/#history
 
 Command-line Usage
 ==========
+
+* To use Mapchef using QGIS runner, please follow the following instructions : https://github.com/mapaction/mapactionpy_qgis/tree/occamlabsqgis#install-qgis-runner-on-windows-using-docker
+* To use Mapchef using Arcpro runner, please follow the following instructions : https://github.com/mapaction/mapactionpy_arcpro/tree/occamlabsarcpro#mapchef
+* To use Mapchef using Arcmap runner, please follow the following instructions : https://github.com/mapaction/mapactionpy_arcmap
+
+* The new command line for Mapchef will be as follows : 
+```
+mapchef maps --build "%CMF_PATH%/honduras/event_description.json" --map-number "MA9001" --runner "qgis_via_docker"
+```
+* The runner parameter could have the following values :
+    * **"arcpro"** for runing ArcProRunner;
+    * **"qgis_via_docker"** for runing DockerRunner;
+    * **"arcmap"** for runing ArcMapRunner;
+ * the parameter **CMF_PATH** contain the CMF path.
+
 There are two key files, typically named `cmf_description.json` and `event_description.json` that need to be in the root of the crash move folder. Most command-line options require one or the other of these.
 
 General help:
@@ -35,15 +50,6 @@ Check the compliance with the Data Naming Convention.
 mapchef gisdata --verify /path/to/current/cmf/2019gbr01/event_description.json
 ```
 
-Create all maps in the cookbook file:
-```
-mapchef maps --build /path/to/current/cmf/2019gbr01/event_description.json
-```
-
-Create the map "MA001" from the cookbook file:
-```
-mapchef maps --build --map-number "MA001" /path/to/current/cmf/2019gbr01/event_description.json
-```
 
 Programmatic Usage
 =====
@@ -133,3 +139,5 @@ Extra information associated with clause `datatheme`:
 
 The Administrative boundary (level 3) data generously supplied by the World Food Program, downloaded from https://www.wfp.org/.
 ```
+
+

mapactionpy_controller/__init__.py

@@ -1,7 +1,6 @@
 import json
 import logging
 from os import path
-
 from jsonschema import validate
 
 CONFIG_SCHEMAS_DIR = path.join(path.abspath(path.dirname(__file__)), 'schemas')

mapactionpy_controller/cli.py

@@ -1,4 +1,5 @@
 import argparse
+import logging
 import os
 
 import mapactionpy_controller.check_naming_convention as cnc
@@ -39,19 +40,24 @@ def noun_gisdata_print_output(args):
 
 def noun_maps_print_output(args):
     if args.verb == VERB_BUILD:
-        build_maps(args.humevent_desc_path, args.map_number, args.dry_run)
+        build_maps(args.humevent_desc_path, args.map_number, args.dry_run,args.runner_name)
     else:
         raise NotImplementedError(args)
 
 
-def build_maps(humevent_desc_path, map_number, dry_run):
+def build_maps(humevent_desc_path, map_number, dry_run,runner_name):
     # build_steps = config_verify.get_config_verify_steps(args.cmf_desc_path, ['.lyr'])
     # build_steps.append(cnc.get_defaultcmf_step_list(args.cmf_desc_path, False))
     # build_steps.append(cnc.get_active_data_step_list(args.humevent_desc_path, True))
     # main_stack.process_stack(build_steps)
-    my_runner = process_stack(plugin_controller.get_plugin_step(), humevent_desc_path)
-    process_stack(plugin_controller.get_cookbook_steps(my_runner, map_number, dry_run), None)
-
+    logging.info(f"loading runnnner{runner_name}")
+    my_runner = process_stack(plugin_controller.get_plugin_step(), {"state":humevent_desc_path,"runner_name":runner_name})
+    if(isinstance(my_runner,plugin_controller.DockerRunner)):
+
+        my_runner.start_runner(cmf_path=humevent_desc_path,args = map_number)
+    else:
+        process_stack(plugin_controller.get_cookbook_steps(my_runner, map_number, dry_run), None)
+#mapchef maps --build "C:\Users\BLAIT\Desktop\prepared-country-data\honduras\event_description.json" --map-number "MA9001"
     # map_nums = None
     # if map_number:
     #     map_nums = [map_number]
@@ -186,6 +192,14 @@ def get_args():
               ' all maps in the MapCookbook will be created.')
     )
 
+    prs_maps.add_argument(
+        '--runner',
+        metavar='"Runner Name"',
+        dest = "runner_name",
+        help=('the identification name of the target runner;'
+         'supported runner are : <{" ; ".join(plugin_controller.supported_runners)}>')
+    )
+
     maps_options_grp = prs_maps.add_mutually_exclusive_group(required=False)
 
     maps_options_grp.add_argument(

mapactionpy_controller/jira_tasks.py

@@ -5,7 +5,7 @@
 from datetime import datetime
 
 import pytz
-from jira import JIRA
+#from jira import JIRA
 
 from mapactionpy_controller.task_renderer import TaskReferralBase
 

mapactionpy_controller/layer_properties.py

@@ -62,7 +62,7 @@ def _parse(self):
                 mapLayer = RecipeLayer(layer, self)
                 self.properties[mapLayer.name] = mapLayer
 
-    def _get_lyr_rendering_names_as_set(self):
+    def _get_lyr_rendering_names_as_set(self): #this should append the runners specific rendering folder name to the cmf.layer_rendering to handle new structure of cmf 
         files_unique = set()
         dir_content = os.listdir(self.cmf.layer_rendering)
         for f in dir_content:

mapactionpy_controller/main_stack.py

@@ -31,6 +31,7 @@
 
 import humanfriendly.terminal as hft
 import humanfriendly.terminal.spinners as spinners
+from mapactionpy_controller.map_recipe import MapRecipe
 
 from mapactionpy_controller.steps import Step
 from mapactionpy_controller.task_renderer import TaskReferralBase
@@ -110,6 +111,7 @@ def _add_steps_from_state_to_stack(new_state, stack, old_state):
     :returns: If `new_state` contains Step objects then `old_state` is returned. Else `new_state`
        is returned
     """
+    # print(type(new_state),new_state)
     if isinstance(new_state, Step):
         stack.append(new_state)
         return old_state
@@ -118,7 +120,7 @@ def _add_steps_from_state_to_stack(new_state, stack, old_state):
         new_state.reverse()
         stack.extend(new_state)
         return old_state
-
+    
     return new_state
 
 
@@ -147,7 +149,7 @@ def process_stack(step_list, initial_state):
             # `nplus_state` = the state for the next iteraction (eg N+1)
             step = stack.pop()
             kwargs = {'state': n_state}
-
+            # print(n_state)
             if hft.connected_to_terminal():
                 with spinners.AutomaticSpinner(step.running_msg, show_time=True):
                     nplus_state = step.run(parse_feedback, **kwargs)

mapactionpy_controller/map_recipe.py

@@ -78,7 +78,7 @@ def _check_schemas_with_backward_compat(self, recipe_def):
                 validate_against_recipe_schema_v0_2(recipe_def)
                 # Do something useful here
                 # Hack some values? Or raise a JIRA ticket?
-                logger.warn('Attempting to load backwards compatible v0.2 MapRecipe')
+                logger.warn('Attempting to load backwards compatable v0.2 MapRecipe')
                 # raise ValueError('old maprecipe format')
                 return 0.2
             except jsonschema.ValidationError:
@@ -105,10 +105,10 @@ def _parse_map_project_path(self, recipe_def):
         mp_path = recipe_def.get('map_project_path', None)
 
         if mp_path:
-            mp_path = path.abspath(self.map_project_path)
+            mp_path = path.abspath(mp_path)
 
         return mp_path
-
+    
     def _parse_core_file_name(self, recipe_def):
         core_fname = recipe_def.get('core_file_name', None)
 

mapactionpy_controller/plugin_base.py

@@ -42,7 +42,8 @@ def get_lyr_render_extension(self, **kwargs):
             'BaseRunnerPlugin is an abstract class and the `get_lyr_render_extension`'
             ' method cannot be called directly')
 
-    def _get_all_templates_by_regex(self, recipe):
+    def _get_all_templates_by_regex(self, recipe): #Todo should we use the layoutManager from project instance 
+                                                    #to get embeded templates and match the regex against their names 
         """
         Gets the fully qualified filenames of map templates, which exist in `self.cmf.map_templates` whose
         filenames match the regex `recipe.template`.
@@ -53,25 +54,28 @@ def _get_all_templates_by_regex(self, recipe):
                  `self.get_projectfile_extension()`
         """
         def _is_relevant_file(f):
+
             extension = os.path.splitext(f)[1]
-            logger.debug('checking file "{}", with extension "{}", against pattern "{}" and "{}"'.format(
+            logger.info('checking file "{}", with extension "{}", against pattern "{}" and "{}"'.format(
                 f, extension, recipe.template, self.get_projectfile_extension()
             ))
             if re.search(recipe.template, f):
                 logger.debug('file {} matched regex'.format(f))
                 f_path = os.path.join(self.cmf.map_templates, f)
                 logger.debug('file {} joined with self.cmf.map_templates "{}"'.format(f, f_path))
-                return (os.path.isfile(f_path)) and (extension == self.get_projectfile_extension())
+                is_relevent =  (os.path.isfile(f_path)) and (extension == self.get_projectfile_extension())
+                if(is_relevent): logging.info(f"got relevent {f_path}")
+                return is_relevent
             else:
                 return False
 
         # TODO: This results in calling `os.path.join` twice for certain files
-        logger.debug('searching for map templates in; {}'.format(self.cmf.map_templates))
+        logger.info('searching for map templates in; {}'.format(self.cmf.map_templates))
         all_filenames = os.listdir(self.cmf.map_templates)
-        logger.debug('all available template files:\n\t{}'.format('\n\t'.join(all_filenames)))
+        logger.info('all available template files:\n\t{}'.format('\n\t'.join(all_filenames)))
         relevant_filenames = [os.path.realpath(os.path.join(self.cmf.map_templates, fi))
                               for fi in all_filenames if _is_relevant_file(fi)]
-        logger.debug('possible template files:\n\t{}'.format('\n\t'.join(relevant_filenames)))
+        logger.info('possible template files:\n\t{}'.format('\n\t'.join(relevant_filenames)))
         return relevant_filenames
 
     def _get_template_by_aspect_ratio(self, template_aspect_ratios, target_ar):
@@ -168,10 +172,10 @@ def get_templates(self, **kwargs):
 
         # use `recipe.template` as regex to locate one or more templates
         possible_templates = self._get_all_templates_by_regex(recipe)
-
+        
         # Select the template with the most appropriate aspect ratio
         possible_aspect_ratios = self.get_aspect_ratios_of_templates(possible_templates, recipe)
-
+        logging.info(f"possible ARio : {possible_aspect_ratios}")
         mf = recipe.get_frame(recipe.principal_map_frame)
         # Default value
         target_aspect_ratio = 1.0
@@ -182,7 +186,7 @@ def get_templates(self, **kwargs):
         # use logic to workout which template has best aspect ratio
         # obviously not this logic though:
         recipe.template_path = self._get_template_by_aspect_ratio(possible_aspect_ratios, target_aspect_ratio)
-
+        
         # TODO re-enable "Have the input files changed?"
         # Have the input shapefiles changed?
         return recipe
@@ -205,7 +209,7 @@ def get_next_map_version_number(self, mapNumberDirectory, mapNumber, mapFileName
         versionNumber = versionNumber + 1
         return versionNumber
 
-    # Is it possible to avoid the need to hardcode the naming convention for the output mxds? Eg could a
+    # TODO Is it possible to aviod the need to hardcode the naming convention for the output mxds? Eg could a
     # String.Template be specified within the Cookbook?
     # https://docs.python.org/2/library/string.html#formatspec
     # https://www.python.org/dev/peps/pep-3101/
@@ -277,7 +281,7 @@ def _create_export_dir(self, recipe):
 
     def _do_export(self, export_params, recipe):
         """
-        Note implementing subclasses, must return the dict `export_params`, with
+        Note implenmenting subclasses, must return the dict `export_params`, with
         key/value pairs which satisfies the `_check_plugin_supplied_params` method.
         """
         raise NotImplementedError(

mapactionpy_controller/plugin_controller.py

@@ -1,34 +1,102 @@
+from ast import Import
+from itertools import count
 import logging
-
+from lzma import CHECK_ID_MAX
+import subprocess
 from mapactionpy_controller.event import Event
 from mapactionpy_controller.layer_properties import LayerProperties
 from mapactionpy_controller.map_cookbook import MapCookbook
 from mapactionpy_controller.steps import Step
 import mapactionpy_controller.data_search as data_search
-
+import os
+from sys import platform, stdout
 # logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
+class DockerRunner :
+
+    _run_container_command = "docker run -it -v \"{CMF_PATH}\":/cmf {container_name}  conda run --no-capture-output  -n myenv mapchef maps --build \"{linux_cmf_path}\"  --map-number \"{map_number}\""
+    def __init__(self,container_name) -> None:
+        #from python_on_whales import docker
+        self.container_name = container_name
+        self.status = None
+    def check_docker_container(self):
+        container_name = "qgisrunner"
+        try :
+            docker_subprocess = subprocess.check_output("docker container inspect -f '{{.State.Status}}' " +container_name,shell=True,stderr=subprocess.STDOUT)
+            self.status = docker_subprocess.decode('ascii').strip()
+
+        except subprocess.CalledProcessError as e:
+            logging.info(f"cant find any docker container with name {container_name} --> {e}")
+            raise e
+        return self.status
+            #raise a specific exception 
+
+    def start_runner(self,cmf_path="%CMF_PATH%",args = ""):
+        #if(self.check_docker_container()):
+        #    if(self.status == "running"):
+        #        logging.info(f"there is already a tunning container {self.container_name}")
+        #    else :
+        country_path,event_file = os.path.split(cmf_path)
+        cmf_root,country_path = os.path.split(country_path)
+        run_cmd = self._run_container_command.format(CMF_PATH=cmf_root,linux_cmf_path = f"/cmf/{country_path}/{event_file}" ,container_name = self.container_name,map_number = args)
+        docker_subprocess = subprocess.Popen(run_cmd,shell=False)
+        docker_subprocess.communicate()
+        #logging.info(f"docker result : {docker_subprocess.decode('ascii').strip()}")
+
+supported_runners = {"arcpro":"mapactionpy_arcpro.arcpro_runner.ArcProRunner",\
+                        "qgis":"mapactionpy_qgis.qgis_runner.QGisRunner",\
+                        "qgis_via_docker":DockerRunner,\
+                        "arcmap":"mapactionpy_arcmap.arcmap_runner.ArcMapRunner"}
+
 
 def get_plugin_step():
+
     def get_plugin(**kwargs):
-        hum_event = kwargs['state']
+        logging.info(f"inside runner loader {kwargs.keys()}")
+        hum_event = kwargs['state']['hum_event']
+        if(kwargs['state']["runner_name"]):
+            runner_name = kwargs["state"]["runner_name"]
+            try:
+
+                logging.info("inside runner loader")
+                if(runner_name == "qgis_via_docker"):
+                    return DockerRunner("qgisrunner")
+                runner_name = kwargs['state']['runner_name']
+                pak,mod,rclass = supported_runners[runner_name].split('.')
+                runner_class =getattr(getattr(__import__(pak),mod),rclass)
+                return runner_class(hum_event)
+            except ImportError as e :
+                logging.debug(f"Failed to load the {runner_name}")
         try:
-            logger.debug('Attempting to load the ArcMapRunner')
-            from mapactionpy_arcmap.arcmap_runner import ArcMapRunner
-            runner = ArcMapRunner(hum_event)
-            logger.info('Successfully loaded the ArcMapRunner')
+            logger.debug('Attempting to load the ArcProRunner')
+            from mapactionpy_arcpro.arcpro_runner import ArcProRunner
+            runner = ArcProRunner(hum_event)
+            logger.info('Successfully loaded the ArcProRunner')
         except ImportError:
-            logger.debug('Failed to load the ArcMapRunner')
+            logger.debug('Failed to load the ArcProRunner')
             logger.debug('Attempting to load the QGisRunner')
-            from mapactionpy_qgis.qgis_runner import QGisRunner
-            runner = QGisRunner()
-            logger.info('Failed to load the ArcMapRunner')
+            try :
+                if(platform != 'win32'):
+                    from mapactionpy_qgis.qgis_runner import QGisRunner
+                    runner = QGisRunner(hum_event)
+                    logger.info('Successfully loaded the QGisRunner')
+                else:
+                    logger.debug('Failed to load the QGisRunner')
+                    logger.debug('Attempting to load the DockerRunner')
+                    runner = DockerRunner("qgisrunner")
+                        #if(runner.check_docker_container()):
+                    return runner
+            except ImportError :            
+                logger.debug('Failed to load the DockerRunner')
+                logger.debug('Attempting to load the ArcMapRunner')
+                from mapactionpy_arcmap.arcmap_runner import ArcMapRunner
+                runner = ArcMapRunner(hum_event)
 
         return runner
 
     def new_event(**kwargs):
-        return Event(kwargs['state'])
+        return {"hum_event":Event(kwargs['state']['state']),"runner_name":kwargs['state']["runner_name"]}
 
     plugin_step = [
         Step(