diff --git a/README.md b/README.md
index fe85442..78c27d7 100644
--- a/README.md
+++ b/README.md
@@ -7,9 +7,9 @@ Master branch [ data generously supplied by the World Food Program, downloaded from https://www.wfp.org/.
```
+
+
diff --git a/mapactionpy_controller/__init__.py b/mapactionpy_controller/__init__.py
index 09ceeb6..af8eba7 100644
--- a/mapactionpy_controller/__init__.py
+++ b/mapactionpy_controller/__init__.py
@@ -1,7 +1,6 @@
import json
import logging
from os import path
-
from jsonschema import validate
CONFIG_SCHEMAS_DIR = path.join(path.abspath(path.dirname(__file__)), 'schemas')
diff --git a/mapactionpy_controller/cli.py b/mapactionpy_controller/cli.py
index 31bb38d..917e023 100644
--- a/mapactionpy_controller/cli.py
+++ b/mapactionpy_controller/cli.py
@@ -1,4 +1,5 @@
import argparse
+import logging
import os
import mapactionpy_controller.check_naming_convention as cnc
@@ -39,19 +40,24 @@ def noun_gisdata_print_output(args):
def noun_maps_print_output(args):
if args.verb == VERB_BUILD:
- build_maps(args.humevent_desc_path, args.map_number, args.dry_run)
+ build_maps(args.humevent_desc_path, args.map_number, args.dry_run,args.runner_name)
else:
raise NotImplementedError(args)
-def build_maps(humevent_desc_path, map_number, dry_run):
+def build_maps(humevent_desc_path, map_number, dry_run,runner_name):
# build_steps = config_verify.get_config_verify_steps(args.cmf_desc_path, ['.lyr'])
# build_steps.append(cnc.get_defaultcmf_step_list(args.cmf_desc_path, False))
# build_steps.append(cnc.get_active_data_step_list(args.humevent_desc_path, True))
# main_stack.process_stack(build_steps)
- my_runner = process_stack(plugin_controller.get_plugin_step(), humevent_desc_path)
- process_stack(plugin_controller.get_cookbook_steps(my_runner, map_number, dry_run), None)
-
+ logging.info(f"loading runnnner{runner_name}")
+ my_runner = process_stack(plugin_controller.get_plugin_step(), {"state":humevent_desc_path,"runner_name":runner_name})
+ if(isinstance(my_runner,plugin_controller.DockerRunner)):
+
+ my_runner.start_runner(cmf_path=humevent_desc_path,args = map_number)
+ else:
+ process_stack(plugin_controller.get_cookbook_steps(my_runner, map_number, dry_run), None)
+#mapchef maps --build "C:\Users\BLAIT\Desktop\prepared-country-data\honduras\event_description.json" --map-number "MA9001"
# map_nums = None
# if map_number:
# map_nums = [map_number]
@@ -186,6 +192,14 @@ def get_args():
' all maps in the MapCookbook will be created.')
)
+ prs_maps.add_argument(
+ '--runner',
+ metavar='"Runner Name"',
+ dest = "runner_name",
+ help=('the identification name of the target runner;'
+ 'supported runner are : <{" ; ".join(plugin_controller.supported_runners)}>')
+ )
+
maps_options_grp = prs_maps.add_mutually_exclusive_group(required=False)
maps_options_grp.add_argument(
diff --git a/mapactionpy_controller/jira_tasks.py b/mapactionpy_controller/jira_tasks.py
index cd81747..1bcc65c 100644
--- a/mapactionpy_controller/jira_tasks.py
+++ b/mapactionpy_controller/jira_tasks.py
@@ -5,7 +5,7 @@
from datetime import datetime
import pytz
-from jira import JIRA
+#from jira import JIRA
from mapactionpy_controller.task_renderer import TaskReferralBase
diff --git a/mapactionpy_controller/layer_properties.py b/mapactionpy_controller/layer_properties.py
index 7492ede..6338adf 100644
--- a/mapactionpy_controller/layer_properties.py
+++ b/mapactionpy_controller/layer_properties.py
@@ -62,7 +62,7 @@ def _parse(self):
mapLayer = RecipeLayer(layer, self)
self.properties[mapLayer.name] = mapLayer
- def _get_lyr_rendering_names_as_set(self):
+ def _get_lyr_rendering_names_as_set(self): #this should append the runners specific rendering folder name to the cmf.layer_rendering to handle new structure of cmf
files_unique = set()
dir_content = os.listdir(self.cmf.layer_rendering)
for f in dir_content:
diff --git a/mapactionpy_controller/main_stack.py b/mapactionpy_controller/main_stack.py
index aef5daf..824765e 100644
--- a/mapactionpy_controller/main_stack.py
+++ b/mapactionpy_controller/main_stack.py
@@ -31,6 +31,7 @@
import humanfriendly.terminal as hft
import humanfriendly.terminal.spinners as spinners
+from mapactionpy_controller.map_recipe import MapRecipe
from mapactionpy_controller.steps import Step
from mapactionpy_controller.task_renderer import TaskReferralBase
@@ -110,6 +111,7 @@ def _add_steps_from_state_to_stack(new_state, stack, old_state):
:returns: If `new_state` contains Step objects then `old_state` is returned. Else `new_state`
is returned
"""
+ # print(type(new_state),new_state)
if isinstance(new_state, Step):
stack.append(new_state)
return old_state
@@ -118,7 +120,7 @@ def _add_steps_from_state_to_stack(new_state, stack, old_state):
new_state.reverse()
stack.extend(new_state)
return old_state
-
+
return new_state
@@ -147,7 +149,7 @@ def process_stack(step_list, initial_state):
# `nplus_state` = the state for the next iteraction (eg N+1)
step = stack.pop()
kwargs = {'state': n_state}
-
+ # print(n_state)
if hft.connected_to_terminal():
with spinners.AutomaticSpinner(step.running_msg, show_time=True):
nplus_state = step.run(parse_feedback, **kwargs)
diff --git a/mapactionpy_controller/map_recipe.py b/mapactionpy_controller/map_recipe.py
index 2743115..f253090 100644
--- a/mapactionpy_controller/map_recipe.py
+++ b/mapactionpy_controller/map_recipe.py
@@ -78,7 +78,7 @@ def _check_schemas_with_backward_compat(self, recipe_def):
validate_against_recipe_schema_v0_2(recipe_def)
# Do something useful here
# Hack some values? Or raise a JIRA ticket?
- logger.warn('Attempting to load backwards compatible v0.2 MapRecipe')
+ logger.warn('Attempting to load backwards compatable v0.2 MapRecipe')
# raise ValueError('old maprecipe format')
return 0.2
except jsonschema.ValidationError:
@@ -105,10 +105,10 @@ def _parse_map_project_path(self, recipe_def):
mp_path = recipe_def.get('map_project_path', None)
if mp_path:
- mp_path = path.abspath(self.map_project_path)
+ mp_path = path.abspath(mp_path)
return mp_path
-
+
def _parse_core_file_name(self, recipe_def):
core_fname = recipe_def.get('core_file_name', None)
diff --git a/mapactionpy_controller/plugin_base.py b/mapactionpy_controller/plugin_base.py
index ce1b248..610561e 100644
--- a/mapactionpy_controller/plugin_base.py
+++ b/mapactionpy_controller/plugin_base.py
@@ -42,7 +42,8 @@ def get_lyr_render_extension(self, **kwargs):
'BaseRunnerPlugin is an abstract class and the `get_lyr_render_extension`'
' method cannot be called directly')
- def _get_all_templates_by_regex(self, recipe):
+ def _get_all_templates_by_regex(self, recipe): #Todo should we use the layoutManager from project instance
+ #to get embeded templates and match the regex against their names
"""
Gets the fully qualified filenames of map templates, which exist in `self.cmf.map_templates` whose
filenames match the regex `recipe.template`.
@@ -53,25 +54,28 @@ def _get_all_templates_by_regex(self, recipe):
`self.get_projectfile_extension()`
"""
def _is_relevant_file(f):
+
extension = os.path.splitext(f)[1]
- logger.debug('checking file "{}", with extension "{}", against pattern "{}" and "{}"'.format(
+ logger.info('checking file "{}", with extension "{}", against pattern "{}" and "{}"'.format(
f, extension, recipe.template, self.get_projectfile_extension()
))
if re.search(recipe.template, f):
logger.debug('file {} matched regex'.format(f))
f_path = os.path.join(self.cmf.map_templates, f)
logger.debug('file {} joined with self.cmf.map_templates "{}"'.format(f, f_path))
- return (os.path.isfile(f_path)) and (extension == self.get_projectfile_extension())
+ is_relevent = (os.path.isfile(f_path)) and (extension == self.get_projectfile_extension())
+ if(is_relevent): logging.info(f"got relevent {f_path}")
+ return is_relevent
else:
return False
# TODO: This results in calling `os.path.join` twice for certain files
- logger.debug('searching for map templates in; {}'.format(self.cmf.map_templates))
+ logger.info('searching for map templates in; {}'.format(self.cmf.map_templates))
all_filenames = os.listdir(self.cmf.map_templates)
- logger.debug('all available template files:\n\t{}'.format('\n\t'.join(all_filenames)))
+ logger.info('all available template files:\n\t{}'.format('\n\t'.join(all_filenames)))
relevant_filenames = [os.path.realpath(os.path.join(self.cmf.map_templates, fi))
for fi in all_filenames if _is_relevant_file(fi)]
- logger.debug('possible template files:\n\t{}'.format('\n\t'.join(relevant_filenames)))
+ logger.info('possible template files:\n\t{}'.format('\n\t'.join(relevant_filenames)))
return relevant_filenames
def _get_template_by_aspect_ratio(self, template_aspect_ratios, target_ar):
@@ -168,10 +172,10 @@ def get_templates(self, **kwargs):
# use `recipe.template` as regex to locate one or more templates
possible_templates = self._get_all_templates_by_regex(recipe)
-
+
# Select the template with the most appropriate aspect ratio
possible_aspect_ratios = self.get_aspect_ratios_of_templates(possible_templates, recipe)
-
+ logging.info(f"possible ARio : {possible_aspect_ratios}")
mf = recipe.get_frame(recipe.principal_map_frame)
# Default value
target_aspect_ratio = 1.0
@@ -182,7 +186,7 @@ def get_templates(self, **kwargs):
# use logic to workout which template has best aspect ratio
# obviously not this logic though:
recipe.template_path = self._get_template_by_aspect_ratio(possible_aspect_ratios, target_aspect_ratio)
-
+
# TODO re-enable "Have the input files changed?"
# Have the input shapefiles changed?
return recipe
@@ -205,7 +209,7 @@ def get_next_map_version_number(self, mapNumberDirectory, mapNumber, mapFileName
versionNumber = versionNumber + 1
return versionNumber
- # Is it possible to avoid the need to hardcode the naming convention for the output mxds? Eg could a
+ # TODO Is it possible to aviod the need to hardcode the naming convention for the output mxds? Eg could a
# String.Template be specified within the Cookbook?
# https://docs.python.org/2/library/string.html#formatspec
# https://www.python.org/dev/peps/pep-3101/
@@ -277,7 +281,7 @@ def _create_export_dir(self, recipe):
def _do_export(self, export_params, recipe):
"""
- Note implementing subclasses, must return the dict `export_params`, with
+ Note implenmenting subclasses, must return the dict `export_params`, with
key/value pairs which satisfies the `_check_plugin_supplied_params` method.
"""
raise NotImplementedError(
diff --git a/mapactionpy_controller/plugin_controller.py b/mapactionpy_controller/plugin_controller.py
index ecf5c03..ac9b734 100644
--- a/mapactionpy_controller/plugin_controller.py
+++ b/mapactionpy_controller/plugin_controller.py
@@ -1,34 +1,102 @@
+from ast import Import
+from itertools import count
import logging
-
+from lzma import CHECK_ID_MAX
+import subprocess
from mapactionpy_controller.event import Event
from mapactionpy_controller.layer_properties import LayerProperties
from mapactionpy_controller.map_cookbook import MapCookbook
from mapactionpy_controller.steps import Step
import mapactionpy_controller.data_search as data_search
-
+import os
+from sys import platform, stdout
# logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
+class DockerRunner :
+
+ _run_container_command = "docker run -it -v \"{CMF_PATH}\":/cmf {container_name} conda run --no-capture-output -n myenv mapchef maps --build \"{linux_cmf_path}\" --map-number \"{map_number}\""
+ def __init__(self,container_name) -> None:
+ #from python_on_whales import docker
+ self.container_name = container_name
+ self.status = None
+ def check_docker_container(self):
+ container_name = "qgisrunner"
+ try :
+ docker_subprocess = subprocess.check_output("docker container inspect -f '{{.State.Status}}' " +container_name,shell=True,stderr=subprocess.STDOUT)
+ self.status = docker_subprocess.decode('ascii').strip()
+
+ except subprocess.CalledProcessError as e:
+ logging.info(f"cant find any docker container with name {container_name} --> {e}")
+ raise e
+ return self.status
+ #raise a specific exception
+
+ def start_runner(self,cmf_path="%CMF_PATH%",args = ""):
+ #if(self.check_docker_container()):
+ # if(self.status == "running"):
+ # logging.info(f"there is already a tunning container {self.container_name}")
+ # else :
+ country_path,event_file = os.path.split(cmf_path)
+ cmf_root,country_path = os.path.split(country_path)
+ run_cmd = self._run_container_command.format(CMF_PATH=cmf_root,linux_cmf_path = f"/cmf/{country_path}/{event_file}" ,container_name = self.container_name,map_number = args)
+ docker_subprocess = subprocess.Popen(run_cmd,shell=False)
+ docker_subprocess.communicate()
+ #logging.info(f"docker result : {docker_subprocess.decode('ascii').strip()}")
+
+supported_runners = {"arcpro":"mapactionpy_arcpro.arcpro_runner.ArcProRunner",\
+ "qgis":"mapactionpy_qgis.qgis_runner.QGisRunner",\
+ "qgis_via_docker":DockerRunner,\
+ "arcmap":"mapactionpy_arcmap.arcmap_runner.ArcMapRunner"}
+
def get_plugin_step():
+
def get_plugin(**kwargs):
- hum_event = kwargs['state']
+ logging.info(f"inside runner loader {kwargs.keys()}")
+ hum_event = kwargs['state']['hum_event']
+ if(kwargs['state']["runner_name"]):
+ runner_name = kwargs["state"]["runner_name"]
+ try:
+
+ logging.info("inside runner loader")
+ if(runner_name == "qgis_via_docker"):
+ return DockerRunner("qgisrunner")
+ runner_name = kwargs['state']['runner_name']
+ pak,mod,rclass = supported_runners[runner_name].split('.')
+ runner_class =getattr(getattr(__import__(pak),mod),rclass)
+ return runner_class(hum_event)
+ except ImportError as e :
+ logging.debug(f"Failed to load the {runner_name}")
try:
- logger.debug('Attempting to load the ArcMapRunner')
- from mapactionpy_arcmap.arcmap_runner import ArcMapRunner
- runner = ArcMapRunner(hum_event)
- logger.info('Successfully loaded the ArcMapRunner')
+ logger.debug('Attempting to load the ArcProRunner')
+ from mapactionpy_arcpro.arcpro_runner import ArcProRunner
+ runner = ArcProRunner(hum_event)
+ logger.info('Successfully loaded the ArcProRunner')
except ImportError:
- logger.debug('Failed to load the ArcMapRunner')
+ logger.debug('Failed to load the ArcProRunner')
logger.debug('Attempting to load the QGisRunner')
- from mapactionpy_qgis.qgis_runner import QGisRunner
- runner = QGisRunner()
- logger.info('Failed to load the ArcMapRunner')
+ try :
+ if(platform != 'win32'):
+ from mapactionpy_qgis.qgis_runner import QGisRunner
+ runner = QGisRunner(hum_event)
+ logger.info('Successfully loaded the QGisRunner')
+ else:
+ logger.debug('Failed to load the QGisRunner')
+ logger.debug('Attempting to load the DockerRunner')
+ runner = DockerRunner("qgisrunner")
+ #if(runner.check_docker_container()):
+ return runner
+ except ImportError :
+ logger.debug('Failed to load the DockerRunner')
+ logger.debug('Attempting to load the ArcMapRunner')
+ from mapactionpy_arcmap.arcmap_runner import ArcMapRunner
+ runner = ArcMapRunner(hum_event)
return runner
def new_event(**kwargs):
- return Event(kwargs['state'])
+ return {"hum_event":Event(kwargs['state']['state']),"runner_name":kwargs['state']["runner_name"]}
plugin_step = [
Step(
diff --git a/mapactionpy_controller/tests/fixtures_export_metadata.py b/mapactionpy_controller/tests/fixtures_export_metadata.py
index c137eeb..fe897d2 100644
--- a/mapactionpy_controller/tests/fixtures_export_metadata.py
+++ b/mapactionpy_controller/tests/fixtures_export_metadata.py
@@ -30,6 +30,7 @@
'papersize': 'A3',
'jpgresolutiondpi': '300',
'pdfresolutiondpi': '300',
+ 'kmlresolutiondpi': '50',
'mapfilename': 'MA9001-v05-country-overview-with-admin-1-boundaries-and-topography.mxd',
'paperxmax': '',
'paperxmin': '',
@@ -59,6 +60,7 @@
MA9001-v05-country-overview-with-admin-1-boundaries-and-topography-300dpi.jpeg
2118804
300
+ 50
English
en
@@ -131,6 +133,7 @@
A3
300
300
+ 50
MA9001-v05-country-overview-with-admin-1-boundaries-and-topography.mxd
diff --git a/mapactionpy_controller/tests/test_plugin_base.py b/mapactionpy_controller/tests/test_plugin_base.py
index 4425bdc..8cc279c 100644
--- a/mapactionpy_controller/tests/test_plugin_base.py
+++ b/mapactionpy_controller/tests/test_plugin_base.py
@@ -38,7 +38,7 @@ def get_projectfile_extension(self):
def get_lyr_render_extension(self):
return '.lyr'
- def create_output_map_project(self, **kwargs):
+ def create_ouput_map_project(self, **kwargs):
return kwargs['state']
@@ -110,9 +110,9 @@ def test_get_template_by_aspect_ratio(self):
self.assertEqual(expect_result, actual_result)
template_aspect_ratios = [
- ('landscape_bottom', 1.975),
- ('landscape_side', 1.294117647),
- ('portrait', 0.816816817)
+ ('landscape_bottom', 1.975),
+ ('landscape_side', 1.294117647),
+ ('portrait' , 0.816816817)
]
# linear_aspect_ratios = [
@@ -181,7 +181,7 @@ def test_get_next_map_version_number(self):
self.fail()
@skip('Not ready yet')
- def test_create_output_map_project(self):
+ def test_create_ouput_map_project(self):
self.fail()
@skip('Not ready yet')
diff --git a/mapactionpy_controller/tests/test_xml_export.py b/mapactionpy_controller/tests/test_xml_export.py
index 932dff5..95f315b 100644
--- a/mapactionpy_controller/tests/test_xml_export.py
+++ b/mapactionpy_controller/tests/test_xml_export.py
@@ -72,7 +72,7 @@ def test_write_xml_file(self):
test_recipe.export_metadata = femd.case1_export_metadata_dict
test_recipe.core_file_name = 'my-test-file'
test_recipe.export_path = tempfile.gettempdir()
- expected_export_fpath = os.path.join(test_recipe.export_path, 'my-test-file.xml')
+ excepted_export_fpath = os.path.join(test_recipe.export_path, 'my-test-file.xml')
m = mock.mock_open()
if six.PY2:
@@ -82,8 +82,8 @@ def test_write_xml_file(self):
with mock.patch("builtins.open", m, create=True):
actual_export_path = xml_exporter.write_export_metadata_to_xml(test_recipe)
- m.assert_called_once_with(expected_export_fpath, 'wb')
- self.assertEqual(expected_export_fpath, actual_export_path)
+ m.assert_called_once_with(excepted_export_fpath, 'w')
+ self.assertEqual(excepted_export_fpath, actual_export_path)
# Case 2: Metadata missing from the recipe
# Case 3: Write error
diff --git a/mapactionpy_controller/xml_exporter.py b/mapactionpy_controller/xml_exporter.py
index 52ae71e..4c08f13 100644
--- a/mapactionpy_controller/xml_exporter.py
+++ b/mapactionpy_controller/xml_exporter.py
@@ -1,4 +1,5 @@
import os
+import pycountry
from dicttoxml import dicttoxml
from xml.dom.minidom import parseString
import xml.etree.ElementTree as ET
@@ -11,7 +12,7 @@ def _check_for_export_metadata(recipe):
raises ValueError: If any of the requried keys are missing.
"""
- minimal_keys = {
+ mininal_keys = {
'themes',
'pdffilename',
'jpgfilename',
@@ -26,7 +27,7 @@ def _check_for_export_metadata(recipe):
'product-type'
}
- missing_keys = minimal_keys.difference(set(recipe.export_metadata.keys()))
+ missing_keys = mininal_keys.difference(set(recipe.export_metadata.keys()))
if missing_keys:
if len(missing_keys) > 0:
raise ValueError(
@@ -35,12 +36,22 @@ def _check_for_export_metadata(recipe):
def write_export_metadata_to_xml(recipe):
+ # Set up dictionary for all the values required for the export XML file
+ # map_data = MapData(exportPropertiesDict)
+ # mapDocument = MapDoc(map_data)
+
xml_fname = recipe.core_file_name+".xml"
xml_fpath = os.path.join(recipe.export_path, xml_fname)
+
+ # export_params_dict = _create_export_params_dict(recipe.export_metadata)
+ # xml = dicttoxml(export_params_dict, attr_type=False, custom_root='mapdoc')
+ # print(parseString(xml).toprettyxml())
+
xmls = _export_metadata_to_xmls(recipe)
with open(xml_fpath, "wb") as xml_file:
xml_file.write(xmls)
+
return xml_fpath
@@ -104,31 +115,40 @@ def _create_export_params_dict(recipe):
'accessnotes': "",
'location': "",
'qclevel': "Automatically generated",
+ 'qcname': "",
'proj': "",
'datasource': "",
+ 'kmlresolutiondpi': "",
+ 'paperxmax': "",
+ 'paperxmin': "",
+ 'paperymax': "",
+ 'paperymin': "",
'createdate': "",
'createtime': "",
'scale': "",
- 'datum': ""
- # ,
- # "language-iso2": recipe.hum_event.language_iso2,
- # "pdfresolutiondpi": recipe.hum_event.default_pdf_res_dpi,
- # "jpgresolutiondpi": recipe.hum_event.default_jpeg_res_dpi,
- # "countries": recipe.hum_event.country_name,
- # "glideno": recipe.hum_event.glide_number,
- # "operationID": recipe.hum_event.operation_id,
- # "sourceorg": recipe.hum_event.default_source_organisation
+ 'datum': "",
+ "language-iso2": recipe.hum_event.language_iso2,
+ "pdfresolutiondpi": recipe.hum_event.default_pdf_res_dpi,
+ "jpgresolutiondpi": recipe.hum_event.default_jpeg_res_dpi,
+ "countries": recipe.hum_event.country_name,
+ "glideno": recipe.hum_event.glide_number,
+ "operationID": recipe.hum_event.operation_id,
+ "sourceorg": recipe.hum_event.default_source_organisation
}
- for propertyToRemove in ["exportemf", "exportDirectory"]:
- if propertyToRemove in (recipe.export_metadata):
- del recipe.export_metadata[propertyToRemove]
-
# Copy from params
all_export_metadata.update(recipe.export_metadata)
- versionNumber = int(all_export_metadata.get("versionNumber", 1))
- if (versionNumber == 1):
+ if (all_export_metadata["versionNumber"] == 1):
all_export_metadata["status"] = "New"
+ else:
+ all_export_metadata["status"] = "Update"
+
+ language = pycountry.languages.get(alpha_2=recipe.hum_event.language_iso2)
+ if (language is not None):
+ all_export_metadata["language"] = language.name
+ else:
+ all_export_metadata["language"] = None
+ # return
return {'mapdata': all_export_metadata}
diff --git a/setup.py b/setup.py
index 70702cc..ce16f17 100644
--- a/setup.py
+++ b/setup.py
@@ -82,7 +82,7 @@ def _get_requires_list():
'pyyaml',
'pyshp',
'six>=1.11.0',
- 'slugify'
+ 'python-slugify'
]
if (sys.version_info.major == 2):
@@ -130,7 +130,7 @@ def _get_requires_list():
description='Controls the workflow of map and infographic production',
long_description=readme(),
long_description_content_type="text/markdown",
- url='http://github.com/mapaction/mapactionpy_controller',
+ url='http://github.com/mapaction/mapactionpy_controller/tree/occamlabsarcpro',
author='MapAction',
author_email='github@mapaction.com',
license='GPL3',