diff --git a/acre/__init__.py b/acre/__init__.py index c035e52..ec62368 100644 --- a/acre/__init__.py +++ b/acre/__init__.py @@ -1,20 +1,24 @@ from .core import ( - parse, - append, - compute, + prepare, + join, + build, merge, - get_tools, + discover, + locate, + launch, CycleError, DynamicKeyClashError ) __all__ = [ - "parse", - "append", - "compute", + "prepare", + "join", + "build", "merge", - "get_tools", + "discover", + "locate", + "launch", "CycleError", "DynamicKeyClashError" diff --git a/acre/__main__.py b/acre/__main__.py new file mode 100644 index 0000000..58d5057 --- /dev/null +++ b/acre/__main__.py @@ -0,0 +1,41 @@ +import os +import sys +import time +import argparse + +from . import discover, build, merge, locate, launch + +parser = argparse.ArgumentParser() +parser.add_argument("--tools", + help="The tool environments to include. " + "These should be separated by `;`", + required=True) +parser.add_argument("--executable", + help="The executable to run. ", + required=True) + +kwargs, args = parser.parse_known_args() + +# Build environment based on tools +tools = kwargs.tools.split(";") +tools_env = discover(kwargs.tools.split(";")) +env = build(tools_env) +env = merge(env, current_env=dict(os.environ)) + +# Search for the executable within the tool's environment +# by temporarily taking on its `PATH` settings +exe = locate(kwargs.executable, env) +try: + if not exe: + raise ValueError("Unable to find executable: %s" % kwargs.executable) +except Exception as exc: + time.sleep(10) + sys.exit(1) + +try: + launch(exe, environment=env, args=args) +except Exception as exc: + # Ensure we can capture any exception and give the user (and us) time + # to read it + time.sleep(10) + sys.exit(1) diff --git a/acre/core.py b/acre/core.py index 661d52e..a0fc33e 100644 --- a/acre/core.py +++ b/acre/core.py @@ -3,6 +3,8 @@ import re import os import platform +import subprocess +import sys from . import lib @@ -12,6 +14,8 @@ logging.basicConfig() log = logging.getLogger() +re_url = re.compile(r"^\w+://") + class CycleError(ValueError): """A cyclic dependency in dynamic environment""" @@ -23,11 +27,11 @@ class DynamicKeyClashError(ValueError): pass -def compute(env, - dynamic_keys=True, - allow_cycle=False, - allow_key_clash=False, - cleanup=True): +def build(env, + dynamic_keys=True, + allow_cycle=False, + allow_key_clash=False, + cleanup=True): """Compute the result from recursive dynamic environment. Note: Keys that are not present in the data will remain unformatted as the @@ -110,7 +114,7 @@ def compute(env, return env -def parse(env, platform_name=None): +def prepare(env, platform_name=None): """Parse environment for platform-specific values Args: @@ -126,6 +130,14 @@ def parse(env, platform_name=None): platform_name = platform_name or PLATFORM + lookup = {"windows": ["/", "\\"], + "linux": ["\\", "/"], + "darwin": ["\\", "/"]} + + translate = lookup.get(platform_name, None) + if translate is None: + raise KeyError("Given platform name `%s` is not supported" % platform) + result = {} for variable, value in env.items(): @@ -140,15 +152,22 @@ def parse(env, platform_name=None): if isinstance(value, (list, tuple)): value = ";".join(value) + # Replace the separator to match the given platform's separator + # Skip any value which is a url; ://
+ if not re_url.match(value): + value = value.replace(translate[0], translate[1]) + result[variable] = value return result -def append(env, env_b): - """Append paths of environment b into environment""" - # todo: should this be refactored to "join" or "extend" - # todo: this function name might also be confusing with "merge" +def join(env, env_b): + """Append paths of environment b into environment + + Returns: + env (dict) + """ env = env.copy() for variable, value in env_b.items(): for path in value.split(";"): @@ -160,17 +179,17 @@ def append(env, env_b): return env -def get_tools(tools, platform_name=None): +def discover(tools, platform_name=None): """Return combined environment for the given set of tools. - This will find merge all the required environment variables of the input - tools into a single dictionary. Then it will do a recursive format to + This will find and merge all the required environment variables of the + input tools into a single dictionary. Then it will do a recursive format to format all dynamic keys and values using the same dictionary. (So that tool X can rely on variables of tool Y). Examples: - get_environment(["maya2018", "yeti2.01", "mtoa2018"]) - get_environment(["global", "fusion9", "ofxplugins"]) + get_tools(["maya2018", "yeti2.01", "mtoa2018"]) + get_tools(["global", "fusion9", "ofxplugins"]) Args: tools (list): List of tool names. @@ -201,25 +220,25 @@ def get_tools(tools, platform_name=None): environment = dict() for tool_path in tool_paths: - # Load tool + # Load tool environment try: with open(tool_path, "r") as f: tool_env = json.load(f) log.debug('Read tool successfully: {}'.format(tool_path)) except IOError: - log.debug( + log.error( 'Unable to find the environment file: "{}"'.format(tool_path) ) continue except ValueError as e: - log.debug( + log.error( 'Unable to read the environment file: "{0}", due to:' '\n{1}'.format(tool_path, e) ) continue - tool_env = parse(tool_env, platform_name=platform_name) - environment = append(environment, tool_env) + tool_env = prepare(tool_env, platform_name=platform_name) + environment = join(environment, tool_env) return environment @@ -249,3 +268,93 @@ def merge(env, current_env): return result + +def locate(program, env): + """Locate `program` in PATH + + Ensure `PATHEXT` is declared in the environment if you want to alter the + priority of the system extensions: + + Example : ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC" + + Arguments: + program (str): Name of program, e.g. "python" + env (dict): an environment dictionary + + """ + + def is_exe(fpath): + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + return True + return False + + paths = env["PATH"].split(os.pathsep) + extensions = env.get("PATHEXT", os.getenv("PATHEXT", "")) + + for path in paths: + for ext in extensions.split(os.pathsep): + fname = program + ext.lower() + abspath = os.path.join(path.strip('"'), fname) + if is_exe(abspath): + return abspath + + return None + + +def launch(executable, args=None, environment=None, cwd=None): + """Launch a new subprocess of `args` + + Arguments: + executable (str): Relative or absolute path to executable + args (list): Command passed to `subprocess.Popen` + environment (dict, optional): Custom environment passed + to Popen instance. + cwd (str): the current working directory + + Returns: + Popen instance of newly spawned process + + Exceptions: + OSError on internal error + ValueError on `executable` not found + + """ + + CREATE_NO_WINDOW = 0x08000000 + CREATE_NEW_CONSOLE = 0x00000010 + IS_WIN32 = sys.platform == "win32" + PY2 = sys.version_info[0] == 2 + + abspath = executable + + env = (environment or os.environ) + + if PY2: + # Protect against unicode, and other unsupported + # types amongst environment variables + enc = sys.getfilesystemencoding() + env = {k.encode(enc): v.encode(enc) for k, v in env.items()} + + kwargs = dict( + args=[abspath] + args or list(), + env=env, + cwd=cwd, + + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + + # Output `str` through stdout on Python 2 and 3 + universal_newlines=True, + ) + + if env.get("CREATE_NEW_CONSOLE"): + kwargs["creationflags"] = CREATE_NEW_CONSOLE + kwargs.pop("stdout") + kwargs.pop("stderr") + else: + if IS_WIN32: + kwargs["creationflags"] = CREATE_NO_WINDOW + + popen = subprocess.Popen(**kwargs) + + return popen diff --git a/acre/lib.py b/acre/lib.py index 09b3062..ecd9172 100644 --- a/acre/lib.py +++ b/acre/lib.py @@ -13,6 +13,12 @@ def uniqify_ordered(seq): See: https://stackoverflow.com/questions/480214/how-do-you-remove- duplicates-from-a-list-in-whilst-preserving-order + Args: + seq(list): list of values + + Returns: + list + """ seen = set() seen_add = seen.add @@ -46,7 +52,14 @@ def __missing__(self, key): def topological_sort(dependency_pairs): - """Sort values subject to dependency constraints""" + """Sort values subject to dependency constraints + + Args: + dependency_pairs(list): list of pairs, [a, b] + + Returns: + namedtuple + """ num_heads = defaultdict(int) # num arrows pointing in tails = defaultdict(list) # list of arrows going out heads = [] # unique list of heads in order first seen @@ -64,12 +77,24 @@ def topological_sort(dependency_pairs): num_heads[t] -= 1 if not num_heads[t]: ordered.append(t) + cyclic = [n for n, heads in num_heads.items() if heads] + return Results(ordered, cyclic) def append_path(self, key, path): - """Append *path* to *key* in *self*.""" + """Append *path* to *key* in *self*. + + Args: + self (dict): environment dictionary + key (str): environment variable name + path (str): path + + Returns: + None + + """ try: if path not in self[key]: self[key] = os.pathsep.join([self[key], str(path)]) diff --git a/tests/test_dynamic_environments.py b/tests/test_dynamic_environments.py index d039d11..7c0724d 100644 --- a/tests/test_dynamic_environments.py +++ b/tests/test_dynamic_environments.py @@ -17,15 +17,15 @@ def test_parse_platform(self): "B": "universal" } - result = acre.parse(data, platform_name="darwin") + result = acre.prepare(data, platform_name="darwin") self.assertEqual(result["A"], data["A"]["darwin"]) self.assertEqual(result["B"], data["B"]) - result = acre.parse(data, platform_name="windows") + result = acre.prepare(data, platform_name="windows") self.assertEqual(result["A"], data["A"]["windows"]) self.assertEqual(result["B"], data["B"]) - result = acre.parse(data, platform_name="linux") + result = acre.prepare(data, platform_name="linux") self.assertEqual(result["A"], data["A"]["linux"]) self.assertEqual(result["B"], data["B"]) @@ -44,7 +44,7 @@ def test_nesting_deep(self): "I": "deep_{H}" } - result = acre.compute(data) + result = acre.build(data) self.assertEqual(result, { "A": "bla", @@ -66,11 +66,11 @@ def test_cycle(self): } with self.assertRaises(acre.CycleError): - acre.compute(data, allow_cycle=False) + acre.build(data, allow_cycle=False) # If we compute the cycle the result is unknown, it can be either {Y} # or {X} for both values so we just check whether are equal - result = acre.compute(data, allow_cycle=True) + result = acre.build(data, allow_cycle=True) self.assertEqual(result["X"], result["Y"]) def test_dynamic_keys(self): @@ -82,7 +82,7 @@ def test_dynamic_keys(self): "{B}": "this is C" } - env = acre.compute(data) + env = acre.build(data) self.assertEqual(env, { "A": "D", @@ -99,10 +99,10 @@ def test_dynamic_keys_clash_cycle(self): } with self.assertRaises(acre.DynamicKeyClashError): - acre.compute(data, allow_key_clash=False) + acre.build(data, allow_key_clash=False) # Allow to pass (even if unpredictable result) - acre.compute(data, allow_key_clash=True) + acre.build(data, allow_key_clash=True) def test_dynamic_keys_clash(self): """Dynamic key clash captured correctly""" @@ -113,10 +113,10 @@ def test_dynamic_keys_clash(self): } with self.assertRaises(acre.DynamicKeyClashError): - acre.compute(data, allow_key_clash=False) + acre.build(data, allow_key_clash=False) # Allow to pass (even if unpredictable result) - acre.compute(data, allow_key_clash=True) + acre.build(data, allow_key_clash=True) def test_compute_preserve_reference_to_self(self): """acre.compute() does not format key references to itself""" @@ -125,7 +125,7 @@ def test_compute_preserve_reference_to_self(self): "PATH": "{PATH}", "PYTHONPATH": "x;y/{PYTHONPATH}" } - data = acre.compute(data) + data = acre.build(data) self.assertEqual(data, { "PATH": "{PATH}", "PYTHONPATH": "x;y/{PYTHONPATH}" @@ -148,7 +148,7 @@ def test_append(self): _data_a = data_a.copy() _data_b = data_b.copy() - data = acre.append(data_a, data_b) + data = acre.join(data_a, data_b) self.assertEqual(data, { "A": "A;A2", diff --git a/tests/test_tools.py b/tests/test_tools.py index d8a4f93..6d755a7 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -16,7 +16,7 @@ def test_get_tools(self): """Get tools works""" self.setup_sample("sample1") - env = acre.get_tools(["global"]) + env = acre.discover(["global"]) self.assertEqual(env, { "PIPELINE": "P:/pipeline/dev2_1" @@ -27,14 +27,14 @@ def test_get_with_platform(self): self.setup_sample("sample1") - env = acre.get_tools(["maya_2018"], platform_name="darwin") + env = acre.discover(["maya_2018"], platform_name="darwin") self.assertEqual(env["PATH"], "{MAYA_LOCATION}/bin") # Test Mac only path self.assertTrue("DYLD_LIBRARY_PATH" in env) self.assertEqual(env["DYLD_LIBRARY_PATH"], "{MAYA_LOCATION}/MacOS") - env = acre.get_tools(["maya_2018"], platform_name="windows") + env = acre.discover(["maya_2018"], platform_name="windows") self.assertEqual(env["PATH"], ";".join([ "{MAYA_LOCATION}/bin", "C:/Program Files/Common Files/Autodesk Shared/", @@ -47,9 +47,9 @@ def test_get_with_platform(self): def test_compute_tools(self): self.setup_sample("sample1") - env = acre.get_tools(["maya_2018"], platform_name="windows") + env = acre.discover(["maya_2018"], platform_name="windows") - env = acre.compute(env) + env = acre.build(env) self.assertEqual(env["MAYA_LOCATION"], "C:/Program Files/Autodesk/Maya2018")