From aacb677932b8612bbf29dcf56bdd59b04abc7d7a Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Thu, 17 Sep 2020 09:00:53 +0200 Subject: [PATCH 1/3] Minor fix: unmatched parenthesis in comment --- pythontex/pythontex3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythontex/pythontex3.py b/pythontex/pythontex3.py index bcf81ab..d7195a4 100644 --- a/pythontex/pythontex3.py +++ b/pythontex/pythontex3.py @@ -1299,7 +1299,7 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): engine_dict[family].linenumbers, engine_dict[family].lookbehind, keeptemps, hashdependencies, - pygments_settings]))''' + pygments_settings)''' tasks.append(pool.apply_async(run_code, [encoding, outputdir, workingdir, cc_dict_begin[family], From f6ca8c602c04cb91d92f85ff6e31c32ad497e0f9 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Thu, 17 Sep 2020 09:30:51 +0200 Subject: [PATCH 2/3] Simplify code a bit in do_multiprocessing --- pythontex/pythontex3.py | 46 +++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/pythontex/pythontex3.py b/pythontex/pythontex3.py index d7195a4..6cceab9 100644 --- a/pythontex/pythontex3.py +++ b/pythontex/pythontex3.py @@ -1324,35 +1324,23 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): for key in cons_dict: family = key.split('#')[0] if engine_dict[family].language.startswith('python'): - if family in pygments_settings: - # Uncomment the following for debugging - '''python_console(jobname, encoding, outputdir, workingdir, - fvextfile, pygments_settings[family], - cc_dict_begin[family], cons_dict[key], - cc_dict_end[family], engine_dict[family].startup, - engine_dict[family].banner, - engine_dict[family].filename)''' - tasks.append(pool.apply_async(python_console, [jobname, encoding, - outputdir, workingdir, - fvextfile, - pygments_settings[family], - cc_dict_begin[family], - cons_dict[key], - cc_dict_end[family], - engine_dict[family].startup, - engine_dict[family].banner, - engine_dict[family].filename])) - else: - tasks.append(pool.apply_async(python_console, [jobname, encoding, - outputdir, workingdir, - fvextfile, - None, - cc_dict_begin[family], - cons_dict[key], - cc_dict_end[family], - engine_dict[family].startup, - engine_dict[family].banner, - engine_dict[family].filename])) + # Uncomment the following for debugging + '''python_console(jobname, encoding, outputdir, workingdir, + fvextfile, pygments_settings[family], + cc_dict_begin[family], cons_dict[key], + cc_dict_end[family], engine_dict[family].startup, + engine_dict[family].banner, + engine_dict[family].filename)''' + tasks.append(pool.apply_async(python_console, [jobname, encoding, + outputdir, workingdir, + fvextfile, + pygments_settings[family] if family in pygments_settings else None, + cc_dict_begin[family], + cons_dict[key], + cc_dict_end[family], + engine_dict[family].startup, + engine_dict[family].banner, + engine_dict[family].filename])) else: print('* PythonTeX error') print(' Currently, non-Python consoles are not supported') From 7ff120bbf065f1fcfdaca96bb437f095ae96018e Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Thu, 17 Sep 2020 09:25:02 +0200 Subject: [PATCH 3/3] Support platforms where multiprocessing does not work Check if multiprocessing is supported and else set multiprocessing=False. When later creating list of tasks to run we then choose to either run pool.apply_async(function), or to just eval(function). Import test for multiprocessing is taken from https://github.com/pipxproject/pipx/blob/4870ba6bd0164448c0f6b34dd663cba33b396287/src/pipx/commands/list_packages.py#L12. The full error message from trying to run pythontex on android looks like: ``` This is PythonTeX 0.17 Traceback (most recent call last): File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/synchronize.py", line 28, in from _multiprocessing import SemLock, sem_unlink ImportError: cannot import name 'SemLock' from '_multiprocessing' (/data/data/com.termux/files/usr/lib/python3.8/lib-dynload/_multiprocessing.cpython-38.so) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/data/data/com.termux/files/usr/bin/texlive/pythontex", line 62, in pythontex.main() File "/data/data/com.termux/files/usr/share/texlive/texmf-dist/scripts/pythontex/pythontex3.py", line 2811, in main do_multiprocessing(data, temp_data, old_data, engine_dict) File "/data/data/com.termux/files/usr/share/texlive/texmf-dist/scripts/pythontex/pythontex3.py", line 1269, in do_multiprocessing pool = multiprocessing.Pool(jobs) File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 119, in Pool return Pool(processes, initializer, initargs, maxtasksperchild, File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/pool.py", line 191, in __init__ self._setup_queues() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/pool.py", line 343, in _setup_queues self._inqueue = self._ctx.SimpleQueue() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 113, in SimpleQueue return SimpleQueue(ctx=self.get_context()) File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/queues.py", line 336, in __init__ self._rlock = ctx.Lock() File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/context.py", line 67, in Lock from .synchronize import Lock File "/data/data/com.termux/files/usr/lib/python3.8/multiprocessing/synchronize.py", line 30, in raise ImportError("This platform lacks a functioning sem_open" + ImportError: This platform lacks a functioning sem_open implementation, therefore, the required synchronization primitives needed will not function, see issue 3770. ``` --- pythontex/pythontex3.py | 136 ++++++++++++++++++++++++---------------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/pythontex/pythontex3.py b/pythontex/pythontex3.py index 6cceab9..b2b5b17 100644 --- a/pythontex/pythontex3.py +++ b/pythontex/pythontex3.py @@ -57,13 +57,19 @@ from collections import defaultdict, OrderedDict, namedtuple from re import match, sub, search import subprocess -import multiprocessing from pygments.styles import get_all_styles from pythontex_engines import * import textwrap import platform import itertools +try: + import multiprocessing.synchronize # noqa: F401 + from multiprocessing import Pool +except ImportError: + multiprocessing = False + + if sys.version_info[0] == 2: try: import cPickle as pickle @@ -192,8 +198,9 @@ def process_argv(data, temp_data): temp_data['hashdependencies'] = False if args.jobs is None: try: - jobs = multiprocessing.cpu_count() + jobs = os.cpu_count() except NotImplementedError: + # Is this exception needed with os.cpu_count? jobs = 1 temp_data['jobs'] = jobs else: @@ -1145,6 +1152,13 @@ def negative_one(): +def eval_task(fun_name, arg_list): + ''' + Some platforms, like android, does not support multiprocessing. For such + platforms we fall back to this function when running tasks. + ''' + arg_tuple = tuple(arg_list) + return eval(fun_name)(*arg_tuple) def do_multiprocessing(data, temp_data, old_data, engine_dict): jobname = data['jobname'] @@ -1269,7 +1283,14 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): # concurrent processes to a user-specified value for jobs. If the user # has not specified a value, then it will be None, and # multiprocessing.Pool() will use cpu_count(). - pool = multiprocessing.Pool(jobs) + if multiprocessing: + pool = Pool(jobs) + # Add task to list of tasks to run asynchronously, from function + # name and list of args. + # globals()[fun] converts string with function name into function handle + async_or_eval = lambda fun, args: pool.apply_async(globals()[fun], args) + else: + async_or_eval = eval_task tasks = [] # If verbose, print a list of processes @@ -1284,39 +1305,39 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): family = key.split('#')[0] # Uncomment the following for debugging, and comment out what follows '''run_code(encoding, outputdir, - workingdir, - cc_dict_begin[family], - code_dict[key], - cc_dict_end[family], - engine_dict[family].language, - engine_dict[family].commands, - engine_dict[family].created, - engine_dict[family].extension, - makestderr, stderrfilename, - code_index_dict[key], - engine_dict[family].errors, - engine_dict[family].warnings, - engine_dict[family].linenumbers, - engine_dict[family].lookbehind, - keeptemps, hashdependencies, - pygments_settings)''' - tasks.append(pool.apply_async(run_code, [encoding, outputdir, - workingdir, - cc_dict_begin[family], - code_dict[key], - cc_dict_end[family], - engine_dict[family].language, - engine_dict[family].commands, - engine_dict[family].created, - engine_dict[family].extension, - makestderr, stderrfilename, - code_index_dict[key], - engine_dict[family].errors, - engine_dict[family].warnings, - engine_dict[family].linenumbers, - engine_dict[family].lookbehind, - keeptemps, hashdependencies, - pygments_settings])) + workingdir, + cc_dict_begin[family], + code_dict[key], + cc_dict_end[family], + engine_dict[family].language, + engine_dict[family].commands, + engine_dict[family].created, + engine_dict[family].extension, + makestderr, stderrfilename, + code_index_dict[key], + engine_dict[family].errors, + engine_dict[family].warnings, + engine_dict[family].linenumbers, + engine_dict[family].lookbehind, + keeptemps, hashdependencies, + pygments_settings)''' + tasks.append(async_or_eval('run_code', [encoding, outputdir, + workingdir, + cc_dict_begin[family], + code_dict[key], + cc_dict_end[family], + engine_dict[family].language, + engine_dict[family].commands, + engine_dict[family].created, + engine_dict[family].extension, + makestderr, stderrfilename, + code_index_dict[key], + engine_dict[family].errors, + engine_dict[family].warnings, + engine_dict[family].linenumbers, + engine_dict[family].lookbehind, + keeptemps, hashdependencies, + pygments_settings])) if verbose: print(' - Code process ' + key.replace('#', ':')) @@ -1331,16 +1352,16 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): cc_dict_end[family], engine_dict[family].startup, engine_dict[family].banner, engine_dict[family].filename)''' - tasks.append(pool.apply_async(python_console, [jobname, encoding, - outputdir, workingdir, - fvextfile, - pygments_settings[family] if family in pygments_settings else None, - cc_dict_begin[family], - cons_dict[key], - cc_dict_end[family], - engine_dict[family].startup, - engine_dict[family].banner, - engine_dict[family].filename])) + tasks.append(async_or_eval('python_console', [jobname, encoding, + outputdir, workingdir, + fvextfile, + pygments_settings[family] if family in pygments_settings else None, + cc_dict_begin[family], + cons_dict[key], + cc_dict_end[family], + engine_dict[family].startup, + engine_dict[family].banner, + engine_dict[family].filename])) else: print('* PythonTeX error') print(' Currently, non-Python consoles are not supported') @@ -1353,18 +1374,19 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): # Uncomment the following for debugging # do_pygments(encoding, outputdir, fvextfile, pygments_list, # pygments_settings, typeset_cache, hashdependencies) - tasks.append(pool.apply_async(do_pygments, [encoding, outputdir, - fvextfile, - pygments_list, - pygments_settings, - typeset_cache, - hashdependencies])) + tasks.append(async_or_eval('do_pygments', [encoding, outputdir, + fvextfile, + pygments_list, + pygments_settings, + typeset_cache, + hashdependencies])) if verbose: print(' - Pygments process') # Execute the processes - pool.close() - pool.join() + if multiprocessing: + pool.close() + pool.join() # Get the outputs of processes # Get the files and macros created. Get the number of errors and warnings @@ -1375,7 +1397,11 @@ def do_multiprocessing(data, temp_data, old_data, engine_dict): new_files = False messages = [] for task in tasks: - result = task.get() + if multiprocessing: + result = task.get() + else: + result = task + if result['process'] == 'code': key = result['key'] files[key].extend(result['files'])