diff --git a/scripts/sd-launch-pbs b/scripts/sd-launch-pbs index 0b38733..872b202 100644 --- a/scripts/sd-launch-pbs +++ b/scripts/sd-launch-pbs @@ -7,15 +7,16 @@ import logging from smartdispatch import launch_jobs from smartdispatch import utils + +logger = logging.getLogger() + + LOGS_FOLDERNAME = "SMART_DISPATCH_LOGS" CLUSTER_NAME = utils.detect_cluster() LAUNCHER = utils.get_launcher(CLUSTER_NAME) def main(): - # Necessary if we want 'logging.info' to appear in stderr. - logging.root.setLevel(logging.INFO) - args = parse_arguments() launch_jobs(LAUNCHER if args.launcher is None else args.launcher, [args.pbs], CLUSTER_NAME, args.path_job) @@ -27,8 +28,21 @@ def parse_arguments(): parser.add_argument('pbs', type=str, help='PBS filename to launch.') parser.add_argument('path_job', type=str, help='Path to the job folder.') + parser.add_argument( + '-v', '--verbose', action='count', default=0, + help="Print informations about the process.\n" + " -v: INFO\n" + " -vv: DEBUG") + args = parser.parse_args() + if args.verbose == 0: + logging.basicConfig(level=logging.WARNING) + elif args.verbose == 1: + logging.basicConfig(level=logging.INFO) + elif args.verbose >= 2: + logging.basicConfig(level=logging.DEBUG) + return args diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 86904fa..4c9e0bd 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -1,9 +1,10 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- +import argparse +import logging import os import sys -import argparse import time as t from os.path import join as pjoin from textwrap import dedent @@ -16,9 +17,15 @@ from smartdispatch import get_available_queues from smartdispatch import launch_jobs from smartdispatch import utils -import logging import smartdispatch + +logger = logging.getLogger() + +VERBOSE_WARNING = 0 +VERBOSE_INFO = 1 +VERBOSE_DEBUG = 2 + LOGS_FOLDERNAME = "SMART_DISPATCH_LOGS" CLUSTER_NAME = utils.detect_cluster() AVAILABLE_QUEUES = get_available_queues(CLUSTER_NAME) @@ -29,27 +36,57 @@ TIMEOUT_EXIT_CODE = 124 AUTORESUME_TRIGGER_AFTER = '$(($PBS_WALLTIME - 60))' # By default, 60s before the maximum walltime. AUTORESUME_WORKER_CALL_PREFIX = 'timeout -s TERM {trigger_after} '.format(trigger_after=AUTORESUME_TRIGGER_AFTER) AUTORESUME_WORKER_CALL_SUFFIX = ' WORKER_PIDS+=" $!"' -AUTORESUME_PROLOG = 'WORKER_PIDS=""' +AUTORESUME_PROLOG = """\ +WORKER_PIDS="" +VERBOSE={verbose} +""" AUTORESUME_EPILOG = """\ NEED_TO_RESUME=false +if [ $VERBOSE -ge {debug} ]; then + echo "DEBUG: NEED_TO_RESUME=$NEED_TO_RESUME" + echo "DEBUG: WORKER_PIDS=$WORKER_PIDS" +fi for WORKER_PID in $WORKER_PIDS; do + if [ $VERBOSE -ge {debug} ]; then + echo "DEBUG: WORKER_PID=$WORKER_PID" + fi wait "$WORKER_PID" RETURN_CODE=$? + if [ $VERBOSE -ge {debug} ]; then + echo "DEBUG: RETURN_CODE is $RETURN_CODE while timeout_exit_code is {timeout_exit_code}" + fi if [ $RETURN_CODE -eq {timeout_exit_code} ]; then NEED_TO_RESUME=true fi + if [ $VERBOSE -ge {debug} ]; then + echo "DEBUG: NEED_TO_RESUME=$NEED_TO_RESUME" + fi done +if [ $VERBOSE -ge {debug} ]; then + echo "DEBUG: NEED_TO_RESUME=$NEED_TO_RESUME" +fi if [ "$NEED_TO_RESUME" = true ]; then echo "Autoresuming using: {{launcher}} $PBS_FILENAME" - sd-launch-pbs --launcher {{launcher}} $PBS_FILENAME {{path_job}} + if [ $VERBOSE -ge 0 ] + then + VERBOSE_OPTION="-" + for ((i=0;i= VERBOSE_DEBUG: + logging.basicConfig(level=logging.DEBUG) + return args