add --async_api_timeout to opengrok-sync

allow to propagate the async API timeout to CommandSequence

Author: vladak

tools/src/main/python/opengrok_tools/mirror.py

@@ -121,7 +121,7 @@ def main():
                         help='Set response timeout in seconds '
                              'for RESTful API calls')
     parser.add_argument('--async_api_timeout', type=int, default=300,
-                        help='Set timeout in seconds for asynchronous RESTful API calls')
+                        help='Set timeout in seconds for asynchronous REST API calls')
 
     try:
         args = parser.parse_args()

tools/src/main/python/opengrok_tools/sync.py

@@ -69,7 +69,7 @@ def worker(base):
 
 def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
             uri, numworkers, driveon=False, print_output=False, logger=None,
-            http_headers=None, timeout=None):
+            http_headers=None, timeout=None, api_timeout=None):
     """
     Process the list of directories in parallel.
     :param logger: logger to be used in this function
@@ -86,6 +86,7 @@ def do_sync(loglevel, commands, cleanup, dirs_to_process, ignore_errors,
     :param logger: optional logger
     :param http_headers: optional dictionary of HTTP headers
     :param timeout: optional timeout in seconds for API call response
+    :param api_timeout: optional timeout in seconds for async API call duration
     :return SUCCESS_EXITVAL on success, FAILURE_EXITVAL on error
     """
 
@@ -156,6 +157,8 @@ def main():
     parser.add_argument('--api_timeout', type=int, default=3,
                         help='Set response timeout in seconds'
                         'for RESTful API calls')
+    parser.add_argument('--async_api_timeout', type=int, default=300,
+                        help='Set timeout in seconds for asynchronous REST API calls')
     add_http_headers(parser)
 
     try:
@@ -275,7 +278,8 @@ def main():
                         dirs_to_process,
                         ignore_errors, uri, args.workers,
                         driveon=args.driveon, http_headers=headers,
-                        timeout=args.api_timeout)
+                        timeout=args.api_timeout,
+                        api_timeout=args.async_api_timeout)
         except CommandConfigurationException as exc:
             logger.error("Invalid configuration: {}".format(exc))
             return FAILURE_EXITVAL
@@ -289,7 +293,8 @@ def main():
                                 dirs_to_process,
                                 ignore_errors, uri, args.workers,
                                 driveon=args.driveon, http_headers=headers,
-                                timeout=args.api_timeout)
+                                timeout=args.api_timeout,
+                                api_timeout=args.async_api_timeout)
                 except CommandConfigurationException as exc:
                     logger.error("Invalid configuration: {}".format(exc))
                     return FAILURE_EXITVAL

tools/src/main/python/opengrok_tools/utils/commandsequence.py

@@ -72,7 +72,7 @@ class CommandSequenceBase:
 
     def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None,
                  driveon=False, url=None, env=None, http_headers=None,
-                 api_timeout=None):
+                 api_timeout=None, async_api_timeout=None):
         self.name = name
 
         if commands is None:
@@ -99,6 +99,7 @@ def __init__(self, name, commands, loglevel=logging.INFO, cleanup=None,
         self.env = env
         self.http_headers = http_headers
         self.api_timeout = api_timeout
+        self.async_api_timeout = async_api_timeout
 
         self.url = url
 
@@ -132,7 +133,8 @@ def __init__(self, base):
                          cleanup=base.cleanup, driveon=base.driveon,
                          url=base.url, env=base.env,
                          http_headers=base.http_headers,
-                         api_timeout=base.api_timeout)
+                         api_timeout=base.api_timeout,
+                         async_api_timeout=base.async_api_timeout)
 
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(base.loglevel)
@@ -169,7 +171,9 @@ def run(self):
                     call_rest_api(command.get(CALL_PROPERTY),
                                   {PROJECT_SUBST: self.name,
                                    URL_SUBST: self.url},
-                                  self.http_headers, self.api_timeout)
+                                  self.http_headers,
+                                  self.api_timeout,
+                                  self.async_api_timeout)
                 except RequestException as e:
                     self.logger.error("REST API call {} failed: {}".
                                       format(command, e))
@@ -229,7 +233,9 @@ def run_cleanup(self):
                     call_rest_api(cleanup_cmd.get(CALL_PROPERTY),
                                   {PROJECT_SUBST: self.name,
                                    URL_SUBST: self.url},
-                                  self.http_headers, self.api_timeout)
+                                  self.http_headers,
+                                  self.api_timeout,
+                                  self.async_api_timeout)
                 except RequestException as e:
                     self.logger.error("API call {} failed: {}".
                                       format(cleanup_cmd, e))