Law commands failing when Rucio is down #115
Description
Occasional crash when rucio is down. Konstantin suggests "local cache of lfns->pfns to smooth temporary glitches"
(flaf_env) [daebi@lxplus948 FLAF_dev]$ law run AnalysisCacheTask --period Run3_2022 --version shared --tasks-per-job 5 --remove-output 0,a,y
remove task output with max_depth 0
task will run after output removal
selected all mode
2025-03-20 12:51:40,524 WARNING Waiting 0.25s due to reason: server returned 503
2025-03-20 12:51:40,778 WARNING Waiting 0.5s due to reason: server returned 503
2025-03-20 12:51:41,282 WARNING Waiting 1.0s due to reason: server returned 503
ERROR: luigi-interface - Uncaught exception in luigi
Traceback (most recent call last):
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/luigi/retcodes.py", line 75, in run_with_retcodes
worker = luigi.interface._run(argv).worker
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/luigi/interface.py", line 217, in _run
return _schedule_and_run([cp.get_task_obj()], worker_scheduler_factory)
^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/luigi/cmdline_parser.py", line 114, in get_task_obj
return self._get_task_cls()(**self._get_task_kwargs())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/base.py", line 376, in __call__
skip_abort = getattr(inst, "_" + param)(value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/base.py", line 650, in _remove_output
return remove_task_output(self, *args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/interactive.py", line 335, in remove_task_output
for dep, next_deps, depth, is_last in task.walk_deps(
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/base.py", line 307, in walk_deps
deps = flatten(task.requires())
^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/remote.py", line 511, in requires
reqs = super(BaseRemoteWorkflowProxy, self).requires()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 123, in requires
workflow_reqs = self.task.workflow_requires()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/Analysis/tasks.py", line 398, in workflow_requires
for br_idx, _ in self.branch_map.items()
^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1020, in __getattribute__
return get_proxy_attribute(self, attr, proxy=proxy, super_cls=Task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/proxy.py", line 140, in get_proxy_attribute
return super_cls.__getattribute__(task, attr)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1231, in branch_map
return self.get_branch_map()
^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1212, in get_branch_map
branch_map = self.create_branch_map(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/Analysis/tasks.py", line 407, in create_branch_map
anaProd_branch_map = AnaTupleTask.req(self, branch=-1, branches=()).branch_map
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1020, in __getattribute__
return get_proxy_attribute(self, attr, proxy=proxy, super_cls=Task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/proxy.py", line 140, in get_proxy_attribute
return super_cls.__getattribute__(task, attr)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1231, in branch_map
return self.get_branch_map()
^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1212, in get_branch_map
branch_map = self.create_branch_map(*args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/AnaProd/tasks.py", line 145, in create_branch_map
self.remote_target(input_file, fs=self.fs_nanoAOD))
^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/workflow/base.py", line 1020, in __getattribute__
return get_proxy_attribute(self, attr, proxy=proxy, super_cls=Task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/k/kandroso/public/flaf_env_2024_08/lib/python3.11/site-packages/law/task/proxy.py", line 140, in get_proxy_attribute
return super_cls.__getattribute__(task, attr)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/run_tools/law_customizations.py", line 63, in fs_nanoAOD
return self.setup.get_fs('nanoAOD')
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/Common/Setup.py", line 160, in get_fs
self.fs_dict[fs_name] = WLCGFileSystem(self.global_params[full_fs_name])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/RunKit/law_wlcg.py", line 24, in __init__
base_pfns = [path_to_pfn(b) for b in base]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/RunKit/law_wlcg.py", line 24, in <listcomp>
base_pfns = [path_to_pfn(b) for b in base]
^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/RunKit/grid_tools.py", line 272, in path_to_pfn
pfn = lfn_to_pfn(server, lfn)
^^^^^^^^^^^^^^^^^^^^^^^
File "/afs/cern.ch/work/d/daebi/diHiggs/FLAF_dev/RunKit/grid_tools.py", line 266, in lfn_to_pfn
result = client.lfns2pfns(server, [ key ])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/cvmfs/cms.cern.ch/rucio/x86_64/rhel9/py3/current/lib/python3.9/site-packages/rucio/client/rseclient.py", line 329, in lfns2pfns
raise exc_cls(exc_msg)
rucio.common.exception.RucioException: An unknown exception occurred.
Details: no error information passed (http status code: 503)