From 32eacb46225a5d8952088c1b8d5705dd0a252157 Mon Sep 17 00:00:00 2001 From: Assaf Giladi Date: Tue, 19 Nov 2019 18:06:06 +0200 Subject: [PATCH 1/7] talker.agent: run agent with python3 --- .travis.yml | 2 +- talker_agent/Dockerfile | 2 +- talker_agent/talker.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6e71156..9632b79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: env: - TEST_SUITE="tests/uts/test_client.py" - stage: agent unit tests - python: 2.7 + python: 3.5 env: - TEST_SUITE="tests/uts/test_agent.py" - stage: integration tests diff --git a/talker_agent/Dockerfile b/talker_agent/Dockerfile index a93aa7e..027048c 100644 --- a/talker_agent/Dockerfile +++ b/talker_agent/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7-slim-buster +FROM python:3.5-slim-buster #args from docker-compose.yaml ARG TALKER_AGENT_VERSION diff --git a/talker_agent/talker.py b/talker_agent/talker.py index 580e1da..76eefdd 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/local/bin/python3 """ From 9fce18adb761dad8d013a06f88aa2808fac752dd Mon Sep 17 00:00:00 2001 From: Assaf Giladi Date: Wed, 20 Nov 2019 12:17:19 +0200 Subject: [PATCH 2/7] talker.agent: encode stderr exception --- talker_agent/talker.py | 2 +- tests/integration/test_sanity.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/talker_agent/talker.py b/talker_agent/talker.py index 76eefdd..189376b 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -304,7 +304,7 @@ def start(self): time.sleep(random.random() * 2) continue - self.stderr.chunks.append(e.strerror) + self.stderr.chunks.append(e.strerror.encode('utf-8')) self.set_result(e.errno) self.finalize() return diff --git a/tests/integration/test_sanity.py b/tests/integration/test_sanity.py index 112939d..7378c2c 100644 --- a/tests/integration/test_sanity.py +++ b/tests/integration/test_sanity.py @@ -1,7 +1,7 @@ import unittest from talker_agent.talker import Config -from talker.errors import CommandAbortedByOverflow +from talker.errors import CommandAbortedByOverflow, CommandExecutionError from tests.utils import get_talker_client, get_retcode, get_stdout @@ -38,3 +38,8 @@ def test_max_output_per_channel_set(self): if expected_ret == 0: res = get_stdout(self.client.redis, cmd.job_id) self.assertEqual(res, val) + + def test_bad_command(self): + cmd = self.client.run(self.host_id, 'foo') + with self.assertRaises(CommandExecutionError): + cmd.result() From e8ffbba3c9b307f194c6a4b51281be5c8598addc Mon Sep 17 00:00:00 2001 From: Assaf Giladi Date: Wed, 20 Nov 2019 12:19:39 +0200 Subject: [PATCH 3/7] changelog: update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b2702..e8d5ba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- upgrade agent to run with python3 ## [1.8.6] - 2019-11-05 ### Added From a8f9eb480739776d2b41303ee70707973db392c1 Mon Sep 17 00:00:00 2001 From: Doron Cohen Date: Mon, 28 Oct 2019 17:34:55 +0200 Subject: [PATCH 4/7] agent: remove weka related code / comments --- talker_agent/talker.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/talker_agent/talker.py b/talker_agent/talker.py index 189376b..1c7f5f5 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -1,23 +1,5 @@ #!/usr/local/bin/python3 - -""" -Talker Agent (Server-Side) -========================== - -* Important: - - keep this free of dependencies (there's only a redis dependency) - - keep this compatible with python2.6+ (no dict comprehension) - -* Packaging: - - update the 'TALKER' version in version_info - - ./teka pack talker - -* Testing: - - See ./wepy/devops/talker.py (client-side) - -""" - import fcntl import json import logging @@ -313,13 +295,7 @@ def start(self): self.job_fn = "%s/job.%s.%s" % (JOBS_DIR, self.job_id, self.popen.pid) with open(self.job_fn, "w") as f: - try: - f.write(repr(self.cmd)) - except IOError: - # to help with WEKAPP-74054 - os.system("df") - os.system("df -i") - raise + f.write(repr(self.cmd)) self.agent.current_processes[self.job_id] = self for channel in self.channels: @@ -940,10 +916,6 @@ def main(*args): config = Config() set_logging_to_file(config.parser.get('logging', 'logpath')) - # to help with WEKAPP-74054 - os.system("df") - os.system("df -i") - open("/var/run/talker.pid", "w").write(str(os.getpid())) atexit.register(os.unlink, "/var/run/talker.pid") From a455c840b745abe5a5cc2591fcbd2c87ac48547d Mon Sep 17 00:00:00 2001 From: Doron Cohen Date: Mon, 28 Oct 2019 17:36:52 +0200 Subject: [PATCH 5/7] agent: remove old ut code --- talker_agent/talker.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/talker_agent/talker.py b/talker_agent/talker.py index 1c7f5f5..df05625 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -25,6 +25,8 @@ except: # python 2.7 from ConfigParser import ConfigParser +import redis + PY3 = sys.version_info[0] == 3 @@ -817,7 +819,6 @@ def setup(self): health_check_interval = config.parser.getfloat('redis', 'health_check_interval') logger.info("Connecting to redis %s:%s", host, port) - import redis # deferring so that importing talker (for ut) doesn't immediately fail if package not available self.redis = redis.StrictRedis( host=host, port=port, db=0, password=password, socket_timeout=socket_timeout, socket_connect_timeout=socket_connect_timeout, @@ -947,7 +948,4 @@ def main(*args): if __name__ == '__main__': args = sys.argv[1:] - if "--ut" in args: - print("Talker don't need no UT") - else: - sys.exit(main(*args)) + sys.exit(main(*args)) From 06cff98296557ee08d20977665fa5b3aed769bdb Mon Sep 17 00:00:00 2001 From: Doron Cohen Date: Mon, 28 Oct 2019 17:39:22 +0200 Subject: [PATCH 6/7] agent: remove some python 2 compat code --- talker_agent/talker.py | 45 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 37 deletions(-) diff --git a/talker_agent/talker.py b/talker_agent/talker.py index df05625..24dfc57 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -16,50 +16,21 @@ import glob import atexit import random -from textwrap import dedent from contextlib import contextmanager from logging import getLogger from logging.handlers import RotatingFileHandler -try: - from configparser import ConfigParser -except: # python 2.7 - from ConfigParser import ConfigParser +from configparser import ConfigParser import redis -PY3 = sys.version_info[0] == 3 - -# =========================================================================================== -# Define a python2/3 compatible 'reraise' function for re-raising exceptions properly -# Since the syntax is different and would not compile between versions, we need to using 'exec' - -if PY3: - def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value -else: - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") - - exec_(dedent(""" - def reraise(tp, value, tb=None): - raise tp, value, tb - """)) - -# =========================================================================================== +def reraise(tp, value, tb=None): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + CONFIG_FILENAME = '/root/talker/config.ini' REBOOT_FILENAME = '/root/talker/reboot.id' From 309629f3c4fc96f5641059eb9c3f0fce20eb37bd Mon Sep 17 00:00:00 2001 From: Assaf Giladi Date: Wed, 20 Nov 2019 17:59:54 +0200 Subject: [PATCH 7/7] agent: handle redis TimeoutError --- talker_agent/talker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/talker_agent/talker.py b/talker_agent/talker.py index 24dfc57..91ed2a3 100644 --- a/talker_agent/talker.py +++ b/talker_agent/talker.py @@ -22,6 +22,7 @@ from configparser import ConfigParser import redis +from redis import TimeoutError def reraise(tp, value, tb=None): @@ -534,7 +535,11 @@ def fetch_new_jobs(self): jobs_key = 'commands-%s' % self.host_id while not self.stop_fetching.is_set(): new_jobs = [] - ret = self.redis.blpop([jobs_key], timeout=1) + try: + ret = self.redis.blpop([jobs_key], timeout=1) + except TimeoutError: + logger.exception("Failed fetching new jobs") + ret = None if not ret: now = time.time() self.scrub_seen_jobs(now=now)