From 9602f7c043719d2ea05e5f98e4f09b8e1b0938d1 Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Thu, 14 Jan 2021 22:32:54 +1030 Subject: [PATCH 1/6] Various bugfixes --- bots/cleanup.py | 6 +++--- bots/communication.py | 20 ++++++++++++++------ bots/models.py | 1 + bots/pluglib.py | 2 +- bots/transform.py | 3 ++- 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/bots/cleanup.py b/bots/cleanup.py index 976bb9c..a1e5499 100644 --- a/bots/cleanup.py +++ b/bots/cleanup.py @@ -16,10 +16,10 @@ def cleanup(do_cleanup_parameter,userscript,scriptname): most cleanup functions are by default done only once a day. ''' whencleanup = botsglobal.ini.get('settings','whencleanup','daily') - if botsglobal.ini.getboolean('acceptance','runacceptancetest',False): #no cleanup during acceptance testing - do_full_cleanup = False - elif do_cleanup_parameter: #if explicit indicated via commandline parameter + if do_cleanup_parameter: #if explicit indicated via commandline parameter do_full_cleanup = True + elif botsglobal.ini.getboolean('acceptance','runacceptancetest',False): # no automatic cleanup during acceptance testing + return elif whencleanup in ['always','daily']: #perform full cleanup only first run of the day. cur_day = int(time.strftime('%Y%m%d')) #get current date, convert to int diff --git a/bots/communication.py b/bots/communication.py index 01cf7d0..a5cd063 100644 --- a/bots/communication.py +++ b/bots/communication.py @@ -111,7 +111,14 @@ def run(self): if self.command == 'new': #only in-communicate for new run #~ print('in communication run 1') #handle maxsecondsperchannel: use global value from bots.ini unless specified in channel. (In database this is field 'rsrv2'.) - self.maxsecondsperchannel = self.channeldict['rsrv2'] if self.channeldict['rsrv2'] is not None and self.channeldict['rsrv2'] > 0 else botsglobal.ini.getint('settings','maxsecondsperchannel',sys.maxsize) + # MJG Python 3 deprecation warning fix + self.maxsecondsperchannel = botsglobal.ini.getint('settings','maxsecondsperchannel',sys.maxint) + try: + secs = int(self.channeldict['rsrv2']) + if secs > 0: + self.maxsecondsperchannel = secs + except: + pass #bots tries to connect several times. this is probably a better stategy than having long time-outs. max_nr_connect_tries = botsglobal.ini.getint('settings','maxconnectiontries',3) #how often does bots try to connect. TODO later version: setting per channel nr_connect_tries = 0 @@ -131,11 +138,12 @@ def run(self): #max_nr_retry : from channel. should be integer, but only textfields where left. so might be ''/None->use 0 max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 if max_nr_retry: - domain = 'bots_communication_failure_' + self.channeldict['idchannel'] + domain = (self.channeldict['idchannel'] + u'_failure')[:35] nr_retry = botslib.unique(domain) #update nr_retry in database if nr_retry >= max_nr_retry: botslib.unique(domain,updatewith=0) #reset nr_retry to zero else: + botsglobal.logger.info(u'Communication failure %s on channel %s',nr_retry,self.channeldict['idchannel']) return #max_nr_retry is not reached. return without error raise finally: @@ -145,7 +153,7 @@ def run(self): # ~ #max_nr_retry : get this from channel. should be integer, but only textfields where left. so might be ''/None->use 0 # ~ max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 # ~ if max_nr_retry: - # ~ domain = 'bots_communication_failure_' + self.channeldict['idchannel'] + # ~ domain = (self.channeldict['idchannel'] + u'_failure')[:35] # ~ botslib.unique(domain,updatewith=0) #set nr_retry to zero self.incommunicate() self.disconnect() @@ -267,8 +275,8 @@ def file2mime(self): confirmtype = '' confirmasked = False charset = row['charset'] - - if row['editype'] == 'email-confirmation': #outgoing MDN: message is already assembled + # MJG 15/01/2019 BUGFIX for automaticretrycommunication + if row['editype'] == 'email-confirmation' or self.command == 'automaticretrycommunication': # message is already assembled outfilename = row['filename'] else: #assemble message: headers and payload. Bots uses simple MIME-envelope; by default payload is an attachment message = email.message.Message() @@ -413,7 +421,7 @@ def savemime(msg): outfile.close() nrmimesaved += 1 ta_file.update(statust=OK, - contenttype=contenttype, + contenttype=contenttype[:35], # MJG 24/08/2016 truncate to fit in db filename=outfilename, filesize=filesize, divtext=attachment_filename) diff --git a/bots/models.py b/bots/models.py index 8ababf6..24e92b2 100644 --- a/bots/models.py +++ b/bots/models.py @@ -166,6 +166,7 @@ def script_link1(script,linktext): ''' if script exists return a plain text name as link; else return "no" icon, plain text name used in translate (all scripts should exist, missing script is an error). ''' + script = script.replace('.',os.sep,script.count('.')-1) # allow mapping script subdirs if os.path.exists(script): return '%s'%(urllib_quote(script.encode("utf-8")),linktext) else: diff --git a/bots/pluglib.py b/bots/pluglib.py index 0bb428c..3d8bdc3 100644 --- a/bots/pluglib.py +++ b/bots/pluglib.py @@ -378,7 +378,7 @@ def plugout_files(cleaned_data): if cleaned_data['fileconfiguration']: #gather from usersys files2return.extend(plugout_files_bydir(usersys,'usersys')) if not cleaned_data['charset']: #if edifact charsets are not needed: remove them (are included in default bots installation). - charsetdirs = plugout_files_bydir(os.path.join(usersys,'charsets'),'usersys/charsets') + charsetdirs = plugout_files_bydir(os.path.join(usersys,'charsets'),os.path.join('usersys','charsets')) for charset in charsetdirs: try: index = files2return.index(charset) diff --git a/bots/transform.py b/bots/transform.py index f7c2dec..06b13af 100644 --- a/bots/transform.py +++ b/bots/transform.py @@ -190,7 +190,8 @@ def _translate_one_file(row,routedict,endstatus,userscript,scriptname): if inn_splitup.ta_info.get('KillWholeFile',False): raise botslib.KillWholeFileException(msg) txt = botslib.txtexc() - ta_splitup.update(statust=ERROR,errortext=txt,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? + inn_splitup.ta_info['errortext'] = txt + ta_splitup.update(statust=ERROR,**inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? ta_splitup.deletechildren() else: ta_splitup.update(statust=DONE, **inn_splitup.ta_info) #update db. inn_splitup.ta_info could be changed by mappingscript. Is this useful? From 20c4917b91a4ac797cb6223e6162a11642e59908 Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Fri, 15 Jan 2021 11:21:21 +1030 Subject: [PATCH 2/6] Update communication.py --- bots/communication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bots/communication.py b/bots/communication.py index a5cd063..fb0aa41 100644 --- a/bots/communication.py +++ b/bots/communication.py @@ -138,7 +138,7 @@ def run(self): #max_nr_retry : from channel. should be integer, but only textfields where left. so might be ''/None->use 0 max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 if max_nr_retry: - domain = (self.channeldict['idchannel'] + u'_failure')[:35] + domain = (self.channeldict['idchannel'] + '_failure')[:35] nr_retry = botslib.unique(domain) #update nr_retry in database if nr_retry >= max_nr_retry: botslib.unique(domain,updatewith=0) #reset nr_retry to zero @@ -153,7 +153,7 @@ def run(self): # ~ #max_nr_retry : get this from channel. should be integer, but only textfields where left. so might be ''/None->use 0 # ~ max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 # ~ if max_nr_retry: - # ~ domain = (self.channeldict['idchannel'] + u'_failure')[:35] + # ~ domain = (self.channeldict['idchannel'] + '_failure')[:35] # ~ botslib.unique(domain,updatewith=0) #set nr_retry to zero self.incommunicate() self.disconnect() From be4230f08de8f5ea2baf8552c8ddd0eabd4d396e Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Fri, 22 Jan 2021 15:00:20 +1030 Subject: [PATCH 3/6] Update communication.py syntax error indent --- bots/communication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bots/communication.py b/bots/communication.py index fb0aa41..e3781f5 100644 --- a/bots/communication.py +++ b/bots/communication.py @@ -143,7 +143,7 @@ def run(self): if nr_retry >= max_nr_retry: botslib.unique(domain,updatewith=0) #reset nr_retry to zero else: - botsglobal.logger.info(u'Communication failure %s on channel %s',nr_retry,self.channeldict['idchannel']) + botsglobal.logger.info('Communication failure %s on channel %s',nr_retry,self.channeldict['idchannel']) return #max_nr_retry is not reached. return without error raise finally: From 88bd2aefc38318c616b2d34ee57c357f02845f33 Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Fri, 22 Jan 2021 15:09:47 +1030 Subject: [PATCH 4/6] Update engine.py fix ini key for daily logging to match botsinit.py print to console all severe errors --- bots/engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bots/engine.py b/bots/engine.py index ba85f29..b1134ee 100644 --- a/bots/engine.py +++ b/bots/engine.py @@ -85,10 +85,11 @@ def start(): botsglobal.logger = botsinit.initenginelogging(process_name) atexit.register(logging.shutdown) except Exception as msg: + print('Error initialising logging:' + msg) botslib.sendbotserrorreport('[Bots severe error] Bots is not running anymore','Bots does not run because logging is not possible.\nOften a rights problem.\n') sys.exit(1) else: - if botsglobal.ini.get('settings','log_file_number','') != 'daily': + if botsglobal.ini.get('settings','log_when',None) != 'daily': for key,value in botslib.botsinfo(): #log info about environement, versions, etc botsglobal.logger.info('%(key)s: "%(value)s".',{'key':key,'value':value}) @@ -96,6 +97,7 @@ def start(): try: botsinit.connect() except Exception as msg: + print('Error connecting to database:' + msg) botsglobal.logger.exception('Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".',{'msg':msg}) sys.exit(1) else: @@ -196,6 +198,7 @@ def start(): cleanup.cleanup(do_cleanup_parameter,userscript,scriptname) except Exception as msg: + print('Severe error:' + msg) botsglobal.logger.exception('Severe error in bots system:\n%(msg)s',{'msg':unicode(msg)}) #of course this 'should' not happen. sys.exit(1) else: From 91d209a716a78e361be6795bc9edaa69a51e7627 Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Thu, 4 Feb 2021 16:36:02 +1030 Subject: [PATCH 5/6] reset failure counter after a successful connection --- bots/communication.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bots/communication.py b/bots/communication.py index e3781f5..eb75a71 100644 --- a/bots/communication.py +++ b/bots/communication.py @@ -146,15 +146,15 @@ def run(self): botsglobal.logger.info('Communication failure %s on channel %s',nr_retry,self.channeldict['idchannel']) return #max_nr_retry is not reached. return without error raise + else: + #in-connection OK. Reset database entry. + #max_nr_retry : get this from channel. should be integer, but only textfields where left. so might be ''/None->use 0 + max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 + if max_nr_retry: + domain = (self.channeldict['idchannel'] + '_failure')[:35] + botslib.unique(domain,updatewith=0) #set nr_retry to zero finally: break - # ~ else: - # ~ #in-connection OK. Reset database entry. - # ~ #max_nr_retry : get this from channel. should be integer, but only textfields where left. so might be ''/None->use 0 - # ~ max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 - # ~ if max_nr_retry: - # ~ domain = (self.channeldict['idchannel'] + '_failure')[:35] - # ~ botslib.unique(domain,updatewith=0) #set nr_retry to zero self.incommunicate() self.disconnect() self.postcommunicate() From f2eec06213919b4b16d8002e1290b76de4f5554b Mon Sep 17 00:00:00 2001 From: Mike GRIFFIN Date: Fri, 5 Feb 2021 18:02:23 +1030 Subject: [PATCH 6/6] Fix up maxconnectiontries logic in run() 1. match variable names to the settings - less confusing reading the code :-) 2. maxconnectiontries applies to both in and out channels. 3. fix the try-except-else-finally (i had to read python docs again... break in finally clause was no good) 4. new python 3 exception context adds confusion to the trace when there is a connection error; fix that to just re-raise the original (raise exc from exc) --- bots/communication.py | 81 ++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/bots/communication.py b/bots/communication.py index eb75a71..9822b87 100644 --- a/bots/communication.py +++ b/bots/communication.py @@ -101,60 +101,71 @@ def __init__(self,channeldict,idroute,userscript,scriptname,command,rootidta): self.rootidta = rootidta def run(self): + # Max connection tries; bots tries to connect several times per run. this is probably a better strategy than having long time-outs. + # This is useful for both incoming and outgoing channels. TODO later version: setting per channel [MJG not sure this is needed per channel] + maxconnectiontries = botsglobal.ini.getint('settings','maxconnectiontries',3) + nr_connectiontries = 0 + if self.channeldict['inorout'] == 'out': self.precommunicate() - self.connect() + + while True: + nr_connectiontries += 1 + try: + #print('connect',nr_connectiontries) + self.connect() + except Exception as exc: + #print(exc) + if nr_connectiontries >= maxconnectiontries: + raise(exc) from exc # just re-raise the original exception, no context chain + else: + break # out-connection OK + self.outcommunicate() self.disconnect() self.archive() + else: #do incommunication if self.command == 'new': #only in-communicate for new run - #~ print('in communication run 1') #handle maxsecondsperchannel: use global value from bots.ini unless specified in channel. (In database this is field 'rsrv2'.) - # MJG Python 3 deprecation warning fix - self.maxsecondsperchannel = botsglobal.ini.getint('settings','maxsecondsperchannel',sys.maxint) + # TODO: MJG in hindsight, a "maxfiles" value would be more useful. Sometimes in-channel is very fast and out-channel + # is very slow. If bots receives 1000 files in 30 seconds then they have to be sent even if it takes 3 hours. + # I have needed to carefully "fine-tune" maxseconds on fast in-channels. + self.maxsecondsperchannel = botsglobal.ini.getint('settings','maxsecondsperchannel',sys.maxsize) try: secs = int(self.channeldict['rsrv2']) if secs > 0: self.maxsecondsperchannel = secs except: pass - #bots tries to connect several times. this is probably a better stategy than having long time-outs. - max_nr_connect_tries = botsglobal.ini.getint('settings','maxconnectiontries',3) #how often does bots try to connect. TODO later version: setting per channel - nr_connect_tries = 0 + # Max failures; bots keeps count of consecutive failures across runs for an in-channel before reporting a process error + # from channel. should be integer, but only textfields were left. so might be None->use 0. + maxfailures = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 + domain = (self.channeldict['idchannel'] + '_failure')[:35] while True: - nr_connect_tries += 1 + nr_connectiontries += 1 try: - #~ print('in communication run 2') + #~print('connect try',nr_connectiontries) self.connect() - #~ print('in communication run 3') - except: - #check if max nr tries is reached - if nr_connect_tries < max_nr_connect_tries: - continue - #in-connection failed (no files are received yet via this channel) - #store in database how many failed connection tries for this channel. - #useful if bots is scheduled quite often, and limiting number of error-reports eg when server is down. - #max_nr_retry : from channel. should be integer, but only textfields where left. so might be ''/None->use 0 - max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 - if max_nr_retry: - domain = (self.channeldict['idchannel'] + '_failure')[:35] - nr_retry = botslib.unique(domain) #update nr_retry in database - if nr_retry >= max_nr_retry: - botslib.unique(domain,updatewith=0) #reset nr_retry to zero - else: - botsglobal.logger.info('Communication failure %s on channel %s',nr_retry,self.channeldict['idchannel']) - return #max_nr_retry is not reached. return without error - raise + except Exception as exc: + #~ print(exc) + if nr_connectiontries >= maxconnectiontries: + #in-connection failed (no files are received yet via this channel) + #store in database how many consecutive failures for this channel. + #only raise exception every multiple of maxfailures (use modulo, keep actual count) + #useful if bots is scheduled quite often, and limiting number of error-reports eg when server is down. + if maxfailures: + nr_failures = botslib.unique(domain) # increment nr_failures counter in database + if nr_failures % maxfailures != 0: + botsglobal.logger.info('Communication failure %s on channel %s: %s',nr_failures,self.channeldict['idchannel'],msg) + return #maxfailures is not reached. return without error + raise exc from exc # just re-raise the original exception, no context chain else: - #in-connection OK. Reset database entry. - #max_nr_retry : get this from channel. should be integer, but only textfields where left. so might be ''/None->use 0 - max_nr_retry = int(self.channeldict['rsrv1']) if self.channeldict['rsrv1'] else 0 - if max_nr_retry: - domain = (self.channeldict['idchannel'] + '_failure')[:35] - botslib.unique(domain,updatewith=0) #set nr_retry to zero - finally: + #in-connection OK. Reset failure counter to zero + if maxfailures: + botslib.unique(domain,updatewith=0) break + self.incommunicate() self.disconnect() self.postcommunicate()