From 970110bc96e1ec9e29234e4de059cc8d44d95c81 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Sun, 31 Jul 2022 21:16:40 -0700 Subject: [PATCH 01/21] dlvax commit --- .gitignore | 5 + helpers/config.py | 48 +++++- nparse.py | 6 +- parsers/__init__.py | 1 + parsers/deathloopvaccine.py | 286 ++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +- 6 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 parsers/deathloopvaccine.py diff --git a/.gitignore b/.gitignore index 2ce535a..58323fa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .idea/ .vscode/ +working/ +act.bat +deact.bat # Byte-compiled / optimized / DLL files __pycache__/ @@ -15,6 +18,7 @@ __pycache__/ # Distribution / packaging .Python +.venv/ env/ venv/ bin/ @@ -65,3 +69,4 @@ docs/_build/ wiki/ old/ nparse.config.json + diff --git a/helpers/config.py b/helpers/config.py index 2b539b1..b8c7817 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -287,7 +287,53 @@ def verify_settings(): False ) - + # deathloopvaccine + # note - many of these settings are completely unused, and are only here to satisfy the underlying + # assumption that they are to be manipulated on startup/shutdown + data['deathloopvaccine'] = data.get('deathloopvaccine', {}) + data['deathloopvaccine']['toggled'] = get_setting( + data['deathloopvaccine'].get('toggled', True), + True + ) + data['deathloopvaccine']['geometry'] = get_setting( + data['deathloopvaccine'].get('geometry', [0, 400, 200, 400]), + [0, 400, 200, 400], + lambda x: ( + len(x) == 4 and + isinstance(x[0], int) and + isinstance(x[1], int) and + isinstance(x[2], int) and + isinstance(x[3], int) + ) + ) + data['deathloopvaccine']['url'] = get_setting( + data['deathloopvaccine'].get('url', ''), + '' + ) + data['deathloopvaccine']['opacity'] = get_setting( + data['deathloopvaccine'].get('opacity', 0), + 0, + lambda x: (0 <= x <= 100) + ) + data['deathloopvaccine']['bg_opacity'] = get_setting( + data['deathloopvaccine'].get('bg_opacity', 0), + 0, + lambda x: (0 <= x <= 100) + ) + data['deathloopvaccine']['color'] = data['deathloopvaccine'].get('color', '#000000') + data['deathloopvaccine']['clickthrough'] = get_setting( + data['deathloopvaccine'].get('clickthrough', True), + True + ) + data['deathloopvaccine']['deaths'] = get_setting( + data['deathloopvaccine'].get('deaths', 4), + 4 + ) + data['deathloopvaccine']['seconds'] = get_setting( + data['deathloopvaccine'].get('seconds', 120), + 120 + ) + def get_setting(setting, default, func=None): try: assert(type(setting) == type(default)) diff --git a/nparse.py b/nparse.py index 2d17860..8a4f0d8 100644 --- a/nparse.py +++ b/nparse.py @@ -19,8 +19,8 @@ os.environ['QT_SCALE_FACTOR'] = str( config.data['general']['qt_scale_factor'] / 100) - -CURRENT_VERSION = '0.6.4' +# todo - set this to appropriate value +CURRENT_VERSION = '0.6.4-DLV' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: @@ -67,11 +67,13 @@ def _load_parsers(self): "maps": parsers.Maps(), "spells": parsers.Spells(), "discord": parsers.Discord(), + "deathloopvaccine" : parsers.DeathLoopVaccine(), } self._parsers = [ self._parsers_dict["maps"], self._parsers_dict["spells"], self._parsers_dict["discord"], + self._parsers_dict["deathloopvaccine"], ] for parser in self._parsers: if parser.name in config.data.keys() and 'geometry' in config.data[parser.name].keys(): diff --git a/parsers/__init__.py b/parsers/__init__.py index 427a74c..5a7b78e 100644 --- a/parsers/__init__.py +++ b/parsers/__init__.py @@ -2,3 +2,4 @@ from .maps import Maps # noqa: F401 from .spells import Spells # noqa: F401 from .discord import Discord # noqa: F401 +from .deathloopvaccine import DeathLoopVaccine # noqa: F401 diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py new file mode 100644 index 0000000..5395f0d --- /dev/null +++ b/parsers/deathloopvaccine.py @@ -0,0 +1,286 @@ +import datetime +import re +import os +import signal + +import psutil + +from datetime import datetime +from helpers import ParserWindow, config + + +# +# simple utility to prevent Everquest Death Loop +# +# The utility functions by parsing the current (most recent) Everquest log file, and if it detects +# Death Loop symptoms, it will respond by initiating a system process kill of all "eqgame.exe" +# processes (there should usually only be one). +# +# We will define a death loop as any time a player experiences X deaths in Y seconds, and no player +# activity during that time. The values for X and Y are configurable, via the DeathLoopVaccine.ini file. +# +# For testing purposes, there is a back door feature, controlled by sending a tell to the following +# non-existent player: +# +# death_loop: Simulates a player death. +# +# Note however that this also sets a flag that disarms the conceptual +# "process-killer gun", which will allow every bit of the code to +# execute and be tested, but will stop short of actually killing any +# process +# +# The "process-killer gun" will then be armed again after the simulated +# player deaths trigger the simulated process kill, or after any simulated +# player death events "scroll off" the death loop monitoring window. +# +class DeathLoopVaccine(ParserWindow): + + """Tracks for DL symptoms""" + + def __init__(self): + super().__init__() + self.name = 'deathloopvaccine' + + # parameters that define a deathloop condition, i.e. D deaths in T seconds, + # with no player activity in the interim + # todo - make these configarable via the UI? + self.deathloop_deaths = config.data['deathloopvaccine']['deaths'] + self.deathloop_seconds = config.data['deathloopvaccine']['seconds'] + + # list of death messages + # this will function as a scrolling queue, with the oldest message at position 0, + # newest appended to the other end. Older messages scroll off the list when more + # than deathloop_seconds have elapsed. The list is also flushed any time + # player activity is detected (i.e. player is not AFK). + # + # if/when the length of this list meets or exceeds deathloop_deaths, then + # the deathloop response is triggered + self._death_list = list() + + # flag indicating whether the "process killer" gun is armed + self._kill_armed = True + + def reset(self) -> None: + """ + Utility function to clear the death_list and reset the armed flag + + Returns: + None: + """ + self._death_list.clear() + self._kill_armed = True + + # main parsing logic here + def parse(self, timestamp: datetime, text: str) -> None: + """ + Parse a single line from the logfile + + Args: + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + text: The text following the everquest timestamp + + Returns: + None: + """ + + self.check_for_death(timestamp, text) + self.check_not_afk(timestamp, text) + self.deathloop_response() + + def check_for_death(self, timestamp: datetime, text: str) -> None: + """ + check for indications the player just died, and if we find it, + save the message for later processing + + Args: + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + text: The text following the everquest timestamp + + Returns: + None: + """ + + # reconstruct the full logfile line + # this is a bit counter-intuitive, but the rest of the logic in this function was + # developed assuming the line was the full line, including the time stamp + line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text + + # cut off the leading date-time stamp info + trunc_line = line[27:] + + # does this line contain a death message + slain_regexp = r'^You have been slain' + m = re.match(slain_regexp, trunc_line) + if m: + # add this message to the list of death messages + self._death_list.append(line) + starprint(f'DeathLoopVaccine: Death count = {len(self._death_list)}') + + # a way to test - send a tell to death_loop + slain_regexp = r'^death_loop' + m = re.match(slain_regexp, trunc_line) + if m: + # add this message to the list of death messages + # since this is just for testing, disarm the kill-gun + self._death_list.append(line) + starprint(f'DeathLoopVaccine: Death count = {len(self._death_list)}') + self._kill_armed = False + + # only do the list-purging if there are already some death messages in the list, else skip this + if len(self._death_list) > 0: + + # create a datetime object for this line, using the very capable datetime.strptime() + now = datetime.strptime(line[0:26], '[%a %b %d %H:%M:%S %Y]') + + # now purge any death messages that are too old + done = False + while not done: + # if the list is empty, we're done + if len(self._death_list) == 0: + self.reset() + done = True + # if the list is not empty, check if we need to purge some old entries + else: + oldest_line = self._death_list[0] + oldest_time = datetime.strptime(oldest_line[0:26], '[%a %b %d %H:%M:%S %Y]') + elapsed_seconds = now - oldest_time + + if elapsed_seconds.total_seconds() > self.deathloop_seconds: + # that death message is too old, purge it + self._death_list.pop(0) + starprint(f'DeathLoopVaccine: Death count = {len(self._death_list)}') + else: + # the oldest death message is inside the window, so we're done purging + done = True + + def check_not_afk(self, timestamp: datetime, text: str) -> None: + """ + check for "proof of life" indications the player is really not AFK + + Args: + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + text: The text following the everquest timestamp + + Returns: + None: + """ + + # reconstruct the full logfile line + # this is a bit counter-intuitive, but the rest of the logic in this function was + # developed assuming the line was the full line, including the time stamp + line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text + + # only do the proof of life checks if there are already some death messages in the list, else skip this + if len(self._death_list) > 0: + + # check for proof of life, things that indicate the player is not actually AFK + # begin by assuming the player is AFK + afk = True + + # cut off the leading date-time stamp info + trunc_line = line[27:] + + # does this line contain a proof of life - casting + regexp = r'^You begin casting' + m = re.match(regexp, trunc_line) + if m: + # player is not AFK + afk = False + starprint(f'DeathLoopVaccine: Player Not AFK: {line}') + + # does this line contain a proof of life - communication + # this captures tells, say, group, auction, and shout channels + # todo - get character name for use in this regexp + charname = 'Unknown' + regexp = f'^(You told|You say|You tell|You auction|You shout|{charname} ->)' + m = re.match(regexp, trunc_line) + if m: + # player is not AFK + afk = False + starprint(f'DeathLoopVaccine: Player Not AFK: {line}') + + # does this line contain a proof of life - melee + regexp = r'^You( try to)? (hit|slash|pierce|crush|claw|bite|sting|maul|gore|punch|kick|backstab|bash)' + m = re.match(regexp, trunc_line) + if m: + # player is not AFK + afk = False + starprint(f'DeathLoopVaccine: Player Not AFK: {line}') + + # if they are not AFK, then go ahead and purge any death messages from the list + if not afk: + self.reset() + + def deathloop_response(self) -> None: + """ + are we death looping? if so, kill the process + + Returns: + None: + """ + + # if the death_list contains more deaths than the limit, then trigger the process kill + if len(self._death_list) >= self.deathloop_deaths: + + starprint('---------------------------------------------------') + starprint('DeathLoopVaccine - Killing all eqgame.exe processes') + starprint('---------------------------------------------------') + starprint('DeathLoopVaccine has detected deathloop symptoms:') + starprint(f' {self.deathloop_deaths} deaths in less than {self.deathloop_seconds} seconds, with no player activity') + + # show all the death messages + starprint('Death Messages:') + for line in self._death_list: + starprint(' ' + line) + + # get the list of eqgame.exe process ID's, and show them + pid_list = get_eqgame_pid_list() + starprint(f'eqgame.exe process id list = {pid_list}') + + # kill the eqgame.exe process / processes + for pid in pid_list: + starprint(f'Killing process [{pid}]') + + # for testing the actual kill process using simulated player deaths, uncomment the following line + # self._kill_armed = True + if self._kill_armed: + os.kill(pid, signal.SIGTERM) + else: + starprint('(Note: Process Kill only simulated, since death(s) were simulated)') + + # purge any death messages from the list + self.reset() + + +################################################################################################# +# +# standalone functions +# + + +def get_eqgame_pid_list() -> list[int]: + """ + get list of process ID's for eqgame.exe, using psutil module + + Returns: + object: list of process ID's (in case multiple versions of eqgame.exe are somehow running) + """ + + pid_list = list() + for p in psutil.process_iter(['name']): + if p.info['name'] == 'eqgame.exe': + pid_list.append(p.pid) + return pid_list + + +def starprint(line: str) -> None: + """ + utility function to print with leading and trailing ** indicators + + Args: + line: line to be printed + + Returns: + None: + """ + print(f'** {line.rstrip():<100} **') diff --git a/requirements.txt b/requirements.txt index 96ba78b..ae5856e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ websocket-client==0.58.0 requests colorhash pathvalidate -PyQtWebEngine \ No newline at end of file +PyQtWebEngine +psutil \ No newline at end of file From 807cb50d819b06506aa0c28518f578a189559c55 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Sun, 31 Jul 2022 21:54:54 -0700 Subject: [PATCH 02/21] small edits --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ae5856e..83ba159 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ requests colorhash pathvalidate PyQtWebEngine -psutil \ No newline at end of file +psutil From 77ecb8fd1dc383320ff8ef6ecbc780076bb43319 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 1 Aug 2022 11:59:45 -0700 Subject: [PATCH 03/21] cleaned up Parser inheritance --- .gitignore | 2 -- helpers/__init__.py | 1 + helpers/config.py | 36 +++--------------------- helpers/logreader.py | 2 ++ helpers/parser.py | 56 ++++++++++++++++++++++++++++++------- nparse.py | 12 ++++---- parsers/deathloopvaccine.py | 11 ++++---- 7 files changed, 66 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index 58323fa..2be6b01 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,6 @@ .idea/ .vscode/ working/ -act.bat -deact.bat # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/helpers/__init__.py b/helpers/__init__.py index a0b67ca..6ac76eb 100644 --- a/helpers/__init__.py +++ b/helpers/__init__.py @@ -5,6 +5,7 @@ import json from datetime import datetime, timedelta +from .parser import Parser # noqa: F401 from .parser import ParserWindow # noqa: F401 diff --git a/helpers/config.py b/helpers/config.py index b8c7817..00335b7 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -49,6 +49,10 @@ def verify_settings(): data['general'].get('eq_log_dir', ''), '' ) + data['general']['char_name'] = get_setting( + data['general'].get('char_name', ''), + '' + ) data['general']['window_flush'] = get_setting( data['general'].get('window_flush', True), True @@ -288,43 +292,11 @@ def verify_settings(): ) # deathloopvaccine - # note - many of these settings are completely unused, and are only here to satisfy the underlying - # assumption that they are to be manipulated on startup/shutdown data['deathloopvaccine'] = data.get('deathloopvaccine', {}) data['deathloopvaccine']['toggled'] = get_setting( data['deathloopvaccine'].get('toggled', True), True ) - data['deathloopvaccine']['geometry'] = get_setting( - data['deathloopvaccine'].get('geometry', [0, 400, 200, 400]), - [0, 400, 200, 400], - lambda x: ( - len(x) == 4 and - isinstance(x[0], int) and - isinstance(x[1], int) and - isinstance(x[2], int) and - isinstance(x[3], int) - ) - ) - data['deathloopvaccine']['url'] = get_setting( - data['deathloopvaccine'].get('url', ''), - '' - ) - data['deathloopvaccine']['opacity'] = get_setting( - data['deathloopvaccine'].get('opacity', 0), - 0, - lambda x: (0 <= x <= 100) - ) - data['deathloopvaccine']['bg_opacity'] = get_setting( - data['deathloopvaccine'].get('bg_opacity', 0), - 0, - lambda x: (0 <= x <= 100) - ) - data['deathloopvaccine']['color'] = data['deathloopvaccine'].get('color', '#000000') - data['deathloopvaccine']['clickthrough'] = get_setting( - data['deathloopvaccine'].get('clickthrough', True), - True - ) data['deathloopvaccine']['deaths'] = get_setting( data['deathloopvaccine'].get('deaths', 4), 4 diff --git a/helpers/logreader.py b/helpers/logreader.py index a9ed677..9f43556 100644 --- a/helpers/logreader.py +++ b/helpers/logreader.py @@ -46,8 +46,10 @@ def _file_changed(self, changed_file): if changed_file != self._stats['log_file']: self._stats['log_file'] = changed_file char_name = os.path.basename(changed_file).split("_")[1] + config.data['general']['char_name'] = char_name if not config.data['sharing']['player_name_override']: config.data['sharing']['player_name'] = char_name + config.save() location_service.SIGNALS.config_updated.emit() with open(self._stats['log_file'], 'rb') as log: log.seek(0, os.SEEK_END) diff --git a/helpers/parser.py b/helpers/parser.py index 30cf48d..49f6158 100644 --- a/helpers/parser.py +++ b/helpers/parser.py @@ -3,9 +3,54 @@ QPushButton, QVBoxLayout, QWidget) from helpers import config +from datetime import datetime -class ParserWindow(QFrame): +class Parser(): + + def __init__(self): + super().__init__() + self.name = 'Parser' + self._visible = False + + def isVisible(self) -> bool: + return self._visible + + def hide(self): + self._visible = False + + def show(self): + self._visible = True + + # main parsing logic here - derived classed should override this to perform their particular parsing tasks + def parse(self, timestamp: datetime, text: str) -> None: + + # default behavior = simply print passed info + # this strftime mask will recreate the EQ log file timestamp format + line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text + print(f'[{self.name}]:{line}') + + def toggle(self, _=None): + if self.isVisible(): + self.hide() + config.data[self.name]['toggled'] = False + else: + self.set_flags() + self.show() + config.data[self.name]['toggled'] = True + config.save() + + def shutdown(self): + pass + + def set_flags(self): + pass + + def settings_updated(self): + pass + + +class ParserWindow(QFrame, Parser): def __init__(self): super().__init__() @@ -108,15 +153,6 @@ def _toggle_frame(self): def set_title(self, title): self._title.setText(title) - def toggle(self, _=None): - if self.isVisible(): - self.hide() - config.data[self.name]['toggled'] = False - else: - self.set_flags() - self.show() - config.data[self.name]['toggled'] = True - config.save() def closeEvent(self, _): config.data[self.name]['toggled'] = False diff --git a/nparse.py b/nparse.py index 8a4f0d8..ccc2938 100644 --- a/nparse.py +++ b/nparse.py @@ -91,7 +91,8 @@ def _toggle(self): error.args[0], error.args[1], msecs=3000) else: - self._log_reader = logreader.LogReader( + self._log_reader = \ + logreader.LogReader( config.data['general']['eq_log_dir']) self._log_reader.new_line.connect(self._parse) self._toggled = True @@ -186,10 +187,11 @@ def _menu(self, event): # save parser geometry for parser in self._parsers: - g = parser.geometry() - config.data[parser.name]['geometry'] = [ - g.x(), g.y(), g.width(), g.height() - ] + if parser.name in config.data.keys() and 'geometry' in config.data[parser.name].keys(): + g = parser.geometry() + config.data[parser.name]['geometry'] = [ + g.x(), g.y(), g.width(), g.height() + ] config.save() self._system_tray.setVisible(False) diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index 5395f0d..a2773c9 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -6,7 +6,7 @@ import psutil from datetime import datetime -from helpers import ParserWindow, config +from helpers import Parser, config # @@ -33,11 +33,12 @@ # player deaths trigger the simulated process kill, or after any simulated # player death events "scroll off" the death loop monitoring window. # -class DeathLoopVaccine(ParserWindow): +class DeathLoopVaccine(Parser): """Tracks for DL symptoms""" def __init__(self): + super().__init__() self.name = 'deathloopvaccine' @@ -60,6 +61,7 @@ def __init__(self): # flag indicating whether the "process killer" gun is armed self._kill_armed = True + def reset(self) -> None: """ Utility function to clear the death_list and reset the armed flag @@ -190,9 +192,8 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: # does this line contain a proof of life - communication # this captures tells, say, group, auction, and shout channels - # todo - get character name for use in this regexp - charname = 'Unknown' - regexp = f'^(You told|You say|You tell|You auction|You shout|{charname} ->)' + char_name = config.data['general']['char_name'] + regexp = f'^(You told|You say|You tell|You auction|You shout|{char_name} ->)' m = re.match(regexp, trunc_line) if m: # player is not AFK From 223b14eedf4e3e1cd08b93422a93c2127a7abf66 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 1 Aug 2022 12:04:56 -0700 Subject: [PATCH 04/21] small edit --- helpers/logreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/logreader.py b/helpers/logreader.py index 9f43556..7423d86 100644 --- a/helpers/logreader.py +++ b/helpers/logreader.py @@ -47,9 +47,9 @@ def _file_changed(self, changed_file): self._stats['log_file'] = changed_file char_name = os.path.basename(changed_file).split("_")[1] config.data['general']['char_name'] = char_name + config.save() if not config.data['sharing']['player_name_override']: config.data['sharing']['player_name'] = char_name - config.save() location_service.SIGNALS.config_updated.emit() with open(self._stats['log_file'], 'rb') as log: log.seek(0, os.SEEK_END) From fa4fa89b91e87334a3d98e2bcf0dfeab8f37dcb0 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 1 Aug 2022 12:13:26 -0700 Subject: [PATCH 05/21] editorial --- nparse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nparse.py b/nparse.py index ccc2938..c619aa1 100644 --- a/nparse.py +++ b/nparse.py @@ -91,8 +91,7 @@ def _toggle(self): error.args[0], error.args[1], msecs=3000) else: - self._log_reader = \ - logreader.LogReader( + self._log_reader = logreader.LogReader( config.data['general']['eq_log_dir']) self._log_reader.new_line.connect(self._parse) self._toggled = True From e84d05a3514a8bb264f1ca33d17243cc36247d92 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 1 Aug 2022 12:24:34 -0700 Subject: [PATCH 06/21] addressing some PEP warnings --- helpers/config.py | 1 + helpers/parser.py | 11 +++++------ nparse.py | 2 +- parsers/deathloopvaccine.py | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index 00335b7..9ed17d1 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -306,6 +306,7 @@ def verify_settings(): 120 ) + def get_setting(setting, default, func=None): try: assert(type(setting) == type(default)) diff --git a/helpers/parser.py b/helpers/parser.py index 49f6158..a24d632 100644 --- a/helpers/parser.py +++ b/helpers/parser.py @@ -6,7 +6,7 @@ from datetime import datetime -class Parser(): +class Parser: def __init__(self): super().__init__() @@ -30,7 +30,7 @@ def parse(self, timestamp: datetime, text: str) -> None: line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text print(f'[{self.name}]:{line}') - def toggle(self, _=None): + def toggle(self, _=None) -> None: if self.isVisible(): self.hide() config.data[self.name]['toggled'] = False @@ -40,13 +40,13 @@ def toggle(self, _=None): config.data[self.name]['toggled'] = True config.save() - def shutdown(self): + def shutdown(self) -> None: pass - def set_flags(self): + def set_flags(self) -> None: pass - def settings_updated(self): + def settings_updated(self) -> None: pass @@ -153,7 +153,6 @@ def _toggle_frame(self): def set_title(self, title): self._title.setText(title) - def closeEvent(self, _): config.data[self.name]['toggled'] = False config.save() diff --git a/nparse.py b/nparse.py index c619aa1..c71508c 100644 --- a/nparse.py +++ b/nparse.py @@ -67,7 +67,7 @@ def _load_parsers(self): "maps": parsers.Maps(), "spells": parsers.Spells(), "discord": parsers.Discord(), - "deathloopvaccine" : parsers.DeathLoopVaccine(), + "deathloopvaccine": parsers.DeathLoopVaccine(), } self._parsers = [ self._parsers_dict["maps"], diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index a2773c9..5242995 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -61,7 +61,6 @@ def __init__(self): # flag indicating whether the "process killer" gun is armed self._kill_armed = True - def reset(self) -> None: """ Utility function to clear the death_list and reset the armed flag From 7d319335bce5f081031f66bf28e4f66369468f10 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 1 Aug 2022 17:53:24 -0700 Subject: [PATCH 07/21] PR comments. 1. char_name moved out of .json and into a global variable. 2. standalone functions move into helpers --- helpers/__init__.py | 31 +++++++++++++++++++++++++++++ helpers/config.py | 5 +---- helpers/logreader.py | 3 +-- parsers/deathloopvaccine.py | 39 ++----------------------------------- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/helpers/__init__.py b/helpers/__init__.py index 6ac76eb..36a94f8 100644 --- a/helpers/__init__.py +++ b/helpers/__init__.py @@ -3,6 +3,9 @@ import math import requests import json + +import psutil + from datetime import datetime, timedelta from .parser import Parser # noqa: F401 @@ -101,3 +104,31 @@ def text_time_to_seconds(text_time): pass return timedelta(hours=hours, minutes=minutes, seconds=seconds).total_seconds() + + +def get_eqgame_pid_list() -> list[int]: + """ + get list of process ID's for eqgame.exe, using psutil module + + Returns: + object: list of process ID's (in case multiple versions of eqgame.exe are somehow running) + """ + + pid_list = list() + for p in psutil.process_iter(['name']): + if p.info['name'] == 'eqgame.exe': + pid_list.append(p.pid) + return pid_list + + +def starprint(line: str) -> None: + """ + utility function to print with leading and trailing ** indicators + + Args: + line: line to be printed + + Returns: + None: + """ + print(f'** {line.rstrip():<100} **') diff --git a/helpers/config.py b/helpers/config.py index 9ed17d1..d067f72 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -7,6 +7,7 @@ data = {} _filename = '' +_char_name = '' def load(filename): @@ -49,10 +50,6 @@ def verify_settings(): data['general'].get('eq_log_dir', ''), '' ) - data['general']['char_name'] = get_setting( - data['general'].get('char_name', ''), - '' - ) data['general']['window_flush'] = get_setting( data['general'].get('window_flush', True), True diff --git a/helpers/logreader.py b/helpers/logreader.py index 7423d86..1c4d969 100644 --- a/helpers/logreader.py +++ b/helpers/logreader.py @@ -46,8 +46,7 @@ def _file_changed(self, changed_file): if changed_file != self._stats['log_file']: self._stats['log_file'] = changed_file char_name = os.path.basename(changed_file).split("_")[1] - config.data['general']['char_name'] = char_name - config.save() + config._char_name = char_name if not config.data['sharing']['player_name_override']: config.data['sharing']['player_name'] = char_name location_service.SIGNALS.config_updated.emit() diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index 5242995..700f4c9 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -3,11 +3,9 @@ import os import signal -import psutil from datetime import datetime -from helpers import Parser, config - +from helpers import Parser, config, get_eqgame_pid_list, starprint # # simple utility to prevent Everquest Death Loop @@ -191,7 +189,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: # does this line contain a proof of life - communication # this captures tells, say, group, auction, and shout channels - char_name = config.data['general']['char_name'] + char_name = config._char_name regexp = f'^(You told|You say|You tell|You auction|You shout|{char_name} ->)' m = re.match(regexp, trunc_line) if m: @@ -251,36 +249,3 @@ def deathloop_response(self) -> None: # purge any death messages from the list self.reset() - -################################################################################################# -# -# standalone functions -# - - -def get_eqgame_pid_list() -> list[int]: - """ - get list of process ID's for eqgame.exe, using psutil module - - Returns: - object: list of process ID's (in case multiple versions of eqgame.exe are somehow running) - """ - - pid_list = list() - for p in psutil.process_iter(['name']): - if p.info['name'] == 'eqgame.exe': - pid_list.append(p.pid) - return pid_list - - -def starprint(line: str) -> None: - """ - utility function to print with leading and trailing ** indicators - - Args: - line: line to be printed - - Returns: - None: - """ - print(f'** {line.rstrip():<100} **') From 25490a1fe496d10dde39f4cbadecfe2032a8286c Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Tue, 2 Aug 2022 16:24:57 -0700 Subject: [PATCH 08/21] PR comments: 1. deathloop.deaths and deathloop.seconds are initialized at every use, not just at instantiation 2. cleaned up processing of (timestamp, text) parameters in parsing functions 3. reverted version number to 0.6.4 4. cleaned up some additional empty functions in ParserWindow 5. cleaned up global.char_name usage --- helpers/config.py | 3 ++- helpers/logreader.py | 5 ++--- helpers/parser.py | 6 ------ nparse.py | 3 +-- parsers/deathloopvaccine.py | 38 ++++++++++++++----------------------- 5 files changed, 19 insertions(+), 36 deletions(-) diff --git a/helpers/config.py b/helpers/config.py index d067f72..e885d1e 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -5,9 +5,10 @@ from glob import glob import json +# global data data = {} _filename = '' -_char_name = '' +char_name = '' def load(filename): diff --git a/helpers/logreader.py b/helpers/logreader.py index 1c4d969..d1b84e4 100644 --- a/helpers/logreader.py +++ b/helpers/logreader.py @@ -45,10 +45,9 @@ def _file_changed_safe_wrap(self, changed_file): def _file_changed(self, changed_file): if changed_file != self._stats['log_file']: self._stats['log_file'] = changed_file - char_name = os.path.basename(changed_file).split("_")[1] - config._char_name = char_name + config.char_name = os.path.basename(changed_file).split("_")[1] if not config.data['sharing']['player_name_override']: - config.data['sharing']['player_name'] = char_name + config.data['sharing']['player_name'] = config.char_name location_service.SIGNALS.config_updated.emit() with open(self._stats['log_file'], 'rb') as log: log.seek(0, os.SEEK_END) diff --git a/helpers/parser.py b/helpers/parser.py index a24d632..1709c7b 100644 --- a/helpers/parser.py +++ b/helpers/parser.py @@ -164,9 +164,3 @@ def enterEvent(self, event): def leaveEvent(self, event): self._menu.setVisible(False) QFrame.leaveEvent(self, event) - - def shutdown(self): - pass - - def settings_updated(self): - pass diff --git a/nparse.py b/nparse.py index c71508c..5aba849 100644 --- a/nparse.py +++ b/nparse.py @@ -19,8 +19,7 @@ os.environ['QT_SCALE_FACTOR'] = str( config.data['general']['qt_scale_factor'] / 100) -# todo - set this to appropriate value -CURRENT_VERSION = '0.6.4-DLV' +CURRENT_VERSION = '0.6.4' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index 700f4c9..b3aa1f1 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -7,6 +7,7 @@ from datetime import datetime from helpers import Parser, config, get_eqgame_pid_list, starprint + # # simple utility to prevent Everquest Death Loop # @@ -42,9 +43,7 @@ def __init__(self): # parameters that define a deathloop condition, i.e. D deaths in T seconds, # with no player activity in the interim - # todo - make these configarable via the UI? - self.deathloop_deaths = config.data['deathloopvaccine']['deaths'] - self.deathloop_seconds = config.data['deathloopvaccine']['seconds'] + # todo - make the deathloop.deaths and deathloop.seconds values configarable via the UI? # list of death messages # this will function as a scrolling queue, with the oldest message at position 0, @@ -52,7 +51,7 @@ def __init__(self): # than deathloop_seconds have elapsed. The list is also flushed any time # player activity is detected (i.e. player is not AFK). # - # if/when the length of this list meets or exceeds deathloop_deaths, then + # if/when the length of this list meets or exceeds deathloop.deaths, then # the deathloop response is triggered self._death_list = list() @@ -99,14 +98,9 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: None: """ - # reconstruct the full logfile line - # this is a bit counter-intuitive, but the rest of the logic in this function was - # developed assuming the line was the full line, including the time stamp + trunc_line = text line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text - # cut off the leading date-time stamp info - trunc_line = line[27:] - # does this line contain a death message slain_regexp = r'^You have been slain' m = re.match(slain_regexp, trunc_line) @@ -129,7 +123,7 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: if len(self._death_list) > 0: # create a datetime object for this line, using the very capable datetime.strptime() - now = datetime.strptime(line[0:26], '[%a %b %d %H:%M:%S %Y]') + now = timestamp # now purge any death messages that are too old done = False @@ -144,7 +138,7 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: oldest_time = datetime.strptime(oldest_line[0:26], '[%a %b %d %H:%M:%S %Y]') elapsed_seconds = now - oldest_time - if elapsed_seconds.total_seconds() > self.deathloop_seconds: + if elapsed_seconds.total_seconds() > config.data['deathloopvaccine']['seconds']: # that death message is too old, purge it self._death_list.pop(0) starprint(f'DeathLoopVaccine: Death count = {len(self._death_list)}') @@ -164,11 +158,6 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: None: """ - # reconstruct the full logfile line - # this is a bit counter-intuitive, but the rest of the logic in this function was - # developed assuming the line was the full line, including the time stamp - line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text - # only do the proof of life checks if there are already some death messages in the list, else skip this if len(self._death_list) > 0: @@ -176,8 +165,8 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: # begin by assuming the player is AFK afk = True - # cut off the leading date-time stamp info - trunc_line = line[27:] + trunc_line = text + line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text # does this line contain a proof of life - casting regexp = r'^You begin casting' @@ -189,8 +178,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: # does this line contain a proof of life - communication # this captures tells, say, group, auction, and shout channels - char_name = config._char_name - regexp = f'^(You told|You say|You tell|You auction|You shout|{char_name} ->)' + regexp = f'^(You told|You say|You tell|You auction|You shout|{config.char_name} ->)' m = re.match(regexp, trunc_line) if m: # player is not AFK @@ -217,14 +205,17 @@ def deathloop_response(self) -> None: None: """ + deaths = config.data['deathloopvaccine']['deaths'] + seconds = config.data['deathloopvaccine']['seconds'] + # if the death_list contains more deaths than the limit, then trigger the process kill - if len(self._death_list) >= self.deathloop_deaths: + if len(self._death_list) >= deaths: starprint('---------------------------------------------------') starprint('DeathLoopVaccine - Killing all eqgame.exe processes') starprint('---------------------------------------------------') starprint('DeathLoopVaccine has detected deathloop symptoms:') - starprint(f' {self.deathloop_deaths} deaths in less than {self.deathloop_seconds} seconds, with no player activity') + starprint(f' {deaths} deaths in less than {seconds} seconds, with no player activity') # show all the death messages starprint('Death Messages:') @@ -248,4 +239,3 @@ def deathloop_response(self) -> None: # purge any death messages from the list self.reset() - From 82587289fa2bac586d19272baece0440c4252181 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Wed, 3 Aug 2022 19:57:49 -0700 Subject: [PATCH 09/21] added 'slice' to melee regexp --- parsers/deathloopvaccine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index b3aa1f1..b1c7e64 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -186,7 +186,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: starprint(f'DeathLoopVaccine: Player Not AFK: {line}') # does this line contain a proof of life - melee - regexp = r'^You( try to)? (hit|slash|pierce|crush|claw|bite|sting|maul|gore|punch|kick|backstab|bash)' + regexp = r'^You( try to)? (hit|slash|pierce|crush|claw|bite|sting|maul|gore|punch|kick|backstab|bash|slice)' m = re.match(regexp, trunc_line) if m: # player is not AFK From c046203be84930fe2c81bed3a003a5537e9c4af9 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Thu, 6 Oct 2022 20:14:42 -0700 Subject: [PATCH 10/21] updated coordinates of bank in Crystal caverns --- data/maps/map_files/Crystal_1.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/maps/map_files/Crystal_1.txt b/data/maps/map_files/Crystal_1.txt index b16d2de..070355c 100644 --- a/data/maps/map_files/Crystal_1.txt +++ b/data/maps/map_files/Crystal_1.txt @@ -1,4 +1,4 @@ -P -298.0000, 184.0000, 0.0000, 127, 64, 0, 2, Bank +P -298.0000, 192.0000, -384, 127, 64, 0, 2, Bank P -758.0000, 265.0000, 0.0000, 127, 64, 0, 2, Broken_Bridge P 939.1472, 589.2308, -538.4664, 127, 0, 0, 2, Queen P -692.0000, 176.0000, 0.0000, 127, 64, 0, 2, Waterfall From c22d5a119e86271d8b932d455c06c8e34de93942 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Fri, 7 Oct 2022 12:50:12 -0700 Subject: [PATCH 11/21] build of upstream/master working. Added Makefile. --- .gitignore | 1 + Makefile | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 60 insertions(+) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 2be6b01..77eefd9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ __pycache__/ # Distribution / packaging .Python .venv/ +.nparse.venv/ env/ venv/ bin/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8e9469f --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ + +PACKAGE=nparse + +############################################################################## +# do this while not in venv +venv: + python -m venv .$(PACKAGE).venv + +venv.clean: + rm -rfd .$(PACKAGE).venv + + + +############################################################################## +# do these while in venv +run: libs.quiet + py $(PACKAGE).py + + +# libs make targets ########################### +libs: requirements.txt + pip install -r requirements.txt + +libs.quiet: requirements.txt + pip install -q -r requirements.txt + +libs.clean: + pip uninstall -r requirements.txt + + +# exe make targets ########################### +exe: libs + pyinstaller nparse_py.spec + +exe.clean: + rm -rfd build + rm dist/$(PACKAGE).exe + + +# install make targets ########################### +#DIRS=dist/data dist/xxx +DIRS=dist/data dist/data/maps dist/data/spells +install: exe + $(shell mkdir $(DIRS)) + cp -r ./data/maps/* ./dist/data/maps/ + cp -r ./data/spells/* ./dist/data/spells/ + +install.clean: + rm -rfd $(DIRS) + + +# general make targets ########################### + +all: libs exe install + +all.clean: libs.clean exe.clean install.clean + +clean: all.clean diff --git a/requirements.txt b/requirements.txt index 293ede9..fce2175 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ colorhash pathvalidate psutil PyQt6-WebEngine +pyinstaller==5.3 From a209abe0985f20cc90a355cb0ffac3a8a52acde7 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Fri, 7 Oct 2022 21:43:28 -0700 Subject: [PATCH 12/21] added LogEventParser functionality --- helpers/__init__.py | 2 +- helpers/config.py | 13 +- helpers/logreader.py | 8 + helpers/parser.py | 2 +- nparse.py | 8 +- parsers/LogEvent.py | 641 ++++++++++++++++++++++++++++++++++++ parsers/LogEventParser.py | 114 +++++++ parsers/__init__.py | 4 +- parsers/deathloopvaccine.py | 20 +- parsers/maps/mapcanvas.py | 2 +- parsers/maps/mapdata.py | 2 +- 11 files changed, 798 insertions(+), 18 deletions(-) create mode 100644 parsers/LogEvent.py create mode 100644 parsers/LogEventParser.py diff --git a/helpers/__init__.py b/helpers/__init__.py index 75921fb..0f6ef12 100644 --- a/helpers/__init__.py +++ b/helpers/__init__.py @@ -128,7 +128,7 @@ def starprint(line: str) -> None: utility function to print with leading and trailing ** indicators Args: - line: line to be printed + line: text to be printed Returns: None: diff --git a/helpers/config.py b/helpers/config.py index 1dd868d..d8c2a54 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -7,8 +7,8 @@ # global data data = {} -_filename = '' -char_name = '' +_filename: str = '' +char_name: str = '' APP_EXIT = False @@ -302,6 +302,15 @@ def verify_settings(): 120 ) + # logeventparser + # todo - replace this general LogEventParser.toggled setting, with one for each LogEvent type + section_name = 'LogEventParser' + data[section_name] = data.get(section_name, {}) + data[section_name]['toggled'] = get_setting( + data[section_name].get('toggled', True), + True + ) + def get_setting(setting, default, func=None): try: diff --git a/helpers/logreader.py b/helpers/logreader.py index 121888c..635c442 100644 --- a/helpers/logreader.py +++ b/helpers/logreader.py @@ -1,12 +1,17 @@ import os import datetime from glob import glob +from typing import Optional from PyQt6.QtCore import QFileSystemWatcher, pyqtSignal from helpers import config from helpers import location_service from helpers import strip_timestamp +import parsers + +# pointer to the LogEventParser object, so this code can update the character name when the logfile changes +theLogEventParser: Optional[parsers.LogEventParser] = None class LogReader(QFileSystemWatcher): @@ -46,6 +51,9 @@ def _file_changed(self, changed_file): if changed_file != self._stats['log_file']: self._stats['log_file'] = changed_file config.char_name = os.path.basename(changed_file).split("_")[1] + # use the global pointer to update the charname + if theLogEventParser: + theLogEventParser.set_char_name(config.char_name) if not config.data['sharing']['player_name_override']: config.data['sharing']['player_name'] = config.char_name location_service.SIGNALS.config_updated.emit() diff --git a/helpers/parser.py b/helpers/parser.py index f5539e4..e3ba2b8 100644 --- a/helpers/parser.py +++ b/helpers/parser.py @@ -10,7 +10,7 @@ class Parser: def __init__(self): super().__init__() - self.name = 'Parser' + self.name = 'Parser' # could this be self.__class__.__name__ instead? self._visible = False def isVisible(self) -> bool: diff --git a/nparse.py b/nparse.py index c9b56a7..4a0a621 100644 --- a/nparse.py +++ b/nparse.py @@ -28,7 +28,7 @@ config.data['general']['qt_scale_factor'] / 100) -CURRENT_VERSION = '0.6.6-rc1' +CURRENT_VERSION = '0.6.6-logeventparser.rc1' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: @@ -76,13 +76,19 @@ def _load_parsers(self): "spells": parsers.Spells(), "discord": parsers.Discord(), "deathloopvaccine": parsers.DeathLoopVaccine(), + "LogEventParser": parsers.LogEventParser(), } self._parsers = [ self._parsers_dict["maps"], self._parsers_dict["spells"], self._parsers_dict["discord"], self._parsers_dict["deathloopvaccine"], + self._parsers_dict["LogEventParser"], ] + + # save a pointer to the LogEventParser so the logreader can update the char name when needed + logreader.theLogEventParser = self._parsers_dict['LogEventParser'] + for parser in self._parsers: if parser.name in config.data.keys() and 'geometry' in config.data[parser.name].keys(): g = config.data[parser.name]['geometry'] diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py new file mode 100644 index 0000000..d88857c --- /dev/null +++ b/parsers/LogEvent.py @@ -0,0 +1,641 @@ +import re +import time +from datetime import datetime, timezone, timedelta + + +# define some ID constants for the derived classes +LOGEVENT_BASE: int = 0 +LOGEVENT_VD: int = 1 +LOGEVENT_VT: int = 2 +LOGEVENT_YAEL: int = 3 +LOGEVENT_DAIN: int = 4 +LOGEVENT_SEV: int = 5 +LOGEVENT_CT: int = 6 +LOGEVENT_FTE: int = 7 +LOGEVENT_PLAYERSLAIN: int = 8 +LOGEVENT_QUAKE: int = 9 +LOGEVENT_RANDOM: int = 10 +LOGEVENT_ABC: int = 11 +LOGEVENT_GRATSS: int = 12 +LOGEVENT_TOD: int = 13 +LOGEVENT_GMOTD: int = 14 + + +######################################################################################################################### +# +# Base class +# +# Notes for the developer: +# - derived classes constructor should correctly populate the following fields, according to whatever event this +# parser is watching for: +# self.log_event_ID, a unique integer for each LogEvent class, to help the server side +# self.short_description, a text description, and +# self._search_list, a list of regular expression(s) that indicate this event has happened +# - derived classes can optionally override the _custom_match_hook() method, if special/extra parsing is needed, +# or if a customized self.short_description is desired. This method gets called from inside the standard matches() +# method. The default base case behavior is to simply return True. +# see Random_Event() class for a good example, which deals with the fact that Everquest /random events +# are actually reported in TWO lines of text in the log file +# +# - See the example derived classes in this file to get a better idea how to set these items up +# +# - IMPORTANT: These classes make use of the self.parsing_player field to embed the character name in the report. +# If and when the parser begins parsing a new log file, it is necessary to sweep through whatever list of LogEvent +# objects are being maintained, and update the self.parsing_player field in each LogEvent object, e.g. something like: +# +# for log_event in self.log_event_list: +# log_event.parsing_player = name +# +######################################################################################################################### + +# +# +class LogEvent: + """ + Base class that encapsulates all information about any particular event that is detected in a logfile + """ + + # + # + def __init__(self): + """ + constructor + """ + + # boolean for whether a LogEvent class should be checked. + # todo - get this from config file rather than hardcoding to True + self.parse = True + # self.parse = config.config_data.getboolean('LogEventParser', self.__class__.__name__) + + # modify these as necessary in child classes + self.log_event_ID = LOGEVENT_BASE + self.short_description = 'Generic Target Name spawn!' + self._search_list = [ + '^Generic Target Name begins to cast a spell', + '^Generic Target Name engages (?P[\\w ]+)!', + '^Generic Target Name has been slain', + '^Generic Target Name says', + '^You have been slain by Generic Target Name' + ] + + # the actual text from the logfile + self._matching_line = None + + # timezone info + self._local_datetime = None + self._utc_datetime = None + + # parsing player name and field separation character, used in the report() function + self.parsing_player = 'Unknown' + self.field_separator = '|' + self.eqmarker = 'EQ__' + + # + # + def matches(self, eq_datetime: datetime, text: str) -> bool: + """ + Check to see if the passed text matches the search criteria for this LogEvent + + Args: + eq_datetime: a datetime object constructed from the leading 26 characters of the line of text from the logfile + text: text of text from the logfile WITHOUT the EQ date-time stamp + + Returns: + True/False + + """ + # return value + rv = False + + if self.parse: + # # cut off the leading date-time stamp info + # trunc_line = text[27:] + + # walk through the target list and trigger list and see if we have any match + for trigger in self._search_list: + + # return value m is either None of an object with information about the RE search + m = re.match(trigger, text) + if m: + + # allow for any additional logic to be applied, if needed, by derived classes + if self._custom_match_hook(m, eq_datetime, text): + rv = True + + # save the matching text and set the timestamps + self._set_timestamps(eq_datetime, text) + + # return self.matched + return rv + + # + # send the re.Match info, as well as the datetime stamp and line text, in case the info is needed + def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> bool: + """ + provide a hook for derived classes to override this method and specialize the search + default action is simply return true + + Args: + m: re.Match object from the search + eq_datetime: a datetime object constructed from the leading 26 characters of the line of text from the logfile + text: text of text from the logfile WITHOUT the EQ date-time stamp + + Returns: + True/False if this is a match + """ + return True + + def _set_timestamps(self, eq_datetime: datetime, text: str) -> None: + """ + Utility function to set the local and UTC timestamp information, + using the EQ timestamp information present in the first 26 characters + of every Everquest log file text + + Args: + eq_datetime: a datetime object constructed from the leading 26 characters of the line of text from the logfile + text: text of text from the logfile WITHOUT the EQ date-time stamp + """ + + # save the matching line by reconstructing the entire EQ log line + self._matching_line = f"{eq_datetime.strftime('[%a %b %d %H:%M:%S %Y]')} " + text + + # the passed eq_datetime is a naive datetime, i.e. it doesn't know the TZ + # convert it to an aware datetime, by adding the local tzinfo using replace() + # time.timezone = offset of the local, non-DST timezone, in seconds west of UTC + local_tz = timezone(timedelta(seconds=-time.timezone)) + self._local_datetime = eq_datetime.replace(tzinfo=local_tz) + + # now convert it to a UTC datetime + self._utc_datetime = self._local_datetime.astimezone(timezone.utc) + + # print(f'{eq_datetime}') + # print(f'{self._local_datetime}') + # print(f'{self._utc_datetime}') + + # + # + def report(self) -> str: + """ + Return a text of text with all relevant data for this event, + separated by the field_separation character + + Returns: + str: single text with all fields + """ + rv = f'{self.eqmarker}{self.field_separator}' + rv += f'{self.parsing_player}{self.field_separator}' + rv += f'{self.log_event_ID}{self.field_separator}' + rv += f'{self.short_description}{self.field_separator}' + rv += f'{self._utc_datetime}{self.field_separator}' + # rv += f'{self._local_datetime}{self.field_separator}' + rv += f'{self._matching_line}' + return rv + + +######################################################################################################################### +# +# Derived classes +# + +class VesselDrozlin_Event(LogEvent): + """ + Parser for Vessel Drozlin spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_VD + self.short_description = 'Vessel Drozlin spawn!' + self._search_list = [ + '^Vessel Drozlin begins to cast a spell', + '^Vessel Drozlin engages (?P[\\w ]+)!', + '^Vessel Drozlin has been slain', + '^Vessel Drozlin says', + '^You have been slain by Vessel Drozlin' + ] + + +class VerinaTomb_Event(LogEvent): + """ + Parser for Verina Tomb spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_VT + self.short_description = 'Verina Tomb spawn!' + self._search_list = [ + '^Verina Tomb begins to cast a spell', + '^Verina Tomb engages (?P[\\w ]+)!', + '^Verina Tomb has been slain', + '^Verina Tomb says', + '^You have been slain by Verina Tomb' + ] + + +class MasterYael_Event(LogEvent): + """ + Parser for Master Yael spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_YAEL + self.short_description = 'Master Yael spawn!' + self._search_list = [ + '^Master Yael begins to cast a spell', + '^Master Yael engages (?P[\\w ]+)!', + '^Master Yael has been slain', + '^Master Yael says', + '^You have been slain by Master Yael' + ] + + +class DainFrostreaverIV_Event(LogEvent): + """ + Parser for Dain Frostreaver IV spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_DAIN + self.short_description = 'Dain Frostreaver IV spawn!' + self._search_list = [ + '^Dain Frostreaver IV engages (?P[\\w ]+)!', + '^Dain Frostreaver IV says', + '^Dain Frostreaver IV has been slain', + '^You have been slain by Dain Frostreaver IV' + ] + + +class Severilous_Event(LogEvent): + """ + Parser for Severilous spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_SEV + self.short_description = 'Severilous spawn!' + self._search_list = [ + '^Severilous begins to cast a spell', + '^Severilous engages (?P[\\w ]+)!', + '^Severilous has been slain', + '^Severilous says', + '^You have been slain by Severilous' + ] + + +class CazicThule_Event(LogEvent): + """ + Parser for Cazic Thule spawn + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_CT + self.short_description = 'Cazic Thule spawn!' + self._search_list = [ + '^Cazic Thule engages (?P[\\w ]+)!', + '^Cazic Thule has been slain', + '^Cazic Thule says', + '^You have been slain by Cazic Thule', + "Cazic Thule shouts 'Denizens of Fear, your master commands you to come forth to his aid!!" + ] + + +class FTE_Event(LogEvent): + """ + Parser for general FTE messages + overrides _additional_match_logic() for additional info to be captured + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_FTE + self.short_description = 'FTE' + self._search_list = [ + '^(?P[\\w ]+) engages (?P[\\w ]+)!' + ] + + # overload the default base class behavior to add some additional logic + def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> bool: + if m: + target_name = m.group('target_name') + playername = m.group('playername') + self.short_description = f'FTE: {target_name} engages {playername}' + return True + + +class PlayerSlain_Event(LogEvent): + """ + Parser for player has been slain + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_PLAYERSLAIN + self.short_description = 'Player Slain!' + self._search_list = [ + '^You have been slain by (?P[\\w ]+)' + ] + + +class Earthquake_Event(LogEvent): + """ + Parser for Earthquake + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_QUAKE + self.short_description = 'Earthquake!' + self._search_list = [ + '^The Gods of Norrath emit a sinister laugh as they toy with their creations' + ] + + +class Random_Event(LogEvent): + """ + Parser for Random (low-high) + overrides _additional_match_logic() for additional info to be captured + """ + + def __init__(self): + super().__init__() + self.playername = None + self.low = -1 + self.high = -1 + self.value = -1 + self.log_event_ID = LOGEVENT_RANDOM + self.short_description = 'Random!' + self._search_list = [ + '\\*\\*A Magic Die is rolled by (?P[\\w ]+)\\.', + '\\*\\*It could have been any number from (?P[0-9]+) to (?P[0-9]+), but this time it turned up a (?P[0-9]+)\\.' + ] + + # overload the default base class behavior to add some additional logic + def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> bool: + rv = False + if m: + # if m is true, and contains the playername group, this represents the first text of the random dice roll event + # save the playername for later + if 'playername' in m.groupdict().keys(): + self.playername = m.group('playername') + # if m is true but doesn't have the playername group, then this represents the second text of the random dice roll event + else: + self.low = m.group('low') + self.high = m.group('high') + self.value = m.group('value') + self.short_description = f'Random roll: {self.playername}, {self.low}-{self.high}, Value={self.value}' + rv = True + + return rv + + +class AnythingButComms_Event(LogEvent): + """ + Parser for Comms Filter + allows filtering on/off the various communications channels + """ + + def __init__(self): + super().__init__() + + # individual communication channel exclude flags, + # just in case wish to customize this later for finer control, for whatever reason + # this is probably overkill... + exclude_tell = True + exclude_say = True + exclude_group = True + exclude_auc = True + exclude_ooc = True + exclude_shout = True + exclude_guild = True + + # tells + # [Sun Sep 18 15:22:41 2022] You told Snoiche, 'gotcha' + # [Sun Sep 18 15:16:43 2022] Frostclaw tells you, 'vog plz' + # [Thu Aug 18 14:31:34 2022] Azleep -> Berrma: have you applied? + # [Thu Aug 18 14:31:48 2022] Berrma -> Azleep: ya just need someone to invite i believe + tell_regex = '' + if exclude_tell: + tell_regex1 = "You told [\\w]+, '" + tell_regex2 = "[\\w]+ tells you, '" + tell_regex3 = "[\\w]+ -> [\\w]+:" + # note that the tell_regexN bits filter tells IN, and then we surround it with (?! ) to filter then OUT + tell_regex = f'(?!^{tell_regex1}|{tell_regex2}|{tell_regex3})' + + # say + # [Sat Aug 13 15:36:21 2022] You say, 'lfg' + # [Sun Sep 18 15:17:28 2022] Conceded says, 'where tf these enchs lets goo' + say_regex = '' + if exclude_say: + say_regex1 = "You say, '" + say_regex2 = "[\\w]+ says, '" + say_regex = f'(?!^{say_regex1}|{say_regex2})' + + # group + # [Fri Aug 12 18:12:46 2022] You tell your party, 'Mezzed << froglok ilis knight >>' + # [Fri Aug 12 18:07:08 2022] Mezmurr tells the group, 'a << myconid reaver >> is slowed' + group_regex = '' + if exclude_group: + group_regex1 = "You tell your party, '" + group_regex2 = "[\\w]+ tells the group, '" + group_regex = f'(?!^{group_regex1}|{group_regex2})' + + # auction + # [Wed Jul 20 15:39:25 2022] You auction, 'wts Smoldering Brand // Crystal Lined Slippers // Jaded Electrum Bracelet // Titans Fist' + # [Wed Sep 21 17:54:28 2022] Dedguy auctions, 'WTB Crushed Topaz' + auc_regex = '' + if exclude_auc: + auc_regex1 = "You auction, '" + auc_regex2 = "[\\w]+ auctions, '" + auc_regex = f'(?!^{auc_regex1}|{auc_regex2})' + + # ooc + # [Sat Aug 20 22:19:09 2022] You say out of character, 'Sieved << a scareling >>' + # [Sun Sep 18 15:25:39 2022] Piesy says out of character, 'Come port with the Puffbottom Express and ! First-Class travel' + ooc_regex = '' + if exclude_ooc: + ooc_regex1 = "You say out of character, '" + ooc_regex2 = "[\\w]+ says out of character, '" + ooc_regex = f'(?!^{ooc_regex1}|{ooc_regex2})' + + # shout + # [Fri Jun 04 16:16:41 2021] You shout, 'I'M SORRY WILSON!!!' + # [Sun Sep 18 15:21:05 2022] Abukii shouts, 'ASSIST -- Cleric of Zek ' + shout_regex = '' + if exclude_shout: + shout_regex1 = "You shout, '" + shout_regex2 = "[\\w]+ shouts, '" + shout_regex = f'(?!^{shout_regex1}|{shout_regex2})' + + # guild + # [Fri Aug 12 22:15:07 2022] You say to your guild, 'who got fright' + # [Fri Sep 23 14:18:03 2022] Kylarok tells the guild, 'whoever was holding the chain coif for Pocoyo can nvermind xD' + guild_regex = '' + if exclude_guild: + guild_regex1 = "You say to your guild, '" + guild_regex2 = "[\\w]+ tells the guild, '" + guild_regex = f'(?!^{guild_regex1}|{guild_regex2})' + + # put them all together + # if we weren't interested in being able to filter only some channels, then this could + # all be boiled down to just + # (?!^[\\w]+ (told|tell(s)?|say(s)?|auction(s)?|shout(s)?|-> [\\w]+:)) + self.log_event_ID = LOGEVENT_ABC + self.short_description = 'Comms Filter' + self._search_list = [ + f'{tell_regex}{say_regex}{group_regex}{auc_regex}{ooc_regex}{shout_regex}{guild_regex}', + ] + + +class Gratss_Event(LogEvent): + """ + Parser for gratss messages + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_GRATSS + self.short_description = 'Possible Gratss sighting!' + self._search_list = [ + ".*gratss(?i)", + ] + + +class TOD_Event(LogEvent): + """ + Parser for tod messages + + Low fidelity version: if someone says 'tod' in one of the channels + High fidelity version: the phrase 'XXX has been slain', where XXX is one of the known targets of interest + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_TOD + self.short_description = 'Possible TOD sighting!' + self._search_list = [ + ".*tod(?i)", + '^(?P[\\w ]+) has been slain', + ] + + self.known_targets = [ + 'Kelorek`Dar', + 'Vaniki', + 'Vilefang', + 'Zlandicar', + 'Narandi the Wretched', + 'Lodizal', + 'Stormfeather', + 'Dain Frostreaver IV', + 'Derakor the Vindicator', + 'Keldor Dek`Torek', + 'King Tormax', + 'The Statue of Rallos Zek', + 'The Avatar of War', + 'Tunare', + 'Lord Yelinak', + 'Master of the Guard', + 'The Final Arbiter', + 'The Progenitor', + 'An angry goblin', + 'Casalen', + 'Dozekar the Cursed', + 'Essedera', + 'Grozzmel', + 'Krigara', + 'Lepethida', + 'Midayor', + 'Tavekalem', + 'Ymmeln', + 'Aaryonar', + 'Cekenar', + 'Dagarn the Destroyer', + 'Eashen of the Sky', + 'Ikatiar the Venom', + 'Jorlleag', + 'Lady Mirenilla', + 'Lady Nevederia', + 'Lord Feshlak', + 'Lord Koi`Doken', + 'Lord Kreizenn', + 'Lord Vyemm', + 'Sevalak', + 'Vulak`Aerr', + 'Zlexak', + 'Gozzrem', + 'Lendiniara the Keeper', + 'Telkorenar', + 'Wuoshi', + 'Druushk', + 'Hoshkar', + 'Nexona', + 'Phara Dar', + 'Silverwing', + 'Xygoz', + 'Lord Doljonijiarnimorinar', + 'Velketor the Sorcerer', + 'Guardian Kozzalym', + 'Klandicar', + 'Myga NE PH', + 'Myga ToV PH', + 'Scout Charisa', + 'Sontalak', + 'Gorenaire', + 'Vessel Drozlin', + 'Severilous', + 'Venril Sathir', + 'Trakanon', + 'Talendor', + 'Faydedar', + 'a shady goblin', + 'Phinigel Autropos', + 'Lord Nagafen', + 'Zordak Ragefire', + 'Verina Tomb', + 'Lady Vox', + 'A dracoliche', + 'Cazic Thule', + 'Dread', + 'Fright', + 'Terror', + 'Wraith of a Shissir', + 'Innoruuk', + 'Noble Dojorn', + 'Nillipuss', + 'Master Yael', + 'Sir Lucan D`Lere', + ] + + # overload the default base class behavior to add some additional logic + def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> bool: + rv = False + if m: + rv = True + # reset the description in case it has been set to something else + self.short_description = 'Possible TOD sighting!' + if 'target_name' in m.groupdict().keys(): + target_name = m.group('target_name') + if target_name in self.known_targets: + # since we saw the 'has been slain' message, + # change the short description to a more definitive TOD message + self.short_description = f'TOD {target_name}' + + return rv + + +class GMOTD_Event(LogEvent): + """ + Parser for GMOTD messages + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_GMOTD + self.short_description = 'GMOTD' + self._search_list = [ + '^GUILD MOTD:', + ] diff --git a/parsers/LogEventParser.py b/parsers/LogEventParser.py new file mode 100644 index 0000000..3b40820 --- /dev/null +++ b/parsers/LogEventParser.py @@ -0,0 +1,114 @@ +import logging +import logging.handlers + +from helpers import Parser +from parsers.LogEvent import * + + +# list of rsyslog (host, port) information +# todo - move this info to config file +# todo - add Stanvern server rsyslog hostname/port to this list +remote_list = [ + ('192.168.1.127', 514), + ('ec2-3-133-158-247.us-east-2.compute.amazonaws.com', 22514), +] + + +# +# create a global list of parsers +# +log_event_list = [ + VesselDrozlin_Event(), + VerinaTomb_Event(), + MasterYael_Event(), + DainFrostreaverIV_Event(), + Severilous_Event(), + CazicThule_Event(), + FTE_Event(), + PlayerSlain_Event(), + Earthquake_Event(), + Random_Event(), + AnythingButComms_Event(), + Gratss_Event(), + TOD_Event(), + GMOTD_Event(), +] + + +# +# todo - replace this with code to retrieve settings from config file +# todo - this code just turns them all on, except for the ABC parser +for log_entry in log_event_list: + # parse_dict[log_entry.__class__.__name] = True + if log_entry.log_event_ID == LOGEVENT_ABC: + log_entry.parse = False + else: + log_entry.parse = True + + +################################################################################################# +# +# class to do all the LogEvent work +# +class LogEventParser(Parser): + + # ctor + def __init__(self): + super().__init__() + + super().__init__() + # self.name = 'logeventparser' + self.name = self.__class__.__name__ + + # set up a custom logger to use for rsyslog comms + self.logger_list = [] + for (host, port) in remote_list: + eq_logger = logging.getLogger(f'{host}:{port}') + eq_logger.setLevel(logging.INFO) + + # create a handler for the rsyslog communications + log_handler = logging.handlers.SysLogHandler(address=(host, port)) + eq_logger.addHandler(log_handler) + self.logger_list.append(eq_logger) + + def set_char_name(self, name: str) -> None: + """ + override base class setter function to also sweep through list of parse targets + and set their parsing player names + + Args: + name: player whose log file is being parsed + """ + + # todo - gotta get nparse to call this whenever it switches log files/toons + global log_event_list + for log_event in log_event_list: + log_event.parsing_player = name + + # + # + # main parsing logic here + def parse(self, timestamp: datetime, text: str) -> None: + """ + Parse a single text from the logfile + + Args: + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile text + text: The text following the everquest timestamp + + Returns: + None: + """ + + # the global list of log_events + global log_event_list + + # check current text for matches in any of the list of Parser objects + # if we find a match, then send the event report to the remote aggregator + for log_event in log_event_list: + if log_event.matches(timestamp, text): + report_str = log_event.report() + + # send the info to the remote log aggregator + for logger in self.logger_list: + logger.info(report_str) diff --git a/parsers/__init__.py b/parsers/__init__.py index 5a7b78e..9aad818 100644 --- a/parsers/__init__.py +++ b/parsers/__init__.py @@ -2,4 +2,6 @@ from .maps import Maps # noqa: F401 from .spells import Spells # noqa: F401 from .discord import Discord # noqa: F401 -from .deathloopvaccine import DeathLoopVaccine # noqa: F401 +from .deathloopvaccine import DeathLoopVaccine # noqa: F401 +from .LogEventParser import LogEventParser # noqa: F401 +from .LogEvent import LogEvent # noqa: F401 diff --git a/parsers/deathloopvaccine.py b/parsers/deathloopvaccine.py index b1c7e64..c327203 100644 --- a/parsers/deathloopvaccine.py +++ b/parsers/deathloopvaccine.py @@ -71,10 +71,10 @@ def reset(self) -> None: # main parsing logic here def parse(self, timestamp: datetime, text: str) -> None: """ - Parse a single line from the logfile + Parse a single text from the logfile Args: - timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile text text: The text following the everquest timestamp Returns: @@ -91,7 +91,7 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: save the message for later processing Args: - timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile text text: The text following the everquest timestamp Returns: @@ -101,7 +101,7 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: trunc_line = text line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text - # does this line contain a death message + # does this text contain a death message slain_regexp = r'^You have been slain' m = re.match(slain_regexp, trunc_line) if m: @@ -122,7 +122,7 @@ def check_for_death(self, timestamp: datetime, text: str) -> None: # only do the list-purging if there are already some death messages in the list, else skip this if len(self._death_list) > 0: - # create a datetime object for this line, using the very capable datetime.strptime() + # create a datetime object for this text, using the very capable datetime.strptime() now = timestamp # now purge any death messages that are too old @@ -151,7 +151,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: check for "proof of life" indications the player is really not AFK Args: - timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile line + timestamp: A datetime.datetime object, created from the timestamp text of the raw logfile text text: The text following the everquest timestamp Returns: @@ -168,7 +168,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: trunc_line = text line = f'[{timestamp.strftime("%a %b %d %H:%M:%S %Y")}] ' + text - # does this line contain a proof of life - casting + # does this text contain a proof of life - casting regexp = r'^You begin casting' m = re.match(regexp, trunc_line) if m: @@ -176,7 +176,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: afk = False starprint(f'DeathLoopVaccine: Player Not AFK: {line}') - # does this line contain a proof of life - communication + # does this text contain a proof of life - communication # this captures tells, say, group, auction, and shout channels regexp = f'^(You told|You say|You tell|You auction|You shout|{config.char_name} ->)' m = re.match(regexp, trunc_line) @@ -185,7 +185,7 @@ def check_not_afk(self, timestamp: datetime, text: str) -> None: afk = False starprint(f'DeathLoopVaccine: Player Not AFK: {line}') - # does this line contain a proof of life - melee + # does this text contain a proof of life - melee regexp = r'^You( try to)? (hit|slash|pierce|crush|claw|bite|sting|maul|gore|punch|kick|backstab|bash|slice)' m = re.match(regexp, trunc_line) if m: @@ -230,7 +230,7 @@ def deathloop_response(self) -> None: for pid in pid_list: starprint(f'Killing process [{pid}]') - # for testing the actual kill process using simulated player deaths, uncomment the following line + # for testing the actual kill process using simulated player deaths, uncomment the following text # self._kill_armed = True if self._kill_armed: os.kill(pid, signal.SIGTERM) diff --git a/parsers/maps/mapcanvas.py b/parsers/maps/mapcanvas.py index 222f6d7..5de021f 100644 --- a/parsers/maps/mapcanvas.py +++ b/parsers/maps/mapcanvas.py @@ -577,7 +577,7 @@ def record_path_loc(self, loc): except Exception as e: print("Failed to write loc to pathfile: %s" % e) - # Also add line to the active map + # Also add text to the active map z_group = self._data.get_closest_z_group(loc[2]) color = MapData.color_transform(QColor(255, 0, 0)) map_line = QGraphicsPathItem() diff --git a/parsers/maps/mapdata.py b/parsers/maps/mapdata.py index c3f1595..8657ead 100644 --- a/parsers/maps/mapdata.py +++ b/parsers/maps/mapdata.py @@ -50,7 +50,7 @@ def _load(self): for line in f.readlines(): line_type = line.lower()[0:1] data = [value.strip() for value in line[1:].split(',')] - if line_type == 'l': # line + if line_type == 'l': # text x1, y1, z1, x2, y2, z2 = list(map(float, data[0:6])) self.raw['lines'].append(MapLine( x1=x1, From bb183c5ea3a52258bad81080f7613c5686ee61f7 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Fri, 7 Oct 2022 23:23:31 -0700 Subject: [PATCH 13/21] separated TOD into TODHI and TODLO for high and low fidelity events --- parsers/LogEvent.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py index d88857c..1ab179a 100644 --- a/parsers/LogEvent.py +++ b/parsers/LogEvent.py @@ -17,8 +17,9 @@ LOGEVENT_RANDOM: int = 10 LOGEVENT_ABC: int = 11 LOGEVENT_GRATSS: int = 12 -LOGEVENT_TOD: int = 13 +LOGEVENT_TODLO: int = 13 LOGEVENT_GMOTD: int = 14 +LOGEVENT_TODHI: int = 15 ######################################################################################################################### @@ -505,7 +506,7 @@ def __init__(self): ] -class TOD_Event(LogEvent): +class TOD_HighFidelity_Event(LogEvent): """ Parser for tod messages @@ -515,10 +516,9 @@ class TOD_Event(LogEvent): def __init__(self): super().__init__() - self.log_event_ID = LOGEVENT_TOD - self.short_description = 'Possible TOD sighting!' + self.log_event_ID = LOGEVENT_TODHI + self.short_description = 'TOD' self._search_list = [ - ".*tod(?i)", '^(?P[\\w ]+) has been slain', ] @@ -616,7 +616,7 @@ def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> b if m: rv = True # reset the description in case it has been set to something else - self.short_description = 'Possible TOD sighting!' + self.short_description = 'TOD' if 'target_name' in m.groupdict().keys(): target_name = m.group('target_name') if target_name in self.known_targets: @@ -627,6 +627,24 @@ def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> b return rv +class TOD_LowFidelity_Event(LogEvent): + + """ + Parser for tod messages + + Low fidelity version: if someone says 'tod' in one of the channels + High fidelity version: the phrase 'XXX has been slain', where XXX is one of the known targets of interest + """ + + def __init__(self): + super().__init__() + self.log_event_ID = LOGEVENT_TODLO + self.short_description = 'Possible TOD sighting!' + self._search_list = [ + ".*tod(?i)", + ] + + class GMOTD_Event(LogEvent): """ Parser for GMOTD messages From bff0fe7a812e1daa761ebcbc93eafd57a78c890a Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Fri, 7 Oct 2022 23:33:52 -0700 Subject: [PATCH 14/21] bug fix --- parsers/LogEventParser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsers/LogEventParser.py b/parsers/LogEventParser.py index 3b40820..7139878 100644 --- a/parsers/LogEventParser.py +++ b/parsers/LogEventParser.py @@ -30,8 +30,9 @@ Random_Event(), AnythingButComms_Event(), Gratss_Event(), - TOD_Event(), + TOD_LowFidelity_Event(), GMOTD_Event(), + TOD_HighFidelity_Event(), ] From a7e3ebbdef34c9b87c0512f48cc818861d349110 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 10 Oct 2022 23:45:11 -0700 Subject: [PATCH 15/21] configure LogEvents so they know which rsyslog server they are speaking to --- parsers/LogEvent.py | 34 ++++++++-- parsers/LogEventParser.py | 136 ++++++++++++++++++++++++++++---------- 2 files changed, 132 insertions(+), 38 deletions(-) diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py index 1ab179a..47fb919 100644 --- a/parsers/LogEvent.py +++ b/parsers/LogEvent.py @@ -1,5 +1,6 @@ import re import time +import logging from datetime import datetime, timezone, timedelta @@ -64,10 +65,13 @@ def __init__(self): """ # boolean for whether a LogEvent class should be checked. - # todo - get this from config file rather than hardcoding to True + # controlled by the ini file setting self.parse = True # self.parse = config.config_data.getboolean('LogEventParser', self.__class__.__name__) + # list of logging.Logger objects + self.logger_list = [] + # modify these as necessary in child classes self.log_event_ID = LOGEVENT_BASE self.short_description = 'Generic Target Name spawn!' @@ -79,7 +83,7 @@ def __init__(self): '^You have been slain by Generic Target Name' ] - # the actual text from the logfile + # the actual line from the logfile self._matching_line = None # timezone info @@ -177,11 +181,11 @@ def _set_timestamps(self, eq_datetime: datetime, text: str) -> None: # def report(self) -> str: """ - Return a text of text with all relevant data for this event, + Return a line of text with all relevant data for this event, separated by the field_separation character Returns: - str: single text with all fields + str: single line with all fields """ rv = f'{self.eqmarker}{self.field_separator}' rv += f'{self.parsing_player}{self.field_separator}' @@ -192,6 +196,27 @@ def report(self) -> str: rv += f'{self._matching_line}' return rv + # + # + def add_logger(self, logger: logging.Logger) -> None: + """ + Add logger to this object + + Args: + logger: a logging.Logger object + """ + self.logger_list.append(logger) + + # + # + def log_report(self) -> None: + """ + send the report for this event to every registered logger + """ + report_str = self.report() + for logger in self.logger_list: + logger.info(report_str) + ######################################################################################################################### # @@ -402,6 +427,7 @@ class AnythingButComms_Event(LogEvent): def __init__(self): super().__init__() + self.parse = False # individual communication channel exclude flags, # just in case wish to customize this later for finer control, for whatever reason diff --git a/parsers/LogEventParser.py b/parsers/LogEventParser.py index 7139878..6516c31 100644 --- a/parsers/LogEventParser.py +++ b/parsers/LogEventParser.py @@ -1,18 +1,56 @@ -import logging import logging.handlers from helpers import Parser from parsers.LogEvent import * - -# list of rsyslog (host, port) information -# todo - move this info to config file -# todo - add Stanvern server rsyslog hostname/port to this list -remote_list = [ - ('192.168.1.127', 514), - ('ec2-3-133-158-247.us-east-2.compute.amazonaws.com', 22514), -] - +# todo - store/retrieve this info from config file +# this info is stored in an ini file as shown: +# [rsyslog servers] +# server1 = 192.168.1.127:514 +# server2 = ec2-3-133-158-247.us-east-2.compute.amazonaws.com:22514 +# server3 = stanvern-hostname:port + +# todo - replace this global with info from config file +rsyslog_servers = {'server1': '192.168.1.127:514', + 'server2': 'ec2-3-133-158-247.us-east-2.compute.amazonaws.com:22514', + 'server3': 'stanvern-hostname:port'} + +# todo - store/retrieve this info from config file +# this info is stored in an ini file as shown: +# [LogEventParser] +# vesseldrozlin_event = True, server1, server2, server3 +# verinatomb_event = True, server1, server2, server3 +# dainfrostreaveriv_event = True, server1, server2, server3 +# severilous_event = True, server1, server2, server3 +# cazicthule_event = True, server1, server2, server3 +# masteryael_event = True, server1, server2, server3 +# fte_event = True, server1, server2, server3 +# playerslain_event = True, server1, server2, server3 +# earthquake_event = True, server1, server2, server3 +# random_event = True, server1, server2, server3 +# anythingbutcomms_event = False, server3 +# gratss_event = True, server1, server2, server3 +# tod_event = True, server1, server2, server3 +# gmotd_event = True, server1, server2, server3 +# tod_lowfidelity_event = True, server1, server2, server3 +# tod_highfidelity_event = True, server1, server2, server3 + +# todo - replace this global with info from config file +parser_config_dict = {'VesselDrozlin_Event': 'True, server1, server2, server3', + 'VerinaTomb_Event': 'True, server1, server2, server3', + 'DainFrostreaverIV_Event': 'True, server1, server2, server3', + 'Severilous_Event': 'True, server1, server2, server3', + 'CazicThule_Event': 'True, server1, server2, server3', + 'MasterYael_Event': 'True, server1, server2, server3', + 'FTE_Event': 'True, server1, server2, server3', + 'PlayerSlain_Event': 'True, server1, server2, server3', + 'Earthquake_Event': 'True, server1, server2, server3', + 'Random_Event': 'True, server1, server2, server3', + 'AnythingButComms_Event': 'False, server3', + 'Gratss_Event': 'True, server1, server2, server3', + 'TOD_LowFidelity_Event': 'True, server1, server2, server3', + 'GMOTD_Event': 'True, server1, server2, server3', + 'TOD_HighFidelity_Event': 'True, server1, server2, server3'} # # create a global list of parsers @@ -36,17 +74,6 @@ ] -# -# todo - replace this with code to retrieve settings from config file -# todo - this code just turns them all on, except for the ABC parser -for log_entry in log_event_list: - # parse_dict[log_entry.__class__.__name] = True - if log_entry.log_event_ID == LOGEVENT_ABC: - log_entry.parse = False - else: - log_entry.parse = True - - ################################################################################################# # # class to do all the LogEvent work @@ -62,15 +89,60 @@ def __init__(self): self.name = self.__class__.__name__ # set up a custom logger to use for rsyslog comms - self.logger_list = [] - for (host, port) in remote_list: - eq_logger = logging.getLogger(f'{host}:{port}') - eq_logger.setLevel(logging.INFO) + self.logger_dict = {} + # server_list = config.config_data.options('rsyslog servers') + # todo - get this info from config file rather than a global dict + server_list = rsyslog_servers.keys() + for server in server_list: + try: + # host_port_str = config.config_data.get('rsyslog servers', server) + host_port_str = rsyslog_servers[server] + host_port_list = host_port_str.split(':') + host = host_port_list[0] + # this will throw an exception if the port number isn't an integer + port = int(host_port_list[1]) + print(f'{host}, {port}') + + # create a handler for the rsyslog communications, with level INFO + # this will throw an exception if host:port are nonsensical + log_handler = logging.handlers.SysLogHandler(address=(host, port)) + eq_logger = logging.getLogger(f'{host}:{port}') + eq_logger.setLevel(logging.INFO) + + # log_handler.setLevel(logging.INFO) + eq_logger.addHandler(log_handler) + + # create a handler for console, and set level to 100 to ensure it is silent + # console_handler = logging.StreamHandler(sys.stdout) + # console_handler.setLevel(100) + # eq_logger.addHandler(console_handler) + self.logger_dict[server] = eq_logger + + except ValueError: + pass + + # print(self.logger_dict) + + # now walk the list of parsers and set their logging parameters + for log_event in log_event_list: + # log_settings_str = config.config_data.get('LogEventParser', log_event.__class__.__name__) + # todo - get this info from config file rather than a global dict + log_settings_str = parser_config_dict[log_event.__class__.__name__] + + log_settings_list = log_settings_str.split(', ') - # create a handler for the rsyslog communications - log_handler = logging.handlers.SysLogHandler(address=(host, port)) - eq_logger.addHandler(log_handler) - self.logger_list.append(eq_logger) + # the 0-th element is a true/false parse flag + if log_settings_list[0].lower() == 'true': + log_event.parse = True + else: + log_event.parse = False + + # index 1 and beyond are rsyslog servers + for n, elem in enumerate(log_settings_list): + if n != 0: + server = log_settings_list[n] + if server in self.logger_dict.keys(): + log_event.add_logger(self.logger_dict[server]) def set_char_name(self, name: str) -> None: """ @@ -108,8 +180,4 @@ def parse(self, timestamp: datetime, text: str) -> None: # if we find a match, then send the event report to the remote aggregator for log_event in log_event_list: if log_event.matches(timestamp, text): - report_str = log_event.report() - - # send the info to the remote log aggregator - for logger in self.logger_list: - logger.info(report_str) + log_event.log_report() From c2e269cc64cc4f955cb833146935ca458dc954cc Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Wed, 12 Oct 2022 12:37:30 -0700 Subject: [PATCH 16/21] bug fix for TODHI to only show known targets, not all targets --- parsers/LogEvent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py index 47fb919..1a89196 100644 --- a/parsers/LogEvent.py +++ b/parsers/LogEvent.py @@ -640,7 +640,6 @@ def __init__(self): def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> bool: rv = False if m: - rv = True # reset the description in case it has been set to something else self.short_description = 'TOD' if 'target_name' in m.groupdict().keys(): @@ -648,7 +647,8 @@ def _custom_match_hook(self, m: re.Match, eq_datetime: datetime, text: str) -> b if target_name in self.known_targets: # since we saw the 'has been slain' message, # change the short description to a more definitive TOD message - self.short_description = f'TOD {target_name}' + rv = True + self.short_description = f'TOD, High Fidelity: {target_name}' return rv From 67a50a9dc1f0523071cf03d8e87dec27f0198189 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Thu, 13 Oct 2022 15:23:51 -0700 Subject: [PATCH 17/21] small edits to config file management --- parsers/LogEventParser.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parsers/LogEventParser.py b/parsers/LogEventParser.py index 6516c31..b01acbd 100644 --- a/parsers/LogEventParser.py +++ b/parsers/LogEventParser.py @@ -30,7 +30,6 @@ # random_event = True, server1, server2, server3 # anythingbutcomms_event = False, server3 # gratss_event = True, server1, server2, server3 -# tod_event = True, server1, server2, server3 # gmotd_event = True, server1, server2, server3 # tod_lowfidelity_event = True, server1, server2, server3 # tod_highfidelity_event = True, server1, server2, server3 @@ -101,7 +100,7 @@ def __init__(self): host = host_port_list[0] # this will throw an exception if the port number isn't an integer port = int(host_port_list[1]) - print(f'{host}, {port}') + # print(f'{host}, {port}') # create a handler for the rsyslog communications, with level INFO # this will throw an exception if host:port are nonsensical From fe780f2242940381caa6020a46b8b36338d33254 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Thu, 13 Oct 2022 15:32:31 -0700 Subject: [PATCH 18/21] version number label --- nparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nparse.py b/nparse.py index 4a0a621..407f076 100644 --- a/nparse.py +++ b/nparse.py @@ -28,7 +28,7 @@ config.data['general']['qt_scale_factor'] / 100) -CURRENT_VERSION = '0.6.6-logeventparser.rc1' +CURRENT_VERSION = '0.6.6-rl-rc1' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: From 06e309c1f48ad051927de46ea80d16492e40b12f Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Thu, 13 Oct 2022 20:30:22 -0700 Subject: [PATCH 19/21] version number --- .gitignore | 2 +- nparse.py | 2 +- parsers/LogEvent.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 77eefd9..6ccc695 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ # PyCharm - .idea/ .vscode/ working/ +.vbump.ini # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/nparse.py b/nparse.py index 407f076..6518807 100644 --- a/nparse.py +++ b/nparse.py @@ -28,7 +28,7 @@ config.data['general']['qt_scale_factor'] / 100) -CURRENT_VERSION = '0.6.6-rl-rc1' +CURRENT_VERSION = '0.6.6-rc1-rl7' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py index 1a89196..8dfcd74 100644 --- a/parsers/LogEvent.py +++ b/parsers/LogEvent.py @@ -103,7 +103,7 @@ def matches(self, eq_datetime: datetime, text: str) -> bool: Args: eq_datetime: a datetime object constructed from the leading 26 characters of the line of text from the logfile - text: text of text from the logfile WITHOUT the EQ date-time stamp + text: the line of text from the logfile WITHOUT the EQ date-time stamp Returns: True/False From 70965c1b0c0b5456f169327c59a1ea8f4ac96785 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Sat, 15 Oct 2022 22:44:37 -0700 Subject: [PATCH 20/21] load correct websocket library, and fix the TOD regex --- nparse.py | 2 +- parsers/LogEvent.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nparse.py b/nparse.py index 6518807..af9cf7b 100644 --- a/nparse.py +++ b/nparse.py @@ -28,7 +28,7 @@ config.data['general']['qt_scale_factor'] / 100) -CURRENT_VERSION = '0.6.6-rc1-rl7' +CURRENT_VERSION = '0.6.6-rc1-rlog13' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: diff --git a/parsers/LogEvent.py b/parsers/LogEvent.py index 8dfcd74..3d25df0 100644 --- a/parsers/LogEvent.py +++ b/parsers/LogEvent.py @@ -667,7 +667,7 @@ def __init__(self): self.log_event_ID = LOGEVENT_TODLO self.short_description = 'Possible TOD sighting!' self._search_list = [ - ".*tod(?i)", + ".*tod(?i) |.* tod(?i)\\'$", ] diff --git a/requirements.txt b/requirements.txt index fce2175..c2accfd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pyqt6 websockets -websocket-client +websocket-client==1.3.3 requests colorhash pathvalidate From 12b9724a2d9c8b7ce8e8fa2b37f564b51f8e1573 Mon Sep 17 00:00:00 2001 From: Elliott Jackson Date: Mon, 17 Oct 2022 22:04:33 -0700 Subject: [PATCH 21/21] merge upstream/master changes that fix websocket library issue --- nparse.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nparse.py b/nparse.py index af9cf7b..21656c0 100644 --- a/nparse.py +++ b/nparse.py @@ -28,7 +28,7 @@ config.data['general']['qt_scale_factor'] / 100) -CURRENT_VERSION = '0.6.6-rc1-rlog13' +CURRENT_VERSION = '0.6.6-rc1-rlog14' if config.data['general']['update_check']: ONLINE_VERSION = get_version() else: diff --git a/requirements.txt b/requirements.txt index c2accfd..fce2175 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pyqt6 websockets -websocket-client==1.3.3 +websocket-client requests colorhash pathvalidate