diff --git a/.github/workflows/linterExercise.yml b/.github/workflows/linterExercise.yml new file mode 100644 index 0000000..a024f05 --- /dev/null +++ b/.github/workflows/linterExercise.yml @@ -0,0 +1,46 @@ +# This is a basic workflow to work with Github's Actions + +name: LinterExercise + +# Controls when the action will run. +on: + # + # + # Triggers the workflow on push or pull request events but only for the ALPython3 branch + #push: + # branches: [ ALPython3 ] + + # Allows you to run this workflow manually from the Github Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + super-lint: + # Steps represent a sequence of tasks that will be executed as part of the job + # Name the Job + name: Lint code base + # Set the type of machine to run on + runs-on: ubuntu-latest + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Runs a single command using the runners shell + # + not convinced by first version added ';' + - name: Run a multi-line script + run: echo Hello from Github Actions !!; + pwd; + ls -ltha + + # Runs the Super-Linter action + - name: Run Super-Linter + uses: github/super-linter@v3 + # this defines environment variables made available to the + # script/action being executed + env: + DEFAULT_BRANCH: ALPython3 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/pythonDBExercise.yml b/.github/workflows/pythonDBExercise.yml new file mode 100644 index 0000000..7f732db --- /dev/null +++ b/.github/workflows/pythonDBExercise.yml @@ -0,0 +1,363 @@ +name: Test python package logtools with Mysql database + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # First idea: perform tests on the VM using the default mysql provided + # by ubuntu-latest + # + # Documentation at : + # https://github.com/actions/virtual-environments/blob/main/images \ + # /linux/Ubuntu2004-README.md + # with following details: + # Mysql (client): MySQL 8.0.25 + # MySQL Server : (user:root password:root) + # MySQL service is disabled by default. Use the following command as + # a part of your job to start the service: + # 'sudo systemctl start mysql.service' + # + # Also included sqlite3 3.31.1 + # + # ------------------------------------------------------------ + # + # Note(s): + # There is a more security minded example at + # https://github.com/marketplace/actions/setup-mysql + # which runs a Mysql in a container after shutting down the default mysql + # Source cloned at ~/src/Github/mysql-action/. + # + # For substituting secrets, this scripts does it with ad-hoc shell scripts + # another way would be to use the following Action + # https://github.com/marketplace/actions/variable-substitution + # ------------------------------------------------------------ + +on: + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + # Defining environment variables at this level + env: + # Warning: this file contains secrets that need to be handled + # in the bash script. For Github repository, the secrets + # are accessible by any collaborator. + ROOT_DB_PASS: ${{ secrets.MYSQL_DB_ROOT_PASS }} + ROOT_ORIG_DB_PASS: ${{ secrets.MYSQL_DB_ROOT_ORIG_PASS }} + USER_DB_PASS: ${{ secrets.MYSQL_DB_USER_PASS }} + # + # These are used in bash inline code, for consistency they are + # global in 'build' step + # + createUFile: aux/.secrets/startCreateUser.sql + rootCNFDefault: aux/.secrets/rootDft.cnf + rootCNFFile: aux/.secrets/root.cnf + userCNFFile: aux/.secrets/user.cnf + # + # Information used for filling the database + createTblFile: aux/testData/ActionsMysqlTest/mysql/createLogtoolsTables.sql + fillTblFile: aux/testData/ActionsMysqlTest/mysql/dumpDBData.sql + + # From: + # https://docs.github.com/en/actions/reference/environment-variables + # ``The workspace directory is a copy of your repository if your + # workflow uses the actions/checkout action. If you don't use + # the actions/checkout action, the directory denoted in + # GITHUB_WORKSPACE will be empty.'' + # + # So using a handwired version (TB CHECKED LATER, not modular) + # There are constraints on src_Dpath_path value, see below + # + src_Dpath_path: /home/runner/work/logtools/logtools/aux/lib/srcDpath + AMTS_PATH: /home/runner/work/logtools/logtools/aux/testData/ActionsMysqlTest/scripts + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + # ------------------------------------------------------------ + # Later see about caching pip dependencies + # https://docs.github.com/en/actions/guides/building-and-testing-python + # ------------------------------------------------------------ + + - name: Ascertain configuration and add some paths + # + # Collect information concerning $HOME (for ~/.logtoolsrc setup) + # i.e. location of file loaded from Github/ + # Also show $LANG ( Locale may be relevant in processing of dates ) + # + run: | + if [ ! -d ${src_Dpath_path} ] ; then + mkdir -p ${src_Dpath_path} + fi + echo Home dir $HOME + echo files at this location + ls -ltha $HOME + echo "+++++" + echo Working dir $(pwd) + echo Files at this location: + ls -ltha + echo "+++++" + echo HOME: $HOME + echo LANG: $LANG + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here :: ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/logtools/logtools + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Prepare for Python installs + # We start by upgrading pip, if needed (?) + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Install full featured dpath from my own github repo + # Need to load dpath from my repository; otherwise the setup.py + # will install the default version without the special features + + # Repo information: origin/AL-addRegexp + # https://github.com/AlainLich/dpath-python.git + # Using https://github.com/actions/checkout for all required + # information on actions/checkout@v2 + + uses: actions/checkout@v2 + with: + # Repository name with owner. For example, actions/checkout + # Default: ${{ github.repository }} + repository: 'AlainLich/dpath-python' + + # The branch, tag or SHA to checkout. When checking out the + # repository that triggered a workflow, this defaults to the + # reference or SHA for that event. Otherwise, uses the default + # branch. + ref: 'AL-addRegexp' + + # Absolute path to place the repository. There seems to be + # constraints, as I received an error "... is not under ..." + # See https://github.com/actions/checkout + # and more specifically https://github.com/actions/checkout/issues/197 + # `` Today the path input is the only way to change the directory + # where the code is cloned. Also there is a restriction that it + # must be under GITHUB_WORKSPACE. '' + # See Feature Request https://github.com/actions/checkout/issues/327 + # + path: ${{ env.src_Dpath_path }} + + - name: Install dpath with config + + run: | + cd ${src_Dpath_path} + echo Files in the dpath directory $(pwd) just loaded + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed dpath successfully in ${src_Dpath_path} + ls -ltha ${src_Dpath_path} + else + echo FAILURE IN INSTALLATION OF dpath + fi + + - name: Install logtools with config.py + if: always() + run: | + pwd + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed logtools successfully + else + echo FAILURE IN INSTALLATION OF logtools + fi + + + - name: Install other python dependencies + if: always() + # Take into account the requirements and setup.py built in + # dependencies. + # Also needed for testing purposes: pymysql + # + run: | + echo installing logtools requirements + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # + echo installing extra test requirements + testsDBReq=${AMTS_PATH}/testsDB.requirements + if [ -f ${testsDBReq} ]; then pip install -r ${testsDBReq} ; fi + + - name: Install .logtoolsrc + if: always() + # + # File ~/.logtoolsrc is used by the package to setup defaults + # - absence results in failure; + # - file name may be overriden by env. variable LOGTOOLS_RC + # - it contains filepaths and secret passwd and thus must be adapted + # to actuals filepaths + # -file may name other config files that must + # exist with proper contents. In particular this test + # uses ~/.logtools.d/tradivariantParsing.txt + + run: | + echo executing: "aux/dot_logtoolsrc.sh $(pwd) ${HOME}/.logtoolsrc ${HOME}/.logtools.d" + aux/dot_logtoolsrc.sh $(pwd) "${HOME}/.logtoolsrc" "${HOME}/.logtools.d" + echo Files in home dir: "$HOME" + ls -ltha ~ + echo Files in home dir: "$HOME/.logtools.d" + ls -ltha ~/.logtools.d + + - name: Confirm list of available docker images + if: always() + # + # These are to be used as source of data via inspect in script + # testDB.sh + # + run: | + echo "Available docker images" + docker image ls -a + echo "----------------" + + - name: Launch and configure Mysql server + if: always() + # This starts the service and preloads authorizations so that non + # root access may occur. Also need to: + # 1/ set user privileges, by editing and running + # aux/testData/ActionsMysqlTest/mysql/startCreateUser.sql + # + # 2/ preload any data that we want to be in database tables + # a) table: define fields and preload data + # b) table: define fields, should be able to start cold (test) + # + run: | + sudo systemctl start mysql.service + # + # + # Sleep so that the mysql server gets a chance to get started + # service mysqld status. To get some info : + # + # 1) check if the service got started + # systemctl status cron.service + # + # 2) this shows possible values + # systemctl --state=help + # 3) this lists sockets being waited on/serviced + # systemctl list-sockets --all + # + sleepStep=1 + sleep ${sleepStep} + # wait a bit for the mysql server to get started if required + for i in {1..60} ; do + activity=$(systemctl is-active mysql) + if [ ${activity} = "active" ] ; then + echo Mysql service active at iter $i : $(expr ${i} \* ${sleepStep} )s + break + fi + sleep ${sleepStep} + done + # + # Now run the configuration script and inject the data with mysql + # client. + # Arg1: BASEPATH + # Arg2: location of generated file (with secrets) (create a hidden + # directory to contain secrets)! + maskOrig=$(umask -p) + umask 0077 + mkdir aux/.secrets + # reset umask + $maskOrig + # + # This will prepare for user creation in Mysql, set privileges, + # the port is 3306. + aux/mysql_config.sh ${createUFile} \ + ${rootCNFFile} ${rootCNFDefault} \ + ${userCNFFile} + # + # Concerning passwords man (mysql client ) says: + # To avoid giving the password on the command line, use an option + # file. See Section 6.1.2.1, “End-User Guidelines for Password + # Security”. + # + # Selected to hide the default root password in yet another secret! + # WATCH OUT: here must use root default password + # ALSO NOTE that the options used to specify option files must + # precede any other options + mysql --defaults-file=${rootCNFDefault} -u root <${createUFile} + # + # now, inject the table structure and data connecting as a user + # with the secret password. + # + cat ${createTblFile} ${fillTblFile} | \ + mysql --defaults-file=${userCNFFile} -u user + + + - name: Non Database Tests + if: always() + # Test Z : runs logtools/test/test_logtools.py + # Test A : runs a test scenario + # Test + : runs all tests + # + # These are redundant, ordered by increasing dependency on the + # environment: Z:self contained, A requires 1 data file, + # + # Parameter -s ZZZ indicates logging level; INFO, ERROR and DEBUG + # are of interest + # Environment variables: documented in aux/manTests.py + # Note that if DEBUG not set, some tests suppress error output + # + # Configuration: + # This uses ~/.logtoolsrc + # and this file may name other config files that must + # exist with proper contents. In particular this test + # uses ~/.logtools.d/tradivariantParsing.txt + # + run: | + export ZTESTPGM="logtools/test/test_logtools.py" + export TESTDATADIR="aux/testData" + export DEBUG="-" + python aux/manTests.py Z -s INFO + python aux/manTests.py A -s ERROR + python aux/manTests.py + -s INFO + + - name: Mysql Tests + if: always() + # + # testDB.sh with parameter sets + # + run: | + export PATH=${PATH}:${AMTS_PATH} + echo PATH "${PATH}" + hash -r + # ---------------------------------- + # warn user: testsDB.sh may pull a docker image + echo testsDB may pull docker image to be inspected to generate JSON + # ---------------------------------- + testsDB.sh -s sDBJ + testsDB.sh -s sDBOP + + - name: Dump Values in database + if: always() + # Using SELECT sql verb rather than mysqldump, since we are only + # getting the log back, and this seems easier to read. + # Also, only table 'tree' gets modified / filled + run: | + echo "USE logfromtool; SELECT * FROM tree;" | + mysql --defaults-file=${userCNFFile} -u user + echo dumped logfromtool.tree + + - name: Close mysql server + if: always() + # + # This will probably occur anyways, or not be required since VM + # gets destroyed. This way, I experiment to check how this works.... + run: | + sudo systemctl status mysql.service + sudo systemctl stop mysql.service diff --git a/.github/workflows/pythonExercise.yml b/.github/workflows/pythonExercise.yml new file mode 100644 index 0000000..5f7f21a --- /dev/null +++ b/.github/workflows/pythonExercise.yml @@ -0,0 +1,195 @@ +name: Test python package logtools without database + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Script performs basic test of the Python-3 version of the package + # including added functionality. + # ------------------------------------------------------------ + +on: #[push] + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + # Defining environment variables at this level + env: + # From: + # https://docs.github.com/en/actions/reference/environment-variables + # ``The workspace directory is a copy of your repository if your + # workflow uses the actions/checkout action. If you don't use + # the actions/checkout action, the directory denoted in + # GITHUB_WORKSPACE will be empty.'' + # + # So using a handwired version (TB CHECKED LATER, not modular) + # There are constraints on src_Dpath_path value, see below + # + src_Dpath_path: /home/runner/work/logtools/logtools/aux/lib/srcDpath + + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + # ------------------------------------------------------------ + # Later see about caching pip dependencies + # https://docs.github.com/en/actions/guides/building-and-testing-python + # ------------------------------------------------------------ + + - name: Ascertain configuration and add some paths + # + # Collect information concerning $HOME (for ~/.logtoolsrc setup) + # i.e. location of file loaded from Github/ + # Also show $LANG ( Locale may be relevant in processing of dates ) + # + run: | + if [ ! -d ${src_Dpath_path} ] ; then + mkdir -p ${src_Dpath_path} + fi + echo Home dir $HOME + echo files at this location + ls -ltha $HOME + echo "+++++" + echo Working dir $(pwd) + echo Files at this location: + ls -ltha + echo "+++++" + echo HOME: $HOME + echo LANG: $LANG + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here :: ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/logtools/logtools + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Prepare for Python installs + # We start by upgrading pip, if needed (?) + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Install full featured dpath from my own github repo + # Need to load dpath from my repository; otherwise the setup.py + # will install the default version without the special features + + # Repo information: origin/AL-addRegexp + # https://github.com/AlainLich/dpath-python.git + # Using https://github.com/actions/checkout for all required + # information on actions/checkout@v2 + + uses: actions/checkout@v2 + with: + # Repository name with owner. For example, actions/checkout + # Default: ${{ github.repository }} + repository: 'AlainLich/dpath-python' + # The branch, tag or SHA to checkout. When checking out the + # repository that triggered a workflow, this defaults to the + # reference or SHA for that event. Otherwise, uses the default + # branch. + ref: 'AL-addRegexp' + # Absolute path to place the repository. There seems to be + # constraints, as I received an error "... is not under ..." + # See https://github.com/actions/checkout + # and more specifically https://github.com/actions/checkout/issues/197 + # `` Today the path input is the only way to change the directory + # where the code is cloned. Also there is a restriction that it + # must be under GITHUB_WORKSPACE. '' + # See FR https://github.com/actions/checkout/issues/327 + # + path: ${{ env.src_Dpath_path }} + + - name: Install dpath with config + + run: | + cd ${src_Dpath_path} + echo Files in the dpath directory $(pwd) just loaded + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed dpath successfully in ${src_Dpath_path} + ls -ltha ${src_Dpath_path} + else + echo FAILURE IN INSTALLATION OF dpath + fi + + - name: Install logtools with config.py + if: always() + run: | + pwd + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed logtools successfully + else + echo FAILURE IN INSTALLATION OF logtools + fi + + - name: Install other python dependencies + if: always() + # Take into account the requirements and setup.py built in + # dependencies. + # Do not install : pymysql + # + run: | + echo installing logtools requirements + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # + echo installing extra test requirements + testsDBReq=${AMTS_PATH}/testsNoDB.requirements + if [ -f ${testsDBReq} ]; then pip install -r ${testsDBReq} ; fi + + - name: Install .logtoolsrc + # + # File ~/.logtoolsrc is used by the package to setup defaults + # absence results in failure; it contains filepaths and + # must be adapted to actual filepaths + # + run: | + echo executing: aux/dot_logtoolsrc.sh $(pwd) "${HOME}/.logtoolsrc" \ + "${HOME}/.logtools.d" + aux/dot_logtoolsrc.sh $(pwd) "${HOME}/.logtoolsrc" \ + "${HOME}/.logtools.d" + echo Files in home dir: "$HOME" + ls -ltha ~ + + - name: Test + # + # Test Z : runs logtools/test/test_logtools.py + # Test A : runs a test scenario + # Test + : runs all tests + # + # These are redundant, ordered by increasing dependency on the + # environment: Z:self contained, A requires 1 data file, + # + + + # Parameter -s ZZZ indicates logging level; INFO, ERROR and DEBUG + # are also of interest + # + # Environment variables: documented in aux/matTests.py + + # Configuration: + # This uses ~/.logtoolsrc + # and this file may name other config files that must + # exist with proper contents. In particular this test + # uses ~/.logtools.d/tradivariantParsing.txt + # + + # Note that if DEBUG not set, some tests suppress error output + run: | + export ZTESTPGM="logtools/test/test_logtools.py" + export TESTDATADIR="aux/testData" + export DEBUG="-" + python aux/manTests.py Z -s INFO + python aux/manTests.py A -s ERROR + python aux/manTests.py + -s INFO diff --git a/.github/workflows/pythonSQLiteExercise.yml b/.github/workflows/pythonSQLiteExercise.yml new file mode 100644 index 0000000..b9a277c --- /dev/null +++ b/.github/workflows/pythonSQLiteExercise.yml @@ -0,0 +1,263 @@ +name: Test python package logtools with SQLite database + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Documentation at : + # https://github.com/actions/virtual-environments/blob/main/images \ + # /linux/Ubuntu2004-README.md + # + # with sqlite3 3.31.1 included (no loading required!) + # + # ------------------------------------------------------------ + +on: + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + # Defining environment variables at this level + env: + # Information used for creating and filling the database + fillTblFile: aux/testData/ActionsMysqlTest/SQLite/dumpDBData.sql + + sqlite_file: /tmp/sqlite_file.db + + # From: + # https://docs.github.com/en/actions/reference/environment-variables + # ``The workspace directory is a copy of your repository if your + # workflow uses the actions/checkout action. If you don't use + # the actions/checkout action, the directory denoted in + # GITHUB_WORKSPACE will be empty.'' + # + # So using a handwired version (TB CHECKED LATER, not modular) + # There are constraints on src_Dpath_path value, see below + # + src_Dpath_path: /home/runner/work/logtools/logtools/aux/lib/srcDpath + AMTS_PATH: /home/runner/work/logtools/logtools/aux/testData/ActionsMysqlTest/scripts + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + + - name: Ascertain configuration and add some paths + # + # Collect information concerning $HOME (for ~/.logtoolsrc setup) + # i.e. location of file loaded from Github/ + # Also show $LANG ( Locale may be relevant in processing of dates ) + # + run: | + if [ ! -d ${src_Dpath_path} ] ; then + mkdir -p ${src_Dpath_path} + fi + echo Home dir $HOME + echo files at this location + ls -ltha $HOME + echo "+++++" + echo Working dir $(pwd) + echo Files at this $(pwd) location: + ls -ltha + echo "+++++" + echo HOME: $HOME + echo LANG: $LANG + + - name: Prepare for Python installs + # We start by upgrading pip, if needed (?) + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Install full featured dpath from my own github repo + # Need to load dpath from my repository; otherwise the setup.py + # will install the default version without the special features + + # Repo information: origin/AL-addRegexp + # https://github.com/AlainLich/dpath-python.git + # Using https://github.com/actions/checkout for all required + # information on actions/checkout@v2 + + uses: actions/checkout@v2 + with: + # Repository name with owner. For example, actions/checkout + # Default: ${{ github.repository }} + repository: 'AlainLich/dpath-python' + + # The branch, tag or SHA to checkout. When checking out the + # repository that triggered a workflow, this defaults to the + # reference or SHA for that event. Otherwise, uses the default + # branch. + ref: 'AL-addRegexp' + + # Absolute path to place the repository. There seems to be + # constraints, as I received an error "... is not under ..." + # See https://github.com/actions/checkout + # and more specifically https://github.com/actions/checkout/issues/197 + # `` Today the path input is the only way to change the directory + # where the code is cloned. Also there is a restriction that it + # must be under GITHUB_WORKSPACE. '' + # See Feature Request https://github.com/actions/checkout/issues/327 + # + path: ${{ env.src_Dpath_path }} + + - name: Install dpath with config + + run: | + cd ${src_Dpath_path} + echo Files in the dpath directory $(pwd) just loaded + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed dpath successfully in ${src_Dpath_path} + ls -ltha ${src_Dpath_path} + else + echo FAILURE IN INSTALLATION OF dpath + fi + + - name: Install logtools with config.py + if: always() + run: | + pwd + ls -ltha + python setup.py install + if [ $? -eq 0 ] ; then + echo Installed logtools successfully + else + echo FAILURE IN INSTALLATION OF logtools + fi + + + - name: Install other python dependencies + if: always() + # Take into account the requirements and setup.py built in + # dependencies. + # Expressly not loaded: pymysql + # + run: | + echo installing logtools requirements + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + # + echo NOT installing more test requirements possible issue cryptography + # testsDBReq=${AMTS_PATH}/testsDB.requirements + # if [ -f ${testsDBReq} ]; then pip install -r ${testsDBReq} ; fi + + - name: Install .logtoolsrc + if: always() + # + # File ~/.logtoolsrc is used by the package to setup defaults + # - absence results in failure; + # - it contains filepaths and secret passwd and thus must be adapted + # to actuals filepaths + # -file may name other config files that must + # exist with proper contents. In particular this test + # uses ~/.logtools.d/tradivariantParsing.txt + # + # Note: although it is the sqlite version of .logtoolsrc, we keep + # the default file path, to avoid adding specifying a + # file by means of setting env. variable LOGTOOLS_RC + + run: | + echo executing: "aux/dot_logtoolsrc_sqlite.sh $(pwd) ${HOME}/.logtoolsrc \ + ${HOME}/.logtools.d" ${sqlite_file} + aux/dot_logtoolsrc_sqlite.sh $(pwd) "${HOME}/.logtoolsrc" \ + "${HOME}/.logtools.d" ${sqlite_file} + echo Files in home dir: "$HOME" + ls -ltha ~ + echo Files in home dir: "$HOME/.logtools.d" + ls -ltha ~/.logtools.d + + - name: Confirm list of available docker images + if: always() + # + # These are to be used as source of data via inspect in script + # testDB.sh + # + run: | + echo "Available docker images" + docker image ls -a + echo "----------------" + + - name: Create and fill SQLite DB + if: always() + run: | + # + # This will create and fill as required the DB tables + # + sqlite3 ${sqlite_file} < ${fillTblFile} + + + - name: Non Database Tests + if: always() + # Test Z : runs logtools/test/test_logtools.py + # Test A : runs a test scenario + # Test + : runs all tests + # + # These are redundant, ordered by increasing dependency on the + # environment: Z:self contained, A requires 1 data file, + # + # Parameter -s ZZZ indicates logging level; INFO, ERROR and DEBUG + # are of interest + # Environment variables: documented in aux/manTests.py + # Note that if DEBUG not set, some tests suppress error output + # + # Configuration: + # This uses ~/.logtoolsrc + # and this file may name other config files that must + # exist with proper contents. In particular this test + # uses ~/.logtools.d/tradivariantParsing.txt + # + run: | + export ZTESTPGM="logtools/test/test_logtools.py" + export TESTDATADIR="aux/testData" + export DEBUG="-" + python aux/manTests.py Z -s INFO + python aux/manTests.py A -s ERROR + python aux/manTests.py + -s INFO + + - name: SQLite Tests + if: always() + # + # Using testDB.sh with parameter sets + # The content of the (join|dbOp)_connect_string in .logtoolsrc + # will ensure that SQLite is used + run: | + export PATH=${PATH}:${AMTS_PATH} + echo PATH "${PATH}" + hash -r + # ---------------------------------- + # warn user: testsDB.sh may pull a docker image + echo testsDB may pull docker image to be inspected to generate JSON + # ---------------------------------- + testsDB.sh -s sDBJ + testsDB.sh -s sDBOP + + - name: Dump Values in database + if: always() + # Only table 'tree' gets modified / filled, so that we do not + # need to print the rest + # + # Watch out:sqlite3 won't return an error code if we omit the + # ${sqlite_file} parameter. The documentation says: + # `` If no database file is specified on the command-line, a + # temporary database is created, then deleted when the "sqlite3" + # program exits. '' + # + # There is a disparity of version for sqlite3 between the dev system + # and the VM: .mode markdown is not supported + # + run: | + echo sqlite3 version: $(sqlite3 --version) + sqlite3 ${sqlite_file} < +Adam Ever-Hadani ##original version +Alain Lichnewsky ##derived, ported to Python 3, extended + diff --git a/ChangeLog b/ChangeLog index 7d8ba9b..65b1e9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +version 0.8.7p3 + Remove SQLsoup + ** Check these things from git logs and markdown files + Added parsers + Improved selection capabilities version 0.8.6 Tweaks to the 'percentiles' utility - Added optional -N argument to force tailing specific number of lines (samples), removed hard-coded 10000 lines diff --git a/FORMAT-CHEATSHEET.md b/FORMAT-CHEATSHEET.md new file mode 100644 index 0000000..783f640 --- /dev/null +++ b/FORMAT-CHEATSHEET.md @@ -0,0 +1,73 @@ +# Cheatsheet on format selection + +## TOC + + +- [Cheatsheet on format selection](#cheatsheet-on-format-selection) + - [TOC](#toc) + - [Goal and summary](#goal-and-summary) + - [Date](#date) + + + +## Goal and summary + +According to the look of a log, which format to look for first! + + - Devised as a self-help, will expand this to cover tests + and use cases + +## Date + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
program default-r/var/log/authMar 19 09:17:26
/var/log/kern.logSun Mar 21 12:59:57 2021
/var/log/apport.log
lastWed Dec 30 11:51
Common Log Format--parser CommonLogFormat10/Oct/2000:13:55:36 -0700
RFC5424--parser SyslogRFC5424Mysql2021-03-19T08:21:10.226123Z
2016-01-15T01:00:43Z
Github2021-03-22T16:50:52.6302582Z
+ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..093f10c --- /dev/null +++ b/Makefile @@ -0,0 +1,53 @@ +# Makefile in logtools directory +# +# Added to assist with some maintenance tasks +# +# ---------------------------------------------------------------------- +# + +# +# Assist with building table of contents in Markdown +# +SHELL= /bin/bash +PYTHON= python3 +MD-TOC-PGM= $(PYTHON) -m md_toc --in-place +MD-TOC-OPTIONS= github + +PYDIR= logtools + +MARKDOWNFILES=$(wildcard *.md) + + + +.PHONY: clean showMd + +clean: + - rm -fr logtools/__pycache__ + - rm logtools/*.pyc + +showMd: + grip -b README.md + + + +# +# Build table of contents in Markdown +# + +.PHONY: md-tocs +md-tocs: $(MARKDOWNFILES) + $(MD-TOC-PGM) $(MD-TOC-OPTIONS) $? + +# +# Generate html documentation +# +.PHONY: doxy +doxy: + doxygen aux/logtools.doxyfile + +# +# Generate UML with pyreverse +# +.PHONY: doUML +doUML: + $(MAKE) -C aux diff --git a/PYTHON3-ADDITIONS.md b/PYTHON3-ADDITIONS.md new file mode 100644 index 0000000..76bbd18 --- /dev/null +++ b/PYTHON3-ADDITIONS.md @@ -0,0 +1,284 @@ + +# Additional features in Python-3 port + +## TOC + + +- [Additional features in Python-3 port](#additional-features-in-python-3-port) + - [TOC](#toc) + - [Note](#note) + - [Intent](#intent) + - [Added functionality](#added-functionality) + - [Database](#database) + - [Non functional changes](#non-functional-changes) + - [Ongoing development](#ongoing-development) + - [Findings](#findings) + + + + +## Note + +These are short notes documenting functions added when porting to Python-3, +still to be considered experimental + +## Intent + +The idea was to make the package more usable in my own context, which concerns +Linux/Ubuntu or Docker produced logs. + +Other aspects: + - emphasis on `fr_FR.UTF-8` locale + - most logs are in `RSYSLOG_TraditionalFileFormat` + - found `logjoin` interesting, but needed to change the DB interface: + since I am running under `Mysql 8.0` which requires an up to date + version of the DB interface `SQLAlchemy`, I removed `SQLSoup` + and converted to current version of`SQLAlchemy`, with interest on the ORM. This + is used in `logjoin` and in the (added) `logdb`. + +## Added functionality + +1. added CLI flags to customize the level of logging; not customizable from + ~/.logtoolsrc (implementation propagates slowly to various entries) + - adds flags `-s` (symbolic designation like ̀`-s DEBUG`) `-n` (numerical + like `-n 10`). Symbolic names and values as in Python `logging` module. + + ``` + logparse --parser JSONParserPlus -f "{(Id|ContainerC.*)}" -s DEBUG + logparse --parser JSONParserPlus -f "{(Id|ContainerC.*)}" -s ERROR + ``` + +1. "INI" configuration files : + CLI commands will look at the environment variable LOGTOOLS_RC, which should + contain a comma separated list of configuration files in INI format; if not set, + defaut "/etc/logtools.cfg:~/.logtoolsrc" is used. + ConfigParser is used in ExtendedInterpolation mode. + +1. Added RFC 5424 parser `SyslogRFC5424`, here ̀-f`supports symbolic field selection. + This addition makes use of package `syslog_rfc5424_parser` from + https://github.com/EasyPost/syslog-rfc5424-parser. + + ``` + cat testData/testRFC5424.data | logparse --parser SyslogRFC5424 -f hostname -s INFO + ``` + + Field names can be found running with flag `-s DEBUG` + +1. Added a facility to handle logs specified by "traditional templates" in the style + defined at url https://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html + - This is not compliant to said specification/standard, only in same style + - This is implemented (mostly) in `logtools/parsers2.py` + - Parametrized by a textual description of templates in `logtools.parsers2.templateDef` + where templates are defined in the style of traditional templates. + + This can be extended by adding more templates. (However not all features are + suppported) + + When extending, you may either: + 1. add special function definition + in `logtools/parsers2.py`like : + ``` + def FileFormat(): + return RSysTradiVariant("FileFormat") + ``` + and add some import instructions in `logtools/_parse.py`. + 2. use configuration definable field descriptions as explained below. + + + these formats forms a set extensible via configuration file(s): + requires adding to `~/.logtoolsrc` + ~~~ + RSysTradiVariant] + HOME: + file: ${HOME}/.logtools.d/ + ~~~ + + and in the extension file: + + ~~~ + #$template TestA,"%HOSTNAME%" + #$template TFFA,"%TIMESTAMP% %HOSTNAME%\n" + #$template TFFB,"%TIMESTAMP% %HOSTNAME% %syslogtag%" + #$template TFFC1,"%TIMESTAMP%" + ~~~ + + + + Fields can be selected by field number or by using the property (symbol) appearing + in the template (for now, look for `templateDefs` in `logtools/parsers2.py`). + + - Usable with /var/log/{syslog,kern.log,auth.log} (on the Ubuntu Linux configuration + described) and ̀docker container logs` (one of my use cases) + + Examples of use: + ``` + cat /var/log/syslog | logparse --parser TraditionalFileFormat -f 3 -s ERROR + cat /var/log/syslog | logparse --parser TraditionalFileFormat -f msg -s ERROR + cat /var/log/syslog | logparse --parser TraditionalFileFormat -f TIMESTAMP \ + -s ERROR + cat testData/testCommonLogFmt.data | logparse --parser CommonLogFormat \ + -s INFO -f4 + ``` + +1. Parsing of JSON Data + + supported in `logjoin` via frontend interface + +1. Changes in `logjoin`. + + + Data Base interface + + removed all `SQLSoup` dependencies; moving to `SQLAlchemy`. This option + was selected because `SQLSoup` was found incompatible with newer versions + of `SQLAlchemy`, making `Mysql 8.0` unavailable. + + supports `Mysql 8.0` and `SQLite` + + tested with both `Mysql 8.0` and ̀`SQLite` + + + added `frontend` filtering + + frontend may return JSON, internally as (recursive dict of list)... + + permits to ingest JSON data (see tests) + + + added a (parameterizable) notion of `backend` + + performs database operations depending of flow of data streaming out of `frontend` + + collect SQL Schema (tables) information from SQL server after connection, + this may be used to simplify emitting more sophisticated queries + + generates SELECT SQL (for parameters in CLI or `~/.logtoolsrc`) using + SQLAlchemy using either ORM or Core. + + + support of additional data base transactions: + + table creation (for now table wired in, ... this may become parametrizable) + + able to use SQLAlchemy via non ORM connections and ORM sessions + + able to use "mapped table" + + +1. Changes in backend filtering + + + started with difficulties when piping into applications expecting ready to parse + JSON structures as lines + + distinguish form of returned data (string, binary, with filtering) + - not systematic yet + +1. Changes to ̀flattenjson`: + + + added --raw to dispense with utf-8 encoding, therefore can pipe to xargs, + did not change default for backwards compatibility. This could have a CLI + like `--output-encoding`in `logjoin` + +1. Addition of `logdb` + + This focuses on the performance of ORM supported database operations, + the first example performing additions and updates based on the incoming flow + of data from the logs. + + This uses quite general SQLAlchemy ORM tools, using Classes `SQLAlchemyDbOperator` + and `SQLAlchemyORMBackBase`, for providing + DB related operations, implemented in CLI program `logdb`, like: + - filling a database table from data collected from a log stream + + this is implemented with some generality in classes `SQLAlchemyORMBackBase` + and `SQLAlchemyDbOperator` + + this feature is parametrized by CLI or config file; for now a single + case has been implemented: + + + `NestedTreeDbOperator ` (parm. `dbOperator: SQLAlcDbOp` + in `~/.logtoolsrc`) which provides for entering in a tree-like database schema the + content of JSON (here from a Docker `inspect` command): + ~~~ + {'TOP': [ {'IPAM': + {'Driver': 'default', + 'Options': {}, + 'Config': [ { 'Subnet': '172.19.0.0/16', + 'Gateway': '172.19.0.1'}]}, + 'Containers': {'4cf07fa937da4b6ad844b3a6ca22a8eb167dc503544becda8789774a6e605c5b': + {'Name': 'mysql1', + 'EndpointID': '7edb7b1e866957f0de6150c73c78b70717a6bb833794009509f1002dec622f41', + 'MacAddress': '02:42:ac:13:00:02', + 'IPv4Address': '172.19.0.2/16', + 'IPv6Address': ''}, + '517acb5295aba5cc347d7aae88b525abf7faef26f5c481f2b82636b0619a192b': + {'Name': 'mysql-workbench', + 'EndpointID': '5255215cc33826d3b90973aaed786d3b3e2b620ddbe6a837c6ae060fc8ce61e4', + 'MacAddress': '02:42:ac:13:00:03', + 'IPv4Address': '172.19.0.3/16', + 'IPv6Address': ''}}}]} + ~~~ + + results in inserting the following data in the database: + + ~~~ + id,parent_id,name,nval + 1000,NULL,RootNode,2021-05-17T15:10:25.642125 + 1001,1000,RangeStart,1010 + 1010,1000,**TOP**,TREEPOS.LIST + 1011,1010,0,TREEPOS.TOP + 1012,1011,Driver,default + 1013,1011,Options,TREEPOS.EMPTY_TREE + 1014,1011,Config,TREEPOS.LIST + 1015,1014,0,TREEPOS.TOP + 1016,1015,Subnet,172.19.0.0/16 + 1017,1015,Gateway,172.19.0.1 + 1018,1010,1,TREEPOS.TOP + 1019,1018,4cf07fa937da4b6ad844b3a6ca22a8eb167dc503544becda8789774a6e605c5b,TREEPOS.TOP + 1020,1019,Name,mysql1 + 1021,1019,EndpointID,7edb7b1e866957f0de6150c73c78b70717a6bb833794009509f1002dec622f41 + 1022,1019,MacAddress,02:42:ac:13:00:02 + 1023,1019,IPv4Address,172.19.0.2/16 + 1024,1019,IPv6Address, + 1025,1018,517acb5295aba5cc347d7aae88b525abf7faef26f5c481f2b82636b0619a192b,TREEPOS.TOP + 1026,1025,Name,mysql-workbench + 1027,1025,EndpointID,5255215cc33826d3b90973aaed786d3b3e2b620ddbe6a837c6ae060fc8ce61e4 + 1028,1025,MacAddress,02:42:ac:13:00:03 + 1029,1025,IPv4Address,172.19.0.3/16 + 1030,1025,IPv6Address, + 1031,1000,RangeEnd,1030 + ~~~ + + + Still **in progress**: + + in above example, the selection mechanism has removed 'IPAM' + and 'Containers' keys.... which is obviously not a terrific idea!! + + structurally, all methods dependent on the DB schema should be moved to + file ̀ext_db.py`. + +## Database + + The ̀`logjoin` and `logdb` programs and the corresponding modules make + use of a database. Since the interface is done through SQLAlchemy, + selection of the database interface is done in the ̀`connect_string` + specified either by means of the INI configuration file or command + line argument (or class constructor). + + The interface (and therefore the programs) is (are) (apparently) mostly + independent of the database. However: + 1. we found that SQLite was not able to use `enum` type data, + which was OK for Mysql. + 1. We provide ̀class `SQLite_Data_Standardize` (and base class + DB_Data_Standardize ) in `db_ext.py` to adjust + for not supported data types. **If more numerous interfaces + need data adaptation, the selection mechanism will need to be + generalized in class `NestedTreeDbOperator` CTOR.** + + Exemples are included in Github Actions CI tests, in ̀̀ + `.github/workflows/`. + +## Non functional changes + +1. removed most of the `eval` function calls by function `utils.getObj`, + mostly to avoid code injection and hard to understand error messages. + +## Ongoing development + +The issues here should be resolved before this file hits Github! + + +Improvements TBD(?): + 1. change --raw in flattenjson to use `--output-encoding`like `logjoin` + 1. document aux/parseRsyslogd.py, which is a **test tool** for users that + add parsing variants via the `RSysTradiVariant` key in `.logtoolsrc` + + +### Findings + 1. flattenjson has (very) limited functionality... + + added --raw to dispense with utf-8 encoding, therefore can pipe to xargs, + did not change default for backwards compatibility. + + 1. Questions about transitionning in SQLAlchemy, found + https://docs.sqlalchemy.org/en/14/tutorial/index.html, have been resolved + by keeping some Core based classes and moving to ORM for `_db.py` and CLI + `logdb` + + + diff --git a/PYTHON3-README.md b/PYTHON3-README.md new file mode 100644 index 0000000..9284c6e --- /dev/null +++ b/PYTHON3-README.md @@ -0,0 +1,322 @@ +# Python-3 port + +## TOC + + +- [Python-3 port](#python-3-port) + - [TOC](#toc) + - [Notes](#notes) + - [Intent and issues found](#intent-and-issues-found) + - [Moving to sqlalchemy](#moving-to-sqlalchemy) + - [Added functionality](#added-functionality) + - [Test and operations](#test-and-operations) + - [Test and operative environment](#test-and-operative-environment) + - [Installation](#installation) + - [Configuration](#configuration) + - [First experiments](#first-experiments) + - [More examples and tests](#more-examples-and-tests) + + + +## Notes + +These are short notes documenting the Python-3 port, which is still to +be considered experimental. + +### Intent and issues found + +The idea was to do a straightforward port to Python-3, since I wanted to +use the package with the native Python-3 on my Ubuntu 20.10 Linux. + +Following issues were encountered: + +- parts concerning `geoIP` have not been ported or tested, therefore are deemed + not functional + +- the `sqlsoup` package uses features of `SQLAlchemy` ( `MapperExtension` ) which have + been deprecated (then suppressed since version 0.7; see + https://docs.sqlalchemy.org/en/13/orm/deprecated.html ). + + Workable but abandonned (after `commit f63db028b5f14cd2aa` on May 5, 2021) story: + + Current versions are `sqlsoup 0.9.1` and `SQLAlchemy 1.4.0b3`. + + Make the port to require specific older + versions. This suggests operate under `virtualenv`; ` setup.py` had been changed + accordingly. + + A file [requirements.txt](./requirements.txt) has been added to document this, and + can be used with `pip3`. + + Current approach: + + Removed all `SQLsoup` dependencies + + Used `SQLAlchemy` instead (`version >= 1.4.13`) + + Attempting to use the higher abstraction ORM, for query preparation and + possibly exploitation. + +- the package's usage of `datetime.strptime` in my locale `"fr_FR.UTF-8"`was found + problematic ( `testQps` fails when parsing date `11/Oct/2000:14:01:14 -0700` + which is fine in my locale) : + disabled statements `locale.setlocale(locale.LC_ALL, "")`in + `_qps.py` and `_plot.py`. + The directory `aux` has been added with script `testStrptime.py` to test + under different locales. Multiple locales where tested in the CI (mostly + Github's Actions framework) + +- when experimenting the use of ̀logjoin` with `mysql` + + installing the `PyMySQL` module proved tricky (On Linux/Ubuntu): + 1. pip3 install mysql-connector + 1. pip3 install PyMySQL + 1. needed to add the following in `__init__.py`: + ~~~ + import pymysql + pymysql.install_as_MySQLdb() + ~~~ + + + For connecting, the string form of the URL is + > dialect[+driver]://user:password@host/dbname[?key=value..], + + Example: `mysql://scott:tiger@hostname/dbname` + where + + dialect is a database name such as mysql, oracle, postgresql, etc., and + + driver the name of a DBAPI, such as psycopg2, pyodbc, cx_oracle, etc. + + Alternatively, the URL can be an instance of URL class. + + this parameter may be set in ~/.logtoolsrc + + + this is required when using SQLAlchemy + + + Finally decided to use an up to date version of SQLAlchemy to handle + MySQL (8.xx). Updated version of PyMySQL: see above about package selection + + +### Moving to sqlalchemy + +1. Documentation for sqlalchemy: https://docs.sqlalchemy.org/en/14/index.html + + a more tutorial thing at + https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls + +1. Used SQLAlchemy as follows: + + minor support of Core tools sufficient to run logjoin + + more extensive support of ORM, in logjoin and logdb (added). + + ORM to be prioritized for future developments + + + only tested with mysql as database server. + +## Added functionality + + See [PYTHON3-ADDITIONS](./PYTHON3-ADDITIONS.md). + In [FORMAT-CHEATSHEET](./FORMAT-CHEATSHEET.md), some help for parser format selection + +## Test and operations + +### Test and operative environment + This is really setting up the development / maintenance environment; I developped + the habit to use virtualenv with precisely versionned Python packages when + trying to operate with SQLSoup, since this entailed compatiblility issues. + It is probably not necessary now. + + - setup a `virtualenv`environment, requiring Python 3.8.6, ( or whatever version + you want to use. Python 3.8.6 happens to be the native Python-3 on my system, + with which all development and tests have been done: + ``` + cd + virtualenv -p 3.8.6 . + ``` + - populate it according to [requirements.txt](./requirements.txt) + - development and maintenance of the package are all performed under this environment + +### Installation + + This is a pure Python package. At the time of this writing, PRs have not been + accepted which would enable a straightforward installation via `Pypi`. + + The full functionality requires also a particular version of package `dpath-python`, + adapted to accept regular expressions in both "bash/globbing" format at Python's `re`; + otherwise it should degrade to accepting only "bash/globbing" format, but some + of our tests will fail. *If this functionality becomes available, the process below + will be simplified...* + + + This process will install both; it is used in test + https://github.com/AlainLich/logtools/blob/ALPython3/.github/workflows/pythonDBExercise.yml + + - setup or activate the `virtualenv` environment using the + [requirements.txt](./requirements.txt) file. + + - copy or clone our versions of the packages: + 1. https://github.com/AlainLich/dpath-python (branch AL-addRegexp) + 2. https://github.com/AlainLich/logtools (branch ̀ALPython3`) + + 3. For each of these (in order) + - change directory to the package (where `setup.py` and source code are found) + - run `setup.py` using the python interpreter in the `virtualenv` environment: + + This is done as follows: +``` + # establish virtualenv + . venvSandBox/bin/activate + # keep track of wd and cd to source + v=`pwd` + + pushd ${dpath_dir} + $v/venvSandBox/bin/python3 setup.py install + popd + + pushd ${logtools_dir} + $v/venvSandBox/bin/python3 setup.py install + popd +``` + + +### Configuration + - establish a `~/.logtoolsrc` file, which will used for setting + parameters or defaults. There are many fields needed from this configuration, + some of which can be overriden from the command line. + + for doing this, a model can be found in + [aux/dot_logtoolsrc.sh](./aux/dot_logtoolsrc.sh) , which + creates test configuration file(s) for CI in Github's Action system + + + if using the database interface, login information will need to be + provided in the files; this allows for keeping password secret. + + + if some files are named in the above configuration, they will need to + be provided too. + + - For the following configuration files, named in `~/.logtoolsrc`, + create and populate them, + empty files are acceptable if there is no blacklist: +``` +touch bots_hosts.txt # File designated in ~/.logtoolsrc +touch bots_useragents.txt # File designated in ~/.logtoolsrc +``` + + - in the case of the .yml script example , file + ` ~/.logtools.d/tradivariantParsing.txt ` + needs to be provided since it is named in `~/.logtoolsrc` + (see example, where the files are generated for password + inclusion by script [aux/dot_logtoolsrc.sh](./aux/dot_logtoolsrc.sh) ). + + +### First experiments + + - `filterbots`: + 1. extract log entries corresponding to some ̀sudo` uses. Notice that + we are using the `-s` flag to define the level of output + ``` +gunzip --stdout /var/log/auth.log.*.gz | \ +cat /var/log/auth.log - | \ +filterbots -s ERROR -r ".*sudo:(?P[^:]+).*COMMAND=(?P\S+).*" --print + ``` + + - filter : + + + - `logmerge` + 1. Merges several logs sorting a field defined by delimiter (1 char) and field number: + ``` + logmerge -d'-' -f1 /var/log/auth.log /var/log/syslog | \ + grep -i upload + ``` + + 2. Use a parser for merging + - the following supposes Apache Common Log Format + see http://httpd.apache.org/docs/1.3/logs.html#accesslog), examples shown in + [MORE-DOC.md](./MORE-DOC.md) : + ``` + logmerge --parser CommonLogFormat -f4 /var/log/auth.log /var/log/syslog + ``` + - by default `format='%h %l %u %t "%r" %>s %b'` + + - `logparse` + 1. Parses according to parser format, select output field: + example extracts the date-time): + ``` + cat testData/testCommonLogFmt.data | logparse --parser CommonLogFormat -s INFO -f4 + ``` + + 2. Same, selects multiple fields, for some reason only 1 line is output + + ``` + cat testData/testCommonLogFmt.data| logparse --parser CommonLogFormat -s DEBUG -f1,4 + ``` + + 3. Added RFC 5424 parser `SyslogRFC5424`, here ̀-f`supports symbolic field selection + + ``` + cat testData/testRFC5424.data | logparse --parser SyslogRFC5424 -f hostname -s INFO + ``` + + Field names can be found running with flag `-s DEBUG` + + 4. Added a facility to handle logs specified by "traditional templates" in the style + defined at url https://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html. + Usable with /var/log/{syslog,kern.log,auth.log}. + ``` + cat testData/TestAuth.data | \ + logparse --parser TraditionalFileFormat -f TIMESTAMP -s ERROR + ``` + Among capabilities, handle Docker log format: + + ``` + docker container logs mysql1 2>/dev/stdout >/dev/null| head -2| \ + logparse --parser DockerCLog --raw \ + -f TIMESTAMP,NUM,bracket,bracket1,bracket2,msg -s ERROR + ``` + where DockerCLog is defined in parsers2.py, but could also be supplied + via configuration file in section `RSysTradiVariant`, outputs + + ``` + 2021-04-26T...Z 0 [System] [MY-010116] [Server] .../mysqld () starting ... + 2021-04-26T...Z 1 [System] [MY-013576] [InnoDB] InnoDB initialization start + ``` + + 5. Added `logdb`, performing more database operations from logs: + + As an example, exploiting docker's inspect output + + ~~~ + docker image inspect nginx | \ + wrapList2Json.py | \ + logparse --parser JSONParserPlus -f "TOP" | head -1 | \ + logdb -f "TOP/*/{RepoTags|Config}/{Env}" --frontend JSONParserPlus \ + --join-remote-key "application" \ + --join-remote-name EventTable + ~~~ + + Enters the following in the database, which is reminiscent of "Triple Store" for + RDF, and borrows ideas from `SQLAlchemy/example/code/adjacencyList.py` + + ~~~ + id,parent_id,name,nval + 1000,NULL,RootNode,2021-05-17T15:10:25.642125 + 1001,1000,RangeStart,1010 + 1010,1000,**TOP**,TREEPOS.LIST + 1011,1010,0,TREEPOS.TOP + 1012,1011,Driver,default + 1013,1011,Options,TREEPOS.EMPTY_TREE + 1014,1011,Config,TREEPOS.LIST + 1015,1014,0,TREEPOS.TOP + 1016,1015,Subnet,172.19.0.0/16 + 1017,1015,Gateway,172.19.0.1 + 1018,1010,1,TREEPOS.TOP + 1019,1018,4cf07fa937da4b6ad844b3a6ca22a8eb167dc503544becda8789774a6e605c5b,TREEPOS.TOP + 1020,1019,Name,mysql1 + 1021,1019,EndpointID,7edb7b1e866957f0de6150c73c78b70717a6bb833794009509f1002dec622f41 + 1022,1019,MacAddress,02:42:ac:13:00:02 + 1023,1019,IPv4Address,172.19.0.2/16 + 1024,1019,IPv6Address, + 1025,1018,517acb5295aba5cc347d7aae88b525abf7faef26f5c481f2b82636b0619a192b,TREEPOS.TOP + 1026,1025,Name,mysql-workbench + 1027,1025,EndpointID,5255215cc33826d3b90973aaed786d3b3e2b620ddbe6a837c6ae060fc8ce61e4 + 1028,1025,MacAddress,02:42:ac:13:00:03 + 1029,1025,IPv4Address,172.19.0.3/16 + 1030,1025,IPv6Address, + 1031,1000,RangeEnd,1030 + ~~~ + + **Note**: utility `wrapList2Json.py`is in directory + `./aux/testData/ActionsMysqlTest/scripts/wrapList2Json.py`, it will probably + move to appear with other executables in later versions. It's function is + to wrap a JSONish list into a directory entry with key `TOP`. + + +### More examples and tests + +See the Github Actions scripts for more details, which include complete configured +test suites. https://github.com/AlainLich/logtools/actions with +test codes at https://github.com/AlainLich/logtools/tree/ALPython3/aux diff --git a/README.md b/README.md index 5277c31..8b57987 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,119 @@ # logtools -[![Build Status](https://travis-ci.org/adamhadani/logtools.svg?branch=master)](https://travis-ci.org/adamhadani/logtools) +## TOC + + +- [logtools](#logtools) + - [TOC](#toc) + - [Derived Version Notice](#derived-version-notice) + - [
Original README
](#original-readme) + - [A log files analysis / filtering framework.](#a-log-files-analysis--filtering-framework) + - [Installation](#installation) + - [Python2](#python2) + - [Python3](#python3) + - [Console Scripts](#console-scripts) + - [Configuration](#configuration) + - [Usage Examples](#usage-examples) + - [Credits](#credits) + - [Contact](#contact) + - [License](#license) + + + + +## Derived Version Notice + This is a derived version of logtools, + including a port to Python-3 and + additional features. + - The original version, for Python-2 (as far as I know), by Adam Ever-Hadani is available at + http://github.com/adamhadani . + - This version, for Python-3 only, with additions and changes by Alain Lichnewsky, is available at + https://github.com/AlainLich/logtools + distributed as is, with no warranty, under the Apache 2.0 license. + + For features see + [PYTHON3-README.md](./PYTHON3-README.md) + and [PYTHON3-ADDITIONS.md](./PYTHON3-ADDITIONS.md). + +Notes: this version + - is not compatible with Python-2 + - has been tested in the following configurations + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ DevelopmentGithub Actions
Status
StatusOK on dev. machine
Status Extensions MysqlOK on dev. machine
Status Extensions SQLiteOK on dev. machine
Test Configurations
SystemUbuntu 20.10Ubuntu 20.04.2 LTS
Ubuntu 21.04
Linux5.8.0-25-generic
5.11.0-16.generic
Python3.8.6CPython 3.8.10
3.9.4
virtualenv20.0.29+dnot used
Used modulessee requirements.txt + requirements.txt, and + for test +
Localefr_FR.UTF-8C.UTF-8
MysqlDockerHub reference + Github VM standard +
mysql/mysql-server:latestMySQL 8.0.25
2021-01-19T14:27:35
SQLite3.34.1
+ + +---------------------------------------------------------- + +##
Original README
+ +---------------------------------------------------------- + @@ -39,6 +152,9 @@ This software is distributed under the Apache 2.0 license. ## Installation +### Python2 +Note This applies to the original package, not the port in this directory; please +see [Adam Ever-Hadani](http://github.com/adamhadani/) site and original version. To install this package and associated console scripts, unpack the distributable tar file, or check out the project directory, and then run: @@ -60,7 +176,14 @@ If for some reason setuptools does not install 'nose' by default, you can easily install it if you have setuptools installed by running: easy_install nose + +### Python3 + +Concerning the Python3 port, see [PYTHON3-README.md](./PYTHON3-README.md) +For installation, see [PYTHON3-README.md](./PYTHON3-README.md#installation), +much information can be +gleaned from [Github Actions YAML](.github/workflows/pythonExercise.yml). ## Console Scripts @@ -71,7 +194,7 @@ you can easily install it if you have setuptools installed by running: Blacklist files can specify both exact match as well as more complex matching types (prefix/suffix match, regexp match. See examples below). -* ``geoip`` +* ``geoip`` (Not ported to Python-3) Simple helper utility for using the Maxmind GeoIP library to tag log lines by the IP's country. The regular expression mask used for matching the IP in the log line is user-specified. This tool requires the Maxmind GeoIP library and python bindings. See http://www.maxmind.com/app/country diff --git a/RECIPES.md b/RECIPES.md index 99340f2..9c0eb96 100644 --- a/RECIPES.md +++ b/RECIPES.md @@ -1,12 +1,13 @@ # logtools RECIPES - - -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* +## TOC + - [logtools RECIPES](#logtools-recipes) + - [TOC](#toc) + + - 1. The following example demonstrates specifying a custom regular expression for matching the ip/user agent using the filterbots tool. diff --git a/aux/Makefile b/aux/Makefile new file mode 100644 index 0000000..4526876 --- /dev/null +++ b/aux/Makefile @@ -0,0 +1,37 @@ +# Makefile in logtools/aux directory +# +# Added to assist with some maintenance tasks +# +# ---------------------------------------------------------------------- +# + +# +# Assist with building table of contents in Markdown +# +SHELL= /bin/bash +PYTHON= python3 + +PYDIR= ../logtools + +MARKDOWNFILES=$(wildcard *.md) + + +# +# Generate UML with pyreverse +# +.PHONY: doUML + +doUML: + pyreverse $(PYDIR) + echo HOU + for f in classes.dot packages.dot ; \ + do \ + echo $f ; \ + if [ -f $$f ] ; \ + then \ + pyreversePlus --in $$f --out UML$$f --add UMLDetails.add; \ + ddot UML$$f; \ + else \ + echo $$f not found ; \ + fi ; \ + done diff --git a/aux/UMLDetails.add b/aux/UMLDetails.add new file mode 100644 index 0000000..93bc319 --- /dev/null +++ b/aux/UMLDetails.add @@ -0,0 +1,12 @@ +#----------------------- Additional info to be used with pyreverse output +#....................... for + +"DB_Tree_Maker" -> "SQLAlchemyDbOperator" : composition deferred_base_protocol +"TreeNode" -> "DB_Tree_Maker" : aggregation generate + +#"AffineBase" -> "Draw3DBasics" : composition displayList +#"RenderingBase" -> "Draw3DBasics" : composition rendererInfo +#"FlexShader" -> "ShaderResourceAndSwitchGlu" : composition resourceDB +#"VisFlexResource" -> "ShaderResourceAndSwitchVis" : composition resourceDB +#"Camera" -> "Draw3DSceneBasics" : composition viewpoint +#"Camera" -> "ViewerBase" : composition viewpoint diff --git a/aux/dot_logtoolsrc.sh b/aux/dot_logtoolsrc.sh new file mode 100755 index 0000000..ce4214a --- /dev/null +++ b/aux/dot_logtoolsrc.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + + +# This script prepares the ~/.logtoolsrc for testing, may evolve to be usable as +# an example to be filled/modified, but for now just a test thing +# +# Arguments +# 1) first argument: PATH leading to logtools package source +# 2) second argument: path for ~/.logtoolsrc; default to ~/.logtoolsrc +# 3) third argument: path for ~/.logtools.d + +BASEPATH="$1" +arg2="$2" +arg3="$3" +DESTFILE="${arg2:=/tmp/.logtoolsrc}" +DESTDIR="${arg3:=/tmp/.logtools.d}" + +echo "In dot_logtoolsrc.sh" +echo " substituting : $BASEPATH to \$BASEPATH, emitting $DESTFILE" +echo " entering USER DB passwd via environment variable" + +# Keep this file for myself +maskOrig=$(umask -p) +umask 0077 +cat >"$DESTFILE" <.*?) -(?:.*?"){5}(?P.*?)" + + +#Disable warnings about dpath standard versions that do not deal with 're' regexps +#[dpath] +#no-dpath-warning: True + +[logtail] +field: 2 + + +[logparse] +field: 1 +raw: + +[logjoin] +field: 1 +backend: sqlalchemy + + # for now used in logjoin only, if non void, filter output with + # unicodedata.normalize(XXXX, str(x)).encode('ascii','ignore') where XXXX may be + # ‘NFC’, ‘NFKC’, ‘NFD’, and ‘NFKD’. + # -- for backward compatibility, enter 'NFKD' or leave this empty + # -- to suppress encoding enter 'None' +output_encoding: None + +frontend: + + ## here we expect an rfc1738 URL + ## these are the connect parameters of sqlalchemy instantiation + ## USER_DB_PASS is a secret that needs to be properly substituted +join_connect_string: mysql://user:${USER_DB_PASS}@localhost/logfromtool + +join_remote_fields: * +join_remote_name: testTable +join_remote_key: - + + +[dbOp] +field: 1 +frontend: + +dbOperator: SQLAlcDbOp + + ## here we expect an rfc1738 URL + ## these are the connect parameters of sqlalchemy instantiation +dbOp_connect_string: mysql://user:${USER_DB_PASS}@localhost/logfromtool + +dbOp_remote_fields: * +dbOp_remote_name: testTable +dbOp_remote_key: - + + +[RSysTradiVariant] +file: ${HOME}/.logtools.d/tradivariantParsing.txt + + +END + +echo Produced file $DESTFILE +# reset umask +$maskOrig + +if [ ! -d ${DESTDIR} ] ; then + mkdir -p ${DESTDIR} + echo Created dir ${DESTDIR} +fi + +cat >${DESTDIR}/tradivariantParsing.txt <"$DESTFILE" <.*?) -(?:.*?"){5}(?P.*?)" + + +#Disable warnings about dpath standard versions that do not deal with 're' regexps +#[dpath] +#no-dpath-warning: True + +[logtail] +field: 2 + + +[logparse] +field: 1 +raw: + +[logjoin] +field: 1 +backend: sqlalchemy + + # for now used in logjoin only, if non void, filter output with + # unicodedata.normalize(XXXX, str(x)).encode('ascii','ignore') where XXXX may be + # ‘NFC’, ‘NFKC’, ‘NFD’, and ‘NFKD’. + # -- for backward compatibility, enter 'NFKD' or leave this empty + # -- to suppress encoding enter 'None' +output_encoding: None + +frontend: + + ## these are the connect parameters of sqlalchemy instantiation + ## for information on connect string, see + ## https://docs.sqlalchemy.org/en/13/dialects/sqlite.html\ + ## #module-sqlalchemy.dialects.sqlite.pysqlite + ## + ## possible sqlite_file path may be absolute or relative: +join_connect_string: sqlite:///${sqlite_file} + +join_remote_fields: * +join_remote_name: testTable +join_remote_key: - + + +[dbOp] +field: 1 +frontend: + +dbOperator: SQLAlcDbOp + + ## see above +dbOp_connect_string: sqlite:///${sqlite_file} + +dbOp_remote_fields: * +dbOp_remote_name: testTable +dbOp_remote_key: - + + +[RSysTradiVariant] +file: ${HOME}/.logtools.d/tradivariantParsing.txt + + +END + +echo Produced file $DESTFILE +# reset umask +$maskOrig + +if [ ! -d ${DESTDIR} ] ; then + mkdir -p ${DESTDIR} + echo Created dir ${DESTDIR} +fi + +cat >${DESTDIR}/tradivariantParsing.txt <