Pre-commit environment, Option for filtering jenkins jobs, change starting folder, build environment using setuptools and pyproject.toml, sample config.yaml.

.gitignore

@@ -0,0 +1,34 @@
+# Configuration file:
+/conf/config.yaml
+
+# Distribution / packaging
+build/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+*.egg-info/
+/dist/
+MANIFEST
+
+# Python cache / compiled files:
+__pycache__/
+*.py[cod]
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Environments
+/.venv*/
+/venv*/
+/.env*/
+/env*/
+
+
+# JetBrains IDE
+/.idea/

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+default_stages: [pre-commit, pre-push]
+default_language_version:
+  # force all unspecified python hooks to run python3
+  python: python3
+minimum_pre_commit_version: "3.4.0"
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

README.md

@@ -5,16 +5,20 @@ The Kibble Scanners collect information for the Kibble Suite.
 
  - Edit conf/config.yaml to match your Kibble service
 
+### Dependencies
+
+ - a running Elasticsearch server
+
 ## How to run:
 
- - On a daily/weekly/whatever basis, run: `python3 src/kibble-scanner.py`.
+ - On a daily/weekly/whatever basis, run in folder src: `python3 kibble-scanner.py`.
 
 ### Command line options:
 
     usage: kibble-scanner.py [-h] [-o ORG] [-f CONFIG] [-a AGE] [-s SOURCE]
                              [-n NODES] [-t TYPE] [-e EXCLUDE [EXCLUDE ...]]
                              [-v VIEW]
-    
+
     optional arguments:
       -h, --help            show this help message and exit
       -o ORG, --org ORG     The organisation to gather stats for. If left out, all
@@ -75,7 +79,44 @@ The Kibble Scanners collect information for the Kibble Suite.
  - python3-certifi
  - python3-yaml
 
-
+### Testing
+
+  TBD
+
+### Pre-commit
+
+  After running
+
+    pip install -r requirements.txt
+
+  Run
+
+    pre-commit install
+
+  to install
+
+  Run it explicitely by
+
+    pre-commit
+
+  to run the checks in .pre-commit-config.yaml
+
+  If installed the pre-commit reads the configuration, and will check on the hooks, currently pre-comit and pre-push.
+  While the checks are not satisfied, just rerun the commit command until the hook checks are passed.
+
+
+### Project build
+
+  After installation of the build tool
+
+    pip install -q build
+
+  build the project by running
+
+    python -m build
+
+  Find more information in project.toml file and [Setuptools](https://setuptools.pypa.io/).
+
 # Get involved
+
   TBD. Please see https://kibble.apache.org/ for details!
-

pyproject.toml

@@ -0,0 +1,60 @@
+##############################
+# Python packaging settings: #
+
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kibble-scanners"
+description = "The Kibble Scanners collect information for the Kibble Suite. Apache Kibble is a tool to collect, aggregate and visualize data about any software project that uses commonly known tools."
+version = "1.0.0"
+
+dependencies = [
+    "python-dateutil",
+    "certifi",
+     "requests",
+     "psutil",
+    "elasticsearch",
+    "PyYAML>=5.2",
+]
+
+requires-python = ">=3.9, <4.0"
+
+authors = [
+    { name = "Apache Software Foundation", email = "dev@kibble.apache.org" },
+]
+maintainers = [
+    { name = "Apache Software Foundation", email="dev@kibble.apache.org" },
+]
+keywords = [
+    "kibble-scanners", "data" ]
+
+license = { text = "Apache License, Version 2.0" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Environment :: Console",
+    "Framework :: Apache Kibble-Scanners",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Topic :: System :: Monitoring"
+]
+
+dynamic = [
+    "readme"
+]
+
+[project.urls]
+repository = "https://github.com/apache/kibble-scanners.git"
+"Bug Tracker" = "https://github.com/apache/kibble-scanners/issues"
+
+
+[tool.setuptools]
+dynamic = { readme = { file = ["README.md"] } }
+packages.find = { where = ["src"] }

requirements.txt

@@ -4,3 +4,4 @@ psutil
 python-dateutil
 requests
 pyyaml
+pre_commit

src/kibble-scanner.py

@@ -28,8 +28,8 @@
 import plugins.brokers.kibbleES
 #import plugins.kibbleJSON
 
-VERSION = "0.1.0"
-CONFIG_FILE = "conf/config.yaml"
+VERSION = "0.2.0"
+CONFIG_FILE = "../conf/config.yaml"
 PENDING_OBJECTS = []
 BIG_LOCK = threading.Lock()
 
@@ -43,15 +43,16 @@ def base_parser():
     arg_parser.add_argument("-t", "--type", help="Specific type of scanner to run (default is run all scanners)")
     arg_parser.add_argument("-e", "--exclude", nargs = '+', help="Specific type of scanner(s) to exclude")
     arg_parser.add_argument("-v", "--view", help="Specific source view to scan (default is scan all sources)")
+    arg_parser.add_argument("-j", "--filter", nargs='+', help="Jenkins-only: Filter the list of jobs (e.g. for debugging). To drill down to the target jobs, all nodes to the leaf node(s) are required, e.g --filter <project> <jobgroup> <targetjob1> <targetjob2>. Type is set to jenkins implicitely.")
     return arg_parser
-   
+
 def pprint(string, err = False):
     line = "[core]: %s" % (string)
     if err:
         sys.stderr.write(line + "\n")
     else:
         print(line)
-        
+
 
 def isMine(ID, config):
     if config['scanner'].get('balance',  None):
@@ -65,20 +66,24 @@ def isMine(ID, config):
             return True
         return False
     return True
-    
+
 class scanThread(threading.Thread):
     """ A thread object that grabs an item from the queue and processes
         it, using whatever plugins will come out to play. """
-    def __init__(self, broker, org, i, t = None, e = None):
+    def __init__(self, broker, org, i, t = None, e = None, f= None):
         super(scanThread, self).__init__()
         self.broker = broker
         self.org = org
         self.id = i
         self.bit = self.broker.bitClass(self.broker, self.org, i)
         self.stype = t
         self.exclude = e
+        self.filter = f
+        # override
+        if self.filter:
+            self.stype = "jenkins"
         pprint("Initialized thread %i" % i)
-    
+
     def run(self):
         global BIG_LOCK, PENDING_OBJECTS
         time.sleep(0.5) # Primarily to align printouts.
@@ -89,6 +94,7 @@ def run(self):
             try:
                 # Try grabbing an object (might not be any left!)
                 obj = PENDING_OBJECTS.pop(0)
+                #print("object: %s" %(obj))
             except:
                 pass
             BIG_LOCK.release()
@@ -97,14 +103,17 @@ def run(self):
                 if isMine(obj['sourceID'], self.broker.config):
                     # Run through list of scanners in order, apply when useful
                     for sid, scanner in plugins.scanners.enumerate():
-                        
+
                         if scanner.accepts(obj):
                             self.bit.pluginname = "plugins/scanners/" + sid
                             # Excluded scanner type?
                             if self.exclude and sid in self.exclude:
                                 continue
+                            # specific jenkins filter
+                            if self.stype and self.stype == sid and self.filter and sid == "jenkins":
+                                scanner.scan(self.bit, obj, self.filter)
                             # Specific scanner type or no types mentioned?
-                            if not self.stype or self.stype == sid:
+                            elif not self.stype or self.stype == sid:
                                 scanner.scan(self.bit, obj)
             else:
                 break
@@ -115,13 +124,13 @@ def main():
     pprint("Kibble Scanner v/%s starting" % VERSION)
     global CONFIG_FILE, PENDING_OBJECTS
     args = base_parser().parse_args()
-    
+
     # Load config yaml
     if args.config:
         CONFIG_FILE = args.config
     config = yaml.load(open(CONFIG_FILE), Loader=yaml.Loader)
     pprint("Loaded YAML config from %s" % CONFIG_FILE)
-    
+
     # Which broker type do we use here?
     broker = None
     if 'elasticsearch' in config and config['elasticsearch'].get('enabled', False):
@@ -130,14 +139,14 @@ def main():
     else:
         pprint("Using HTTP JSON broker model")
         broker = plugins.brokers.kibbleJSON.Broker(config)
-    
+
     orgNo = 0
     sourceNo = 0
     for org in broker.organisations():
         if not args.org or args.org == org.id:
             pprint("Processing organisation %s" % org.id)
             orgNo += 1
-            
+
             # Compile source list
             # If --age is passed, only append source that either
             # have never been scanned, or have been scanned more than
@@ -161,21 +170,21 @@ def main():
                     if not args.source or (args.source == source['sourceID']) or (args.source == source['sourceURL']):
                         PENDING_OBJECTS.append(source)
                 sourceNo += len(PENDING_OBJECTS)
-            
+
             # Start up some threads equal to number of cores on the box,
             # but no more than 4. We don't want an IOWait nightmare.
             threads = []
             core_count = min((4, int( multiprocessing.cpu_count() )))
             for i in range(0, core_count):
-                sThread = scanThread(broker, org, i+1, args.type, args.exclude)
+                sThread = scanThread(broker, org, i+1, args.type, args.exclude, args.filter)
                 sThread.start()
                 threads.append(sThread)
-            
+
             # Wait for them all to finish.
             for t in threads:
                 t.join()
-        
+
     pprint("All done scanning for now, found %i organisations and %i sources to process." % (orgNo, sourceNo))
-    
+
 if __name__ == '__main__':
     main()