combine compliance results for regions/k8s branches (#949)

* Refactoring: reverse order of arguments
* aggregate results for regions in table
* Provide details page for subject group (multiple regions or k8s versions)
* Links to individual subjects at the top
* fix: skip unnecessary upgrade step for empty db

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

compliance-monitor/bootstrap.yaml

@@ -31,15 +31,19 @@ accounts:
         public_key_type: "ssh-ed25519"
         public_key_name: "primary"
   - subject: pco-prod1
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod2
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod3
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: pco-prod4
+    group: pco-prod
     delegates:
       - zuul_ci
   - subject: poc-wgcloud
@@ -56,9 +60,11 @@ accounts:
     delegates:
       - zuul_ci
   - subject: syseleven-dus2
+    group: syseleven
     delegates:
       - zuul_ci
   - subject: syseleven-ham1
+    group: syseleven
     delegates:
       - zuul_ci
   - subject: wavestack

compliance-monitor/monitor.py

@@ -41,7 +41,7 @@
     db_find_account, db_update_account, db_update_publickey, db_filter_publickeys, db_get_reports,
     db_get_keys, db_insert_report, db_get_recent_results2, db_patch_approval2, db_get_report,
     db_ensure_schema, db_get_apikeys, db_update_apikey, db_filter_apikeys, db_clear_delegates,
-    db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate,
+    db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate, db_get_group,
 )
 
 
@@ -79,6 +79,7 @@ def __init__(self):
         self.yaml_path = os.path.abspath("../Tests")
 
 
+GROUP_PREFIX = 'group-'
 ROLES = {'read_any': 1, 'append_any': 2, 'admin': 4, 'approve': 8}
 # number of days that expired results will be considered in lieu of more recent, but unapproved ones
 GRACE_PERIOD_DAYS = 7
@@ -226,7 +227,8 @@ def import_bootstrap(bootstrap_path, conn):
     with conn.cursor() as cur:
         for account in accounts:
             roles = sum(ROLES[r] for r in account.get('roles', ()))
-            accountid = db_update_account(cur, {'subject': account['subject'], 'roles': roles})
+            acc_record = {'subject': account['subject'], 'roles': roles, 'group': account.get('group')}
+            accountid = db_update_account(cur, acc_record)
             db_clear_delegates(cur, accountid)
             for delegate in account.get('delegates', ()):
                 db_add_delegate(cur, accountid, delegate)
@@ -637,6 +639,13 @@ async def get_report_view_full(
     )
 
 
+def _resolve_group(cur, subject, prefix=GROUP_PREFIX):
+    group = subject.removeprefix(prefix)
+    if subject != group:
+        return group, db_get_group(cur, group)
+    return None, [subject]
+
+
 @app.get("/{view_type}/detail/{subject}/{scopeuuid}")
 async def get_detail(
     request: Request,
@@ -646,14 +655,18 @@ async def get_detail(
     scopeuuid: str,
 ):
     with conn.cursor() as cur:
-        rows2 = db_get_relevant_results2(cur, subject, scopeuuid, approved_only=True)
+        group, subjects = _resolve_group(cur, subject)
+        rows2 = []
+        for subj in subjects:
+            rows2.extend(db_get_relevant_results2(cur, subj, scopeuuid, approved_only=True))
     results2 = convert_result_rows_to_dict2(
         rows2, get_scopes(), include_report=True, grace_period_days=GRACE_PERIOD_DAYS,
         subjects=(subject, ), scopes=(scopeuuid, ),
     )
+    title = f'Details for group {group}' if group else f'Details for subject {subject}'
     return render_view(
         VIEW_DETAIL, view_type, results=results2, base_url=settings.base_url,
-        title=f'{subject} compliance',
+        title=title,
     )
 
 
@@ -666,13 +679,17 @@ async def get_detail_full(
     scopeuuid: str,
 ):
     with conn.cursor() as cur:
-        rows2 = db_get_relevant_results2(cur, subject, scopeuuid, approved_only=False)
+        group, subjects = _resolve_group(cur, subject)
+        rows2 = []
+        for subject in subjects:
+            rows2.extend(db_get_relevant_results2(cur, subject, scopeuuid, approved_only=False))
     results2 = convert_result_rows_to_dict2(
         rows2, get_scopes(), include_report=True, subjects=(subject, ), scopes=(scopeuuid, ),
     )
+    title = f'Details for group {group}' if group else f'Details for subject {subject}'
     return render_view(
         VIEW_DETAIL, view_type, results=results2, base_url=settings.base_url,
-        title=f'{subject} compliance (incl. unverified results)',
+        title=f'{title} (incl. unverified results)',
     )
 
 
@@ -785,13 +802,34 @@ async def get_healthz(request: Request):
     return Response()  # empty response with status 200
 
 
-def pick_filter(results, subject, scope):
+def pick_filter(results, scope, *subjects):
     """Jinja filter to pick scope results from `results` for given `subject` and `scope`"""
-    return results.get(subject, {}).get(scope, {})
+    # simple case (backwards compatible): precisely one subject
+    if len(subjects) == 1:
+        return results.get(subjects[0], {}).get(scope, {})
+    # generalized case: multiple subjects
+    # in this case, drop None
+    rs = [results.get(subject, {}).get(scope, {}) for subject in subjects]
+    return [r for r in rs if r is not None]
+
+
+STATUS_ORDERING = {
+    'effective': 10,
+    'warn': 5,
+    'deprecated': 1,
+}
 
 
 def summary_filter(scope_results):
     """Jinja filter to construct summary from `scope_results`"""
+    if not isinstance(scope_results, dict):
+        # new generalized case: "aggregate" results for multiple subjects
+        # simplified computation: just select the worst subject to represent the group
+        scope_results = min(
+            scope_results,
+            default={},
+            key=lambda sr: STATUS_ORDERING.get(sr.get('best_passed'), -1),
+        )
     passed_str = scope_results.get('passed_str', '') or '–'
     best_passed = scope_results.get('best_passed')
     # avoid simple 🟢🔴 (hard to distinguish for color-blind folks)

compliance-monitor/sql.py

@@ -3,9 +3,9 @@
 
 # list schema versions in ascending order
 SCHEMA_VERSION_KEY = 'version'
-SCHEMA_VERSIONS = ['v1', 'v2', 'v3']
+SCHEMA_VERSIONS = ['v1', 'v2', 'v3', 'v4']
 # use ... (Ellipsis) here to indicate that no default value exists (will lead to error if no value is given)
-ACCOUNT_DEFAULTS = {'subject': ..., 'api_key': ..., 'roles': ...}
+ACCOUNT_DEFAULTS = {'subject': ..., 'api_key': ..., 'roles': ..., 'group': None}
 PUBLIC_KEY_DEFAULTS = {'public_key': ..., 'public_key_type': ..., 'public_key_name': ...}
 
 
@@ -135,6 +135,14 @@ def db_ensure_schema_v3(cur: cursor):
     ''')
 
 
+def db_ensure_schema_v4(cur: cursor):
+    # start from v3, do small alteration
+    db_ensure_schema_v2(cur)
+    cur.execute('''
+    ALTER TABLE account ADD COLUMN IF NOT EXISTS "group" text;
+    ''')
+
+
 def db_upgrade_data_v1_v2(cur):
     # we are going to drop table result, but use delete anyway to have the transaction safety
     cur.execute('''
@@ -191,8 +199,8 @@ def db_upgrade_schema(conn: connection, cur: cursor):
         if current is None:
             # this is an empty db, but it also used to be the case with v1
             # I (mbuechse) made sure manually that the value v1 is set on running installations
-            db_ensure_schema_v3(cur)
-            db_set_schema_version(cur, 'v3')
+            db_ensure_schema_v4(cur)
+            db_set_schema_version(cur, 'v4')
             conn.commit()
         elif current == 'v1':
             db_ensure_schema_v2(cur)
@@ -207,6 +215,10 @@ def db_upgrade_schema(conn: connection, cur: cursor):
             db_ensure_schema_v3(cur)
             db_set_schema_version(cur, 'v3')
             conn.commit()
+        elif current == 'v3':
+            db_ensure_schema_v4(cur)
+            db_set_schema_version(cur, 'v4')
+            conn.commit()
 
 
 def db_ensure_schema(conn: connection):
@@ -229,11 +241,12 @@ def db_ensure_schema(conn: connection):
 def db_update_account(cur: cursor, record: dict):
     sanitized = sanitize_record(record, ACCOUNT_DEFAULTS)
     cur.execute('''
-    INSERT INTO account (subject, roles)
-    VALUES (%(subject)s, %(roles)s)
+    INSERT INTO account (subject, roles, "group")
+    VALUES (%(subject)s, %(roles)s, %(group)s)
     ON CONFLICT (subject)
     DO UPDATE
-    SET roles = EXCLUDED.roles
+    SET roles = EXCLUDED.roles,
+    "group" = EXCLUDED."group"
     RETURNING accountid;''', sanitized)
     accountid, = cur.fetchone()
     return accountid
@@ -262,6 +275,11 @@ def db_find_subjects(cur: cursor, delegate):
     return [row[0] for row in cur.fetchall()]
 
 
+def db_get_group(cur: cursor, group):
+    cur.execute('''SELECT subject FROM account WHERE "group" = %s;''', (group, ))
+    return [row[0] for row in cur.fetchall()]
+
+
 def db_update_apikey(cur: cursor, accountid, apikey_hash):
     sanitized = dict(accountid=accountid, apikey_hash=apikey_hash)
     cur.execute('''

compliance-monitor/templates/details.md.j2

@@ -1,8 +1,16 @@
+{% if results | length > 1 %}
+Jump to
+
+{% for subject in results -%}
+- [{{subject}}](#{{subject}})
+{% endfor %}
+{% endif %}
 {% for subject, subject_result in results.items() -%}
 {# omit h1 title here because we can only have one of those,
    and the html wrapper template will add one anyway -#}
 {% for scopeuuid, scope_result in subject_result.items() -%}
-## {{ scope_result.name }}
+<a name="{{ subject }}"></a>
+## {{ subject }}: {{ scope_result.name }}
 
 - [spec overview]({{ scope_url(scopeuuid) }})
 

compliance-monitor/templates/overview.md.j2

@@ -11,40 +11,32 @@ Version numbers are suffixed by a symbol depending on state: * for _draft_, †
 | Name  | Description  | Operator  | [SCS-compatible IaaS](https://docs.scs.community/standards/scs-compatible-iaas/)  | HealthMon  |
 |-------|--------------|-----------|----------------------|:----------:|
 | [scs2](https://docs.scs.community/community/cloud-resources/plusserver-gx-scs) | Dev/Test/Demo environment (2nd gen) provided for SCS & GAIA-X context | plusserver GmbH | 
-{#- #} [{{ results | pick('scs2', iaas) | summary }}]({{ detail_url('scs2', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'scs2') | summary }}]({{ detail_url('scs2', iaas) }}) {# -#}
 | [HM](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&refresh=5m&var-mycmd=All&var-mymethod=All&var-mywait=All&var-mybench=All&var-mycloud=gx-scs2) |
 | [aov.cloud](https://www.aov.de/) | Community cloud for customers | aov IT.Services GmbH |
-{#- #} [{{ results | pick('aov-cloud', iaas) | summary }}]({{ detail_url('aov-cloud', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'aov-cloud') | summary }}]({{ detail_url('aov-cloud', iaas) }}) {# -#}
 | [HM](https://health.aov.cloud/) |
 | [CC@RRZE](https://www.rrze.fau.de/) | Private Compute Cloud (CC) for [FAU](https://www.fau.de/) | Regionales Rechenzentrum Erlangen | 
-{#- #} [{{ results | pick('cc-rrze', iaas) | summary }}]({{ detail_url('cc-rrze', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'cc-rrze') | summary }}]({{ detail_url('cc-rrze', iaas) }}) {# -#}
 | (soon) |
 | [CNDS](https://cnds.io/) | Public cloud for customers | artcodix GmbH |
-{#- #} [{{ results | pick('artcodix', iaas) | summary }}]({{ detail_url('artcodix', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'artcodix') | summary }}]({{ detail_url('artcodix', iaas) }}) {# -#}
 | [HM](https://ohm.muc.cloud.cnds.io/) |
-| [pluscloud open](https://www.plusserver.com/en/products/pluscloud-open)<br />(4 regions) | Public cloud for customers  | plusserver GmbH | {# #}
-{#- #}prod1: [{{ results | pick('pco-prod1', iaas) | summary }}]({{ detail_url('pco-prod1', iaas) }}){# -#}
-<br />
-{#- #}prod2: [{{ results | pick('pco-prod2', iaas) | summary }}]({{ detail_url('pco-prod2', iaas) }}){# -#}
-<br />
-{#- #}prod3: [{{ results | pick('pco-prod3', iaas) | summary }}]({{ detail_url('pco-prod3', iaas) }}){# -#}
-<br />
-{#- #}prod4: [{{ results | pick('pco-prod4', iaas) | summary }}]({{ detail_url('pco-prod4', iaas) }}) {# -#}
-| [HM1](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-pco)<br />[HM2](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod2)<br />[HM3](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod3)<br />[HM4](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod4) |
+| [pluscloud open](https://www.plusserver.com/en/products/pluscloud-open) | Public cloud for customers (4 regions)  | plusserver GmbH | {# #}
+{#- #}[{{ results | pick(iaas, 'pco-prod1', 'pco-prod2', 'pco-prod3', 'pco-prod4') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
+| [HM1](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-pco) [HM2](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod2) [HM3](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod3) [HM4](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod4) |
 | PoC WG-Cloud OSBA | Cloud PoC for FITKO | Cloud&amp;Heat Technologies GmbH |
-{#- #} [{{ results | pick('poc-wgcloud', iaas) | summary }}]({{ detail_url('poc-wgcloud', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'poc-wgcloud') | summary }}]({{ detail_url('poc-wgcloud', iaas) }}) {# -#}
 | [HM](https://health.poc-wgcloud.osba.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?var-mycloud=poc-wgcloud&orgId=1) |
 | [REGIO.cloud](https://regio.digital) | Public cloud for customers | OSISM GmbH |
-{#- #} [{{ results | pick('regio-a', iaas) | summary }}]({{ detail_url('regio-a', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'regio-a') | summary }}]({{ detail_url('regio-a', iaas) }}) {# -#}
 | [HM](https://apimon.services.regio.digital/public-dashboards/17cf094a47404398a5b8e35a4a3968d4?orgId=1&refresh=5m) |
 | [ScaleUp Open Cloud](https://www.scaleuptech.com/cloud-hosting/) | Public cloud for customers | ScaleUp Technologies GmbH & Co. KG |
-{#- #} [{{ results | pick('scaleup-occ2', iaas) | summary }}]({{ detail_url('scaleup-occ2', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'scaleup-occ2') | summary }}]({{ detail_url('scaleup-occ2', iaas) }}) {# -#}
 | [HM](https://health.occ2.scaleup.sovereignit.cloud) |
-| [syseleven](https://www.syseleven.de/en/products-services/openstack-cloud/)<br />(2 SCS regions) | Public OpenStack Cloud | SysEleven GmbH | {# #}
-{#- #}dus2: [{{ results | pick('syseleven-dus2', iaas) | summary }}]({{ detail_url('syseleven-dus2', iaas) }}){# -#}
-<br />
-{#- #}ham1: [{{ results | pick('syseleven-ham1', iaas) | summary }}]({{ detail_url('syseleven-ham1', iaas) }}) {# -#}
-| (soon)<br />(soon) |
+| [syseleven](https://www.syseleven.de/en/products-services/openstack-cloud/) | Public OpenStack Cloud (2 SCS regions) | SysEleven GmbH | {# #}
+{#- #} [{{ results | pick(iaas, 'syseleven-dus2', 'syseleven-ham1') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
+| (soon) |
 | [Wavestack](https://www.noris.de/wavestack-cloud/) | Public cloud for customers | noris network AG/Wavecon GmbH |
-{#- #} [{{ results | pick('wavestack', iaas) | summary }}]({{ detail_url('wavestack', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'wavestack') | summary }}]({{ detail_url('wavestack', iaas) }}) {# -#}
 | [HM](https://health.wavestack1.sovereignit.cloud:3000/) |