chore: release v1.0.5 (#2631)

.github/workflows/acceptance-tests.yml

@@ -28,9 +28,10 @@ jobs:
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
+      persist: ${{ steps.deploy-comment.outputs.persist}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v0.5.0
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -42,7 +43,7 @@ jobs:
       name: renku-ci-rp-${{ github.event.number }}
     steps:
     - name: deploy-pr
-      uses: SwissDataScienceCenter/renku-actions/deploy-renku@v0.3.2
+      uses: SwissDataScienceCenter/renku-actions/deploy-renku@v0.5.0
       env:
         DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
         DOCKER_USERNAME: ${{ secrets.RENKU_DOCKER_USERNAME }}
@@ -84,43 +85,27 @@ jobs:
 
   test-pr:
     runs-on: ubuntu-20.04
-    if: github.event.action != 'closed'
+    if: ${{ github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true' }}
     needs: [check-deploy, deploy-pr]
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v1
-      with:
-        python-version: 3.8
-    - name: Test the PR
-      if: "needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'"
-      env:
-        KUBECONFIG: ${{ github.workspace }}/renkubot-kube.config
-        RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
-        RENKU_RELEASE: renku-ci-rp-${{ github.event.number }}
-      run: |
-        echo "$RENKUBOT_KUBECONFIG" > ${{ github.workspace }}/renkubot-kube.config
-        helm test ${RENKU_RELEASE} --namespace ${RENKU_RELEASE} --timeout 80m --logs
-    - name: Download artifact for packaging on failure
-      if: failure()
-      uses: SwissDataScienceCenter/renku-actions/download-test-artifacts@v0.3.2
-      env:
-        RENKU_VALUES: ${{ secrets.CI_RENKU_VALUES }}
-        TEST_ARTIFACTS_PATH: "tests-artifacts-${{ github.sha }}"
-    - name: Upload screenshots on failure
-      if: failure()
-      uses: actions/upload-artifact@v1
+    - uses: SwissDataScienceCenter/renku-actions/test-renku@v0.5.0
       with:
-        name: acceptance-test-artifacts
-        path: ${{ github.workspace }}/test-artifacts/
+        renkubot-kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
+        renku-release: renku-ci-rp-${{ github.event.number }}
+        gitlab-token: ${{ secrets.DEV_GITLAB_TOKEN }}
+        persist: "${{ needs.check-deploy.outputs.persist }}"
+        ci-renku-values: ${{ secrets.CI_RENKU_VALUES }}
+
   cleanup:
-    if: github.event.action == 'closed'
+    needs: check-deploy
+    if: github.event.action == 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true'
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
     - name: renku teardown
-      uses: SwissDataScienceCenter/renku-actions/teardown-renku@v0.3.2
+      uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v0.5.0
       env:
+        HELM_RELEASE_REGEX: "^renku-ci-rp-${{ github.event.number }}$"
         GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}
-        KUBECONFIG: "${{ github.workspace }}/renkubot-kube.config"
-        RENKU_RELEASE: renku-ci-rp-${{ github.event.number }}
         RENKUBOT_KUBECONFIG: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
+        MAX_AGE_SECONDS: 0
+        DELETE_NAMESPACE: "true"

.github/workflows/test_deploy.yml

@@ -889,19 +889,19 @@ jobs:
           echo "GIT_USER=Renku Bot" >> $GITHUB_ENV
           echo "GIT_EMAIL=renku@datascience.ch" >> $GITHUB_ENV
       - name: Push chart and images
-        uses: SwissDataScienceCenter/renku-actions/publish-chart@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/publish-chart@v0.5.0
         env:
           CHART_NAME: renku-core
-          GITHUB_TOKEN: ${{ secrets.RENKU_CI_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
       - name: Wait for chart to be available
         run: sleep 120
       - name: Update component version
-        uses: SwissDataScienceCenter/renku-actions/update-component-version@v0.3.2
+        uses: SwissDataScienceCenter/renku-actions/update-component-version@v0.5.0
         env:
           CHART_NAME: renku-core
-          GITHUB_TOKEN: ${{ secrets.RENKU_CI_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.RENKUBOT_GITHUB_TOKEN }}
 
   coveralls-final:
     name: Aggregate coveralls data

.github/workflows/test_publish.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[all]
+        python -m pip install .[all]
         git config --global --add user.name "Renku @ SDSC"
         git config --global --add user.email "renku@datascience.ch"
     - name: Test with pytest
@@ -60,7 +60,7 @@ jobs:
         curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
         brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
-        python -m pip install -e .[all]
+        python -m pip install .[all]
         git config --global --add user.name "Renku @ SDSC"
         git config --global --add user.email "renku@datascience.ch"
     - name: Test with pytest

CHANGES.rst

@@ -18,6 +18,21 @@
 Changes
 =======
 
+`1.0.5 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.4...v1.0.5>`__ (2022-02-07)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **core:** replace ``cwlgen`` with ``cwl-utils``
+   (`#2603 <https://github.com/SwissDataScienceCenter/renku-python/issues/2603>`__)
+   (`ab2e9cf <https://github.com/SwissDataScienceCenter/renku-python/commit/ab2e9cf0b1f0c63a025bd6e09fffd4ab350a0d48>`__)
+
+-  **core:** fix jinja2 dependency not being installed by pip
+   (`#2613 <https://github.com/SwissDataScienceCenter/renku-python/issues/2613>`__)
+   (`6effa0e <https://github.com/SwissDataScienceCenter/renku-python/commit/6effa0efe7fe093119212d11a05515cd5f8cdeab>`__)
+
+
 `1.0.4 <https://github.com/SwissDataScienceCenter/renku-python/compare/v1.0.3...v1.0.4>`__ (2022-01-28)
 -------------------------------------------------------------------------------------------------------
 

docs/how-to-guides/implementing_a_provider.rst

@@ -53,7 +53,9 @@ The ``workflow_provider`` function shall return a tuple of ``(object, str)``,  w
 should be the plugin object, i.e. ``self`` and the string is a unique identifier of the
 provider plugin. This unique string will be the string that the user can provide to the
 ``--provider`` command line argument to select this plugin for executing the desired
-workflows.
+workflows. A dummy provider implementation is available
+`here <https://github.com/SwissDataScienceCenter/renku-dummy-provider>`_ in order to ease the
+initial implementation of a provider plugin.
 
 A provider HAS to set environment variables for a plans parameters, so they can be used by scripts.
 These environment variables have to be prefixed with the value of the

docs/reference/plugins.rst

@@ -54,6 +54,8 @@ where `myproject.pluginmodule` points to a Renku `hookimpl` e.g.:
 .. automodule:: renku.core.plugins.run
    :members:
 
+`This <https://github.com/SwissDataScienceCenter/renku-dummy-annotator>`_ repository contains an implementation of
+an activity annotation plugin.
 
 CLI Plugins
 -----------
@@ -82,6 +84,25 @@ where `myproject.pluginmodule:mycmd` points to a click command e.g.:
     def mycmd():
         ...
 
+An example implementation of such plugin is available `here <https://github.com/SwissDataScienceCenter/renku-cli-plugin-example>`_.
+
+Workflow Converter Plugins
+--------------------------
+
+Additional workflow converters can be implemented by extending
+:class:`renku.core.models.workflow.converters.IWorkflowConverter`. By default renku
+provides a CWL converter plugins that is used when exporting a workflow:
+
+.. code-block:: console
+
+   $ renku workflow export --format cwl <my_workflow>
+
+.. autoclass:: renku.core.models.workflow.converters.IWorkflowConverter
+    :members:
+
+We created a `dummy <https://github.com/SwissDataScienceCenter/renku-dummy-format>`_ implementation of such
+a converter plugin.
+
 Workflow Provider Plugins
 -------------------------
 

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 1.0.4
+version: 1.0.5

helm-chart/renku-core/requirements.yaml

@@ -2,4 +2,6 @@ dependencies:
 - name: redis
   version: 10.7.11
   repository: "https://charts.bitnami.com/bitnami"
-
+- name: certificates
+  version: 0.0.1
+  repository: "https://swissdatasciencecenter.github.io/helm-charts/"

helm-chart/renku-core/templates/deployment.yaml

@@ -42,6 +42,9 @@ spec:
       volumes:
         - name: shared-volume
           emptyDir: {}
+        {{- include "certificates.volumes" $ | nindent 8 }}
+      initContainers:
+        {{- include "certificates.initContainer" $ | nindent 8 }}
       containers:
       {{ if $.Values.metrics.enabled }}
         - name: {{ $.Chart.Name}}-rqmetrics
@@ -112,9 +115,11 @@ spec:
               value: {{ $.Values.gitLFSSkipSmudge | quote }}
             - name: RENKU_DOMAIN
               value: {{ $.Values.global.renku.domain }}
+            {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
+            {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
           ports:
             - name: http
               containerPort: 8080
@@ -168,9 +173,11 @@ spec:
               value: {{ $.Values.gitLFSSkipSmudge | quote }}
             - name: RENKU_DOMAIN
               value: {{ $.Values.global.renku.domain }}
+            {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
+            {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-management-workers
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -209,9 +216,11 @@ spec:
               value: {{ $.Values.gitLFSSkipSmudge | quote }}
             - name: RENKU_DOMAIN
               value: {{ $.Values.global.renku.domain }}
+            {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: shared-volume
               mountPath: {{ $.Values.cacheDirectory }}
+            {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
 
         - name: {{ $.Chart.Name }}-scheduler
           image: "{{ $version.image.repository }}:{{ $version.image.tag }}"
@@ -238,6 +247,9 @@ spec:
               value: {{ $.Values.sentry.sample_rate | quote }}
             - name: SENTRY_ENV
               value: {{ $.Values.sentry.environment }}
+            {{- include "certificates.env.python" $ | nindent 12 }}
+          volumeMounts:
+            {{- include "certificates.volumeMounts.system" $ | nindent 12 }}
       {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm-chart/renku-core/values.yaml

@@ -4,6 +4,19 @@
 
 replicaCount: 1
 
+global:
+  ## Specify a secret that containes the certificate
+  ## if you would like to use a custom CA. The key for the secret
+  ## should have the .crt extension otherwise it is ignored. The
+  ## keys across all secrets are mounted as files in one location so
+  ## the keys across all secrets have to be unique.
+  certificates:
+    image:
+      repository: renku/certificates
+      tag: '0.0.1'
+    customCAs: []
+      # - secret:
+
 # base path - this is the reverse proxy base path
 apiBasePath: /api
 
@@ -104,7 +117,7 @@ versions:
     fullnameOverride: ""
     image:
       repository: renku/renku-core
-      tag: "v1.0.4"
+      tag: "v1.0.5"
       pullPolicy: IfNotPresent
   v8:
     name: v8

pyproject.toml

@@ -64,8 +64,7 @@ click-option-group = "<0.6.0,>=0.5.2"
 click-plugins = "==1.1.1"
 coverage = { version = "<6.2,>=4.5.3", optional = true }
 cryptography = "<35.1,>=3.4.1"
-cwl-utils = { version = ">=0.12", optional = true }
-cwlgen = "<=0.4.2,>=0.4.0"
+cwl-utils = ">=0.12"
 cwltool = "==3.1.20210922203925"
 deepmerge = "==0.3.0"
 environ-config = "<21.3.0,>=18.2.0"
@@ -169,7 +168,6 @@ tests = [
     "black",
     "check-manifest",
     "coverage",
-    "cwl-utils",
     "fakeredis",
     "flakehell",
     "flaky",
@@ -199,15 +197,13 @@ all = [
     "check-manifest",
     "circus",
     "coverage",
-    "cwl-utils",
     "fakeredis",
     "flakehell",
     "flaky",
     "flask",
     "freezegun",
     "gunicorn",
     "isort",
-    "jinja2",
     "marshmallow",
     "pexpect",
     "pillow",

renku/core/management/migrate.py

@@ -376,7 +376,7 @@ def get_migrations():
     """Return a sorted list of versions and migration modules."""
     migrations = []
     for entry in importlib_resources.files("renku.core.management.migrations").iterdir():
-        match = re.search(r"m_([0-9]{4})__[a-zA-Z0-9_-]*.py", entry.name)
+        match = re.search(r"^m_([0-9]{4})__[a-zA-Z0-9_-]*.py$", entry.name)
 
         if match is None:  # migration files match m_0000__[name].py format
             continue