M31 rebase

.circleci/config.yml

@@ -1,17 +1,5 @@
 version: 2.0
 
-build-docker-image-job: &build-docker-image-job
-  docker:
-    - image: circleci/node:12
-  steps:
-    - setup_remote_docker
-    - checkout
-    - run: sudo apt update
-    - run: sudo apt install python3-pip
-    - run: sudo pip3 install -r requirements_bundles.txt
-    - run: .circleci/update_version
-    - run: npm run bundle
-    - run: .circleci/docker_build
 jobs:
   backend-lint:
     docker:
@@ -83,68 +71,74 @@ jobs:
           name: Run Visualizations Tests
           command: (cd viz-lib && npm test)
       - run: npm run lint
-  frontend-e2e-tests:
-    environment:
-      COMPOSE_FILE: .circleci/docker-compose.cypress.yml
-      COMPOSE_PROJECT_NAME: cypress
-      PERCY_TOKEN_ENCODED: ZGRiY2ZmZDQ0OTdjMzM5ZWE0ZGQzNTZiOWNkMDRjOTk4Zjg0ZjMxMWRmMDZiM2RjOTYxNDZhOGExMjI4ZDE3MA==
-      CYPRESS_PROJECT_ID_ENCODED: OTI0Y2th
-      CYPRESS_RECORD_KEY_ENCODED: YzA1OTIxMTUtYTA1Yy00NzQ2LWEyMDMtZmZjMDgwZGI2ODgx
+  build-docker-image:
     docker:
       - image: circleci/node:12
     steps:
       - setup_remote_docker
       - checkout
-      - run:
-          name: Install npm dependencies
-          command: |
-            npm ci
-      - run:
-          name: Setup Redash server
-          command: |
-            npm run cypress build
-            npm run cypress start -- --skip-db-seed
-            docker-compose run cypress npm run cypress db-seed
-      - run:
-          name: Execute Cypress tests
-          command: npm run cypress run-ci
-  build-docker-image: *build-docker-image-job
-  build-preview-docker-image: *build-docker-image-job
+      - run: echo "export MOZILLA_VERSION=master" >> $BASH_ENV
+      - run: sudo apt update
+      - run: sudo apt install python3-pip
+      - run: sudo pip3 install -r requirements_bundles.txt
+      - run: .circleci/update_version
+      - run: npm run bundle
+      - run: .circleci/docker_build
+  build-docker-image-tag:
+    docker:
+      - image: circleci/node:12
+    steps:
+      - setup_remote_docker
+      - checkout
+      - run: echo "export MOZILLA_VERSION=$CIRCLE_TAG" >> $BASH_ENV
+      - run: sudo apt update
+      - run: sudo apt install python3-pip
+      - run: sudo pip3 install -r requirements_bundles.txt
+      - run: .circleci/update_version
+      - run: npm run bundle
+      - run: .circleci/docker_build
+      # Create alias from tag to "latest":
+      - run: docker tag $DOCKERHUB_REPO:$CIRCLE_TAG $DOCKERHUB_REPO:latest
+      - run: docker push $DOCKERHUB_REPO:latest
 workflows:
   version: 2
   build:
     jobs:
-      - backend-lint
+      - backend-lint:
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
       - backend-unit-tests:
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
           requires:
             - backend-lint
-      - frontend-lint
+      - frontend-lint:
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
       - frontend-unit-tests:
+          filters:
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/
           requires:
             - backend-lint
             - frontend-lint
-      - frontend-e2e-tests:
-          requires:
-            - frontend-lint
-      - build-preview-docker-image:
+      - build-docker-image:
           requires:
             - backend-unit-tests
             - frontend-unit-tests
-            - frontend-e2e-tests
           filters:
             branches:
               only:
                 - master
-      - hold:
-          type: approval
+      - build-docker-image-tag:
           requires:
             - backend-unit-tests
             - frontend-unit-tests
-            - frontend-e2e-tests
           filters:
             branches:
-              only:
-                - /release\/.*/
-      - build-docker-image:
-          requires:
-            - hold
+              ignore: /.*/
+            tags:
+              only: /^m[0-9]+(\.[0-9]+)?$/

.circleci/docker-compose.circle.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.2'
 services:
   redash:
     build: ../

.circleci/docker-compose.cypress.yml

@@ -1,4 +1,4 @@
-version: '3'
+version: '2.2'
 services:
   server:
     build: ../
@@ -14,6 +14,7 @@ services:
       REDASH_REDIS_URL: "redis://redis:6379/0"
       REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
       REDASH_RATELIMIT_ENABLED: "false"
+      REDASH_ENFORCE_CSRF: "true"
   scheduler:
     build: ../
     command: scheduler

.circleci/docker_build

@@ -1,17 +1,9 @@
 #!/bin/bash
-VERSION=$(jq -r .version package.json)
-VERSION_TAG=$VERSION.b$CIRCLE_BUILD_NUM
+VERSION_TAG="$MOZILLA_VERSION"
 
 docker login -u $DOCKER_USER -p $DOCKER_PASS
 
-if [ $CIRCLE_BRANCH = master ] || [ $CIRCLE_BRANCH = preview-image ]
-then
-    docker build --build-arg skip_dev_deps=true -t redash/redash:preview -t redash/preview:$VERSION_TAG .
-    docker push redash/redash:preview
-    docker push redash/preview:$VERSION_TAG
-else
-    docker build --build-arg skip_dev_deps=true -t redash/redash:$VERSION_TAG .
-    docker push redash/redash:$VERSION_TAG
-fi
+docker build -t $DOCKERHUB_REPO:$VERSION_TAG .
+docker push $DOCKERHUB_REPO:$VERSION_TAG
 
-echo "Built: $VERSION_TAG"
+echo "Built: $VERSION_TAG"

.circleci/update_version

@@ -1,6 +1,8 @@
 #!/bin/bash
+bin/dockerflow-version "$MOZILLA_VERSION"
+
 VERSION=$(jq -r .version package.json)
-FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM-$MOZILLA_VERSION
 
 sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '$FULL_VERSION'/" redash/__init__.py
 sed -i "s/dev/$CIRCLE_SHA1/" client/app/version.json

bin/bundle-extensions

@@ -6,8 +6,8 @@ from pathlib import Path
 from shutil import copy
 from collections import OrderedDict as odict
 
-from importlib_metadata import entry_points
-from importlib_resources import contents, is_resource, path
+import importlib_metadata
+import importlib_resources
 
 # Name of the subdirectory
 BUNDLE_DIRECTORY = "bundle"
@@ -25,18 +25,6 @@ if not extensions_directory.exists():
 os.environ["EXTENSIONS_DIRECTORY"] = str(extensions_relative_path)
 
 
-def resource_isdir(module, resource):
-    """Whether a given resource is a directory in the given module
-
-    https://importlib-resources.readthedocs.io/en/latest/migration.html#pkg-resources-resource-isdir
-    """
-    try:
-        return resource in contents(module) and not is_resource(module, resource)
-    except (ImportError, TypeError):
-        # module isn't a package, so can't have a subdirectory/-package
-        return False
-
-
 def entry_point_module(entry_point):
     """Returns the dotted module path for the given entry point"""
     return entry_point.pattern.match(entry_point.value).group("module")
@@ -77,18 +65,28 @@ def load_bundles():
 
     """
     bundles = odict()
-    for entry_point in entry_points().get("redash.bundles", []):
+    for entry_point in importlib_metadata.entry_points().get("redash.bundles", []):
         logger.info('Loading Redash bundle "%s".', entry_point.name)
         module = entry_point_module(entry_point)
         # Try to get a list of bundle files
-        if not resource_isdir(module, BUNDLE_DIRECTORY):
+        try:
+            bundle_dir = importlib_resources.files(module).joinpath(BUNDLE_DIRECTORY)
+        except (ImportError, TypeError):
+            # Module isn't a package, so can't have a subdirectory/-package
             logger.error(
-                'Redash bundle directory "%s" could not be found.', entry_point.name
+                'Redash bundle module "%s" could not be imported: "%s"',
+                entry_point.name,
+                module,
             )
             continue
-        with path(module, BUNDLE_DIRECTORY) as bundle_dir:
-            bundles[entry_point.name] = list(bundle_dir.rglob("*"))
-
+        if not bundle_dir.is_dir():
+            logger.error(
+                'Redash bundle directory "%s" could not be found or is not a directory: "%s"',
+                entry_point.name,
+                bundle_dir,
+            )
+            continue
+        bundles[entry_point.name] = list(bundle_dir.rglob("*"))
     return bundles
 
 

bin/docker-entrypoint

@@ -2,33 +2,38 @@
 set -e
 
 scheduler() {
+  /app/manage.py db upgrade
   echo "Starting RQ scheduler..."
 
   exec /app/manage.py rq scheduler
 }
 
 dev_scheduler() {
+  /app/manage.py db upgrade
   echo "Starting dev RQ scheduler..."
 
   exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq scheduler
 }
 
 worker() {
+  /app/manage.py db upgrade
   echo "Starting RQ worker..."
 
   export WORKERS_COUNT=${WORKERS_COUNT:-2}
   export QUEUES=${QUEUES:-}
-  
+
   supervisord -c worker.conf
 }
 
 dev_worker() {
+  /app/manage.py db upgrade
   echo "Starting dev RQ worker..."
 
   exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES
 }
 
 server() {
+  /app/manage.py db upgrade
   # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details.
   MAX_REQUESTS=${MAX_REQUESTS:-1000}
   MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100}
@@ -126,4 +131,3 @@ case "$1" in
     exec "$@"
     ;;
 esac
-

bin/dockerflow-version

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eo pipefail
+
+VERSION="$1"
+
+printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+  "$CIRCLE_SHA1" \
+  "$VERSION" \
+  "$CIRCLE_PROJECT_USERNAME" \
+  "$CIRCLE_PROJECT_REPONAME" \
+  "$CIRCLE_BUILD_URL" \
+> version.json

bin/migrations-graph

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+"""
+A quick helper script to print the Alembic migration history
+via Graphiz and show it via GraphvizOnline on
+https://dreampuf.github.io/GraphvizOnline/.
+
+This requires the Graphviz Python library:
+
+   $ pip install --user graphviz
+
+Then run it with the path to the Alembic config file:
+
+   $ migrations-graph --config migrations/alembic.ini
+
+"""
+import os
+import sys
+import urllib.parse
+import urllib.request
+
+import click
+from alembic import util
+from alembic.script import ScriptDirectory
+from alembic.config import Config
+from alembic.util import CommandError
+from graphviz import Digraph
+
+# Make sure redash can be imported in the migration files
+sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+
+
+def get_revisions(config, rev_range=None):
+    script = ScriptDirectory.from_config(config)
+
+    if rev_range is not None:
+        if ":" not in rev_range:
+            raise util.CommandError(
+                "History range requires [start]:[end], [start]:, or :[end]"
+            )
+        base, head = rev_range.strip().split(":")
+    else:
+        base = head = None
+
+    return script.walk_revisions(base=base or "base", head=head or "heads")
+
+
+def generate_revision_graph(revisions):
+    dot = Digraph()
+    for revision in revisions:
+        dot.node(revision.revision)
+        if revision.down_revision is None:
+            dot.edge("base", revision.revision)
+            continue
+        if isinstance(revision.down_revision, str):
+            dot.edge(revision.down_revision, revision.revision)
+            continue
+        for down_revision in revision.down_revision:
+            dot.edge(down_revision, revision.revision)
+    return dot
+
+
+@click.command()
+@click.option("--config", default="alembic.ini", help="path to alembic config file")
+@click.option("--name", default="alembic", help="name of the alembic ini section")
+def cli(config, name):
+    """
+    Generates a simple Graphviz dot file and creates a link to
+    view it online via https://dreampuf.github.io/GraphvizOnline/.
+    """
+    alembic_config = Config(file_=config, ini_section=name)
+    try:
+        revisions = get_revisions(alembic_config)
+    except CommandError as e:
+        sys.exit(e)
+
+    dot = generate_revision_graph(revisions)
+    encoded_dot = urllib.parse.quote(bytes(dot.source, "utf-8"))
+    viz_url = "https://dreampuf.github.io/GraphvizOnline/#%s" % encoded_dot
+    print("Generated graph for migration history in %s: %s " % (config, viz_url))
+
+
+if __name__ == "__main__":
+    cli()