Lint and format

webapp/.eslintrc.json → .eslintrc.json

@@ -6,38 +6,46 @@
     },
     "extends": [
         "eslint:recommended",
-        "plugin:@typescript-eslint/recommended"
+        "plugin:@typescript-eslint/recommended",
+        "plugin:cypress/recommended"
     ],
     "globals": {
         "Atomics": "readonly",
         "SharedArrayBuffer": "readonly"
     },
-    "plugins": ["@typescript-eslint"],
+    "plugins": [
+        "@typescript-eslint"
+    ],
     "parser": "@typescript-eslint/parser",
     "parserOptions": {
         "ecmaVersion": 11,
         "sourceType": "module"
     },
     "rules": {
+        "@typescript-eslint/no-explicit-any": ["off"],
         "@typescript-eslint/no-this-alias": [
             "error",
             {
-              "allowDestructuring": true, // Allow `const { props, state } = this`; false by default
-              "allowedNames": ["self"] // Allow `const vm= this`; `[]` by default
+                "allowDestructuring": true, // Allow `const { props, state } = this`; false by default
+                "allowedNames": [
+                    "self"
+                ] // Allow `const vm= this`; `[]` by default
             }
         ],
         "@typescript-eslint/ban-types": [
             "error",
             {
-              "extendDefaults": true,
-              "types": {
-                "{}": false
-              }
+                "extendDefaults": true,
+                "types": {
+                    "{}": false
+                }
             }
         ],
         "@typescript-eslint/no-unused-vars": [
             "error",
-            { "argsIgnorePattern": "^_" }
+            {
+                "argsIgnorePattern": "^_"
+            }
         ]
     }
 }

.github/workflows/code_quality.yml

@@ -0,0 +1,48 @@
+name: Code quality
+
+on:
+  push:
+  pull_request:
+    branches: [master]
+    types: [opened, synchronize]
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: eu-west-1
+
+jobs:
+  linting:
+    name: Linting
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        tool:
+          [
+            "isort",
+            "black",
+            "pyupgrade",
+            "flake8",
+            "trailing-whitespace",
+            "end-of-file-fixer",
+            "fix-byte-order-marker",
+            "eslint",
+          ]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - name: Install pre-commit
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade pre-commit
+          pre-commit install
+      - name: Run ${{ matrix.tool }}
+        run: |
+          pre-commit run ${{ matrix.tool }} --all-files
+  testing:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: run docker compose
+        run: docker-compose -f docker-compose.test.yml up --build --exit-code-from pytest --abort-on-container-exit

.github/workflows/pipeline.yml

@@ -4,9 +4,6 @@ on:
   push:
     branches:
       - master
-  pull_request:
-    branches: [master]
-    types: [opened, synchronize]
 
 env:
   ML_IMAGE_TAG: activelearning_ml:test
@@ -16,39 +13,12 @@ env:
   WEBAPP: webapp/
 
 jobs:
-  linting:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-      - name: flake8 Lint
-        uses: py-actions/flake8@v2
-        with:
-          ignore: "E402"
-          max-line-length: "88"
-          path: .
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14.x'
-      - name: eslint lint
-        working-directory: ${{ env.WEBAPP }}
-        run: |
-          npm install eslint
-          npm install @typescript-eslint/eslint-plugin@latest --save-dev
-          npm install @typescript-eslint/parser --save-dev
-          npm run lint
-
-
   deployment:
     runs-on: ubuntu-20.04
-    needs: [linting]
-    if: ${{ github.event_name == 'push' }}
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: '50'
+          fetch-depth: "50"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
@@ -72,7 +42,7 @@ jobs:
           tags: ${{ secrets.DOCKERHUB_USERNAME }}/${{ env.API_IMAGE_TAG }}
       - uses: actions/setup-node@v2
         with:
-          node-version: '14.x'
+          node-version: "14.x"
       - name: Build web app
         working-directory: ${{ env.WEBAPP }}
         run: |
@@ -83,7 +53,7 @@ jobs:
         uses: JamesIves/github-pages-deploy-action@v4.2.5
         with:
           branch: gh-pages
-          folder: ${{ env.WEBAPP }}/dist       
+          folder: ${{ env.WEBAPP }}/dist
       - name: Bump version and push tag
         uses: anothrNick/github-tag-action@1.36.0
         if: ${{ !contains(join(github.event.commits.*.message, ' '), '#skip') }}

.gitignore

@@ -25,6 +25,7 @@ unlabeled*/
 node_modules/
 dist/
 .DS_Store
+webapp/cypress/videos/
 
 #api
 app.db

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v2.31.0
+    hooks:
+      - id: pyupgrade
+        args: ["--py39-plus"]
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [pep8-naming]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: "v8.16.0"
+    hooks:
+      - id: eslint
+        files: \.[jt]s?$
+        types: [file]
+        exclude: live\S*\.ts
+        additional_dependencies:
+          - eslint@latest
+          - "@typescript-eslint/eslint-plugin@latest"
+          - "@typescript-eslint/parser@latest"
+          - "eslint-plugin-cypress@latest"

README.md

@@ -21,18 +21,30 @@ The tool has been divided into different modules:
 -   **Preprocessing of unlabeled data:** it contains a Python script that generates mp3 files and spectrograms, given a directory containing audio data.
 -   **A Machine Learning Endpoint:** which is a small flask app that -- given an h5 file, a labeled dataset and an unlabeled dataset -- trains an ML model on the labeled data and predicts on the unlabeled data.
 
-Go into each directory to learn how to get started with it. You will want to visit the directories in this order: preprocess --> train_and_predict --> api --> webapp.
+## Installation and local setup:
 
-**Flowchart of how the tool works:**  
+We highly recommend that you use [Docker](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) to setup the environment, as it will save you a lot of trouble with resolving dependency conflicts and updates.
+
+To run the services in development mode:
+
+`docker-compose -f docker-compose.dev.yml up --build`
+
+To run testing for the services:
+
+`docker-compose -f docker-compose.test.yml up --build --exit-code-from pytest --abort-on-container-exit`
+
+If you still wish to setup each individual service locally, you can find more instructions by going into each directory to learn how to get started. You will want to visit the directories in this order: preprocess --> train_and_predict --> api --> webapp. Do note that some of the instructions might also be outdated so you'll likely need to fix them yourself.
+
+**Flowchart of how the tool works:**
 ![flowchart](api/assets/flowchart.jpg)
 
 **Architecture of the tool:**
 ![architecture](api/assets/architecture.png)
 
 ## Blog posts about the project and GSoC
 
-[My Journey as a Google Summer of Code participant with Orcasound](http://www.orcasound.net/2020/08/26/my-journey-as-a-google-summer-of-code-participant-with-orcasound/)  
-[Two months listening and learning about orca sounds](http://www.orcasound.net/2020/08/02/two-months-as-a-gsoc-participant-with-orcasound/)  
+[My Journey as a Google Summer of Code participant with Orcasound](http://www.orcasound.net/2020/08/26/my-journey-as-a-google-summer-of-code-participant-with-orcasound/)
+[Two months listening and learning about orca sounds](http://www.orcasound.net/2020/08/02/two-months-as-a-gsoc-participant-with-orcasound/)
 [GSoC’s active learning tool: progress after 1 month](http://www.orcasound.net/2020/07/01/gsocs-active-learning-tool-progress-after-1-month/)
 
 ## Questions?
@@ -49,4 +61,3 @@ More questions about being a GSoC mentor or student? Check out the [GSoC mentor
 * [Docker](https://github.com/topics/docker)
 * [PostgreSQL](https://www.postgresql.org)
 * [Flask](https://flask.palletsprojects.com/en/1.1.x/)
-

api/.dockerignore

@@ -0,0 +1,19 @@
+#common
+.vscode/
+__pycache__/
+.gitignore
+.dockerignore
+Dockerfile
+
+#environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+#api related
+.pytest_cache/
+assets/

api/.flake8

@@ -1,2 +1,2 @@
 [flake8]
-ignore = E402, F401, W292, E501
+ignore = E402, F401, W292, E501

api/.flaskenv

@@ -1,2 +1,2 @@
 FLASK_APP=app
-FLASK_ENV=development
+FLASK_ENV=development

api/Dockerfile

@@ -1,19 +1,31 @@
-FROM python:3.8
+FROM python:3.8 as base
 WORKDIR /orcagsoc
-COPY Pipfile Pipfile
-COPY Pipfile.lock Pipfile.lock
-RUN pip install pipenv
-RUN pipenv install --deploy --ignore-pipfile
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-RUN apt install unzip
-RUN unzip awscliv2.zip
-RUN ./aws/install
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+COPY requirements.txt .
+RUN pip install -r requirements.txt
 
+FROM python:3.8 as base_dependencies
+WORKDIR /orcagsoc
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && ./aws/install && \
+    rm -rf awscliv2.zip aws/
+COPY --from=base /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
 COPY app app
 COPY migrations migrations
-COPY boot.sh ./
-
 ENV FLASK_APP app
-
 EXPOSE 5000
-ENTRYPOINT ["./boot.sh"]
+
+FROM base_dependencies as dev
+CMD ["sh", "-c", "flask db upgrade && flask run --host 0.0.0.0"]
+
+FROM base_dependencies as prod
+COPY boot.sh ./
+RUN pip install gunicorn
+ENTRYPOINT ["./boot.sh"]
+
+FROM base_dependencies as test
+COPY tests/ tests/
+RUN pip install pytest
+CMD ["python", "-m", "pytest"]

api/Procfile

@@ -1 +1 @@
-web: flask db upgrade; gunicorn app:app
+web: flask db upgrade; gunicorn app:app