diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a239a1d..d9fbfd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,10 @@ jobs: ${{ runner.os }}-poetry-${{ matrix.python-version }}- - name: Install dependencies - run: make .venv + run: | + poetry install --no-root + echo "VIRTUAL_ENV=$(poetry env info --path)" >> $GITHUB_ENV + echo "$VIRTUAL_ENV/bin" >> $GITHUB_PATH - name: Lint code run: make lint diff --git a/README.md b/README.md index 81098bc..9f49049 100644 --- a/README.md +++ b/README.md @@ -66,18 +66,12 @@ testing, linting, and more. This project uses [Poetry](https://python-poetry.org/) for dependency management. To set up your development environment: -1. **Create and activate the virtual environment:** +**Create and activate the virtual environment:** ```bash make .venv ``` -2. **Install project dependencies:** - - ```bash - poetry install - ``` - ### Running Normalisation Scripts Each data source module under `src` contains scripts for data normalisation. diff --git a/notebooks/exploration.ipynb b/notebooks/exploration.ipynb new file mode 100644 index 0000000..5b5fb07 --- /dev/null +++ b/notebooks/exploration.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notebook: Exploration of csv files" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"Exploration notebook for data analysis.\n", + "\n", + "This notebook contains data exploration steps for disaster analysis.\n", + "\"\"\"\n", + "\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "\n", + "module_path = Path(\"..\").resolve()\n", + "sys.path.append(str(module_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "metadata": {}, + "outputs": [], + "source": [ + "def read_dat(dat_file: str) -> pd:\n", + " \"\"\"Reads a CSV file from the data_prep directory.\"\"\"\n", + " dat_dir = Path(\"../data_prep/\").resolve()\n", + " dat_path = dat_dir / dat_file\n", + " return pd.read_csv(dat_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [], + "source": [ + "glide_prep_df = read_dat(\"glide_prep.csv\")\n", + "gdacs_prep_df = read_dat(\"gdacs_prep.csv\")\n", + "emdat_prep_df = read_dat(\"emdat_prep.csv\")\n", + "disaster_charter_df = read_dat(\"disaster_charter_prep.csv\")\n", + "cerf_df = read_dat(\"cerf_prep.csv\")\n", + "idmc_df = read_dat(\"idmc_prep.csv\")\n", + "ifrc_df = read_dat(\"ifrc_prep.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "pre_dfs = [\n", + " glide_prep_df,\n", + " gdacs_prep_df,\n", + " emdat_prep_df,\n", + " disaster_charter_df,\n", + " cerf_df,\n", + " idmc_df,\n", + " ifrc_df,\n", + "]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/poetry.lock b/poetry.lock index 142c6cd..7e5ffff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -72,6 +72,21 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +[[package]] +name = "autopep8" +version = "2.3.2" +description = "A tool that automatically formats Python code to conform to the PEP 8 style guide" +optional = false +python-versions = ">=3.9" +files = [ + {file = "autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"}, + {file = "autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758"}, +] + +[package.dependencies] +pycodestyle = ">=2.12.0" +tomli = {version = "*", markers = "python_version < \"3.11\""} + [[package]] name = "azure-core" version = "1.32.0" @@ -1948,6 +1963,26 @@ files = [ msal = ">=1.29,<2" portalocker = ">=1.4,<3" +[[package]] +name = "nbqa" +version = "1.9.1" +description = "Run any standard Python code quality tool on a Jupyter Notebook" +optional = false +python-versions = ">=3.9" +files = [ + {file = "nbqa-1.9.1-py3-none-any.whl", hash = "sha256:95552d2f6c2c038136252a805aa78d85018aef922586270c3a074332737282e5"}, + {file = "nbqa-1.9.1.tar.gz", hash = "sha256:a1f4bcf587c597302fed295951001fc4e1be4ce0e77e1ab1b25ac2fbe3db0cdd"}, +] + +[package.dependencies] +autopep8 = ">=1.5" +ipython = ">=7.8.0" +tokenize-rt = ">=3.2.0" +tomli = "*" + +[package.extras] +toolchain = ["black", "blacken-docs", "flake8", "isort", "jupytext", "mypy", "pylint", "pyupgrade", "ruff"] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -3994,6 +4029,17 @@ files = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] +[[package]] +name = "tokenize-rt" +version = "6.1.0" +description = "A wrapper around the stdlib `tokenize` which roundtrips." +optional = false +python-versions = ">=3.9" +files = [ + {file = "tokenize_rt-6.1.0-py2.py3-none-any.whl", hash = "sha256:d706141cdec4aa5f358945abe36b911b8cbdc844545da99e811250c0cee9b6fc"}, + {file = "tokenize_rt-6.1.0.tar.gz", hash = "sha256:e8ee836616c0877ab7c7b54776d2fefcc3bde714449a206762425ae114b53c86"}, +] + [[package]] name = "tomli" version = "2.0.2" @@ -4373,4 +4419,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "f9a97deb62b63308062f23af2146692b8c76ba4f5e1abdc2dc03292e6aa07c41" +content-hash = "edadcb5b56d048af6f5de19d80772a18f105aca53a3c0dadf1ec8144b904c8bc" diff --git a/pyproject.toml b/pyproject.toml index d3ed39b..ec4aba7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ pymarkdownlnt = "^0.9.23" pytest = "^8.3.3" ruff = "^0.6.8" ipykernel = "^6.29.5" +nbqa = "^1.9.1" [build-system] requires = ["poetry-core"]