Merge branch 'main' into pre-commit-ci-update-config

Author: jpn--

.exercises/README.md

@@ -0,0 +1,6 @@
+# .exercises
+
+In this directory you will find scripts that will exercise the Sharrow library.
+The initial exercises are `mtc` and `sandag`, which include bash scripts that
+will reproduce the respective example model's unit tests that are otherwise run
+using GitHub Actions.  This allows the user to easily test those examples locally.

.exercises/mtc/run_mtc.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# This script runs the MTC example model with sharrow, mirroring the GitHub Actions workflow.
+
+cd $(dirname "$0")
+
+for repo in "driftlesslabs/activitysim" "ActivitySim/activitysim-prototype-mtc"; do
+    dir=$(basename "$repo")
+    if [ ! -d "$dir" ] || [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+        gh repo clone "$repo" -- --depth 1
+    else
+        git -C "$dir" pull --ff-only || git -C "$dir" pull
+    fi
+done
+
+uv venv
+source .venv/bin/activate
+uv pip install -e ../.. # install sharrow in editable mode
+uv pip install ./activitysim
+uv pip install pytest nbmake
+
+cd activitysim-prototype-mtc
+python -m pytest ./test

.exercises/sandag/run_sandag.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# This script runs the MTC example model with sharrow, mirroring the GitHub Actions workflow.
+
+cd $(dirname "$0")
+
+for repo in "driftlesslabs/activitysim" "ActivitySim/sandag-abm3-example"; do
+    dir=$(basename "$repo")
+    if [ ! -d "$dir" ] || [ -z "$(ls -A "$dir" 2>/dev/null)" ]; then
+        gh repo clone "$repo" -- --depth 1
+    else
+        git -C "$dir" pull --ff-only || git -C "$dir" pull
+    fi
+done
+
+uv venv
+source .venv/bin/activate
+uv pip install -e ../.. # install sharrow in editable mode
+uv pip install ./activitysim
+uv pip install pytest nbmake
+
+cd sandag-abm3-example
+python -m pytest ./test

.github/workflows/run-tests.yml

@@ -7,8 +7,6 @@ on:
       - 'v[0-9]+.[0-9]+**'
   pull_request:
     branches: [ main, develop ]
-    tags:
-      - 'v[0-9]+.[0-9]+**'
   workflow_dispatch:
 
 jobs:
@@ -21,16 +19,12 @@ jobs:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Code Quality Check with Ruff
+        # code quality check, stop the build for any errors
+        uses: astral-sh/ruff-action@v3
         with:
-          python-version: '3.11'
-      - name: Install Ruff
-        run: |
-          python -m pip install ruff
-      - name: Lint with Ruff
-        run: |
-          # code quality check, stop the build for any errors
-          ruff check . --show-fixes --exit-non-zero-on-fix
+          version: "latest"
+          args: "check . --show-fixes --exit-non-zero-on-fix"
 
   test-minimal:
     needs: fmt
@@ -41,72 +35,61 @@ jobs:
         shell: bash -l {0}
     steps:
         - uses: actions/checkout@v4
-        - uses: actions/setup-python@v5
+        - uses: astral-sh/setup-uv@v7
           with:
+            version: "0.9.6"
+            enable-cache: true
+            cache-dependency-glob: "uv.lock"
             python-version: '3.11'
-        - name: Install pytest
-          run: |
-            python -m pip install pytest pytest-cov pytest-regressions pytest-xdist nbmake
-        - name: Install sharrow
-          run: |
-            python -m pip install .
         - name: Initial simple tests
           # tests that sharrow can be imported and that categorical tests can be run
           run: |
-            python -m pytest sharrow/tests/test_categorical.py
-        - name: Install openmatrix
-          run: |
-            python -m pip install openmatrix
+            uv run pytest sharrow/tests/test_categorical.py
         - name: Dataset tests
           # tests that the datasets can be read and that the tests can be run
           run: |
-            python -m pytest sharrow/tests/test_datasets.py
-        - name: Install zarr and dask-diagnostics
-          run: |
-            python -m pip install zarr "dask[diagnostics]"
+            uv run pytest sharrow/tests/test_datasets.py
         - name: More complete test with pytest
           run: |
-            python -m pytest -v --disable-warnings sharrow/tests
+            uv run pytest -v --disable-warnings sharrow/tests
 
   test:
     needs: fmt
     name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.10", "3.11", "3.12"]
+        os: ["ubuntu-latest", "windows-latest"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+      fail-fast: false
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@v4
-      - name: Install Python and Dependencies
-        uses: conda-incubator/setup-miniconda@v3
+      - uses: astral-sh/setup-uv@v7
         with:
-          miniforge-version: latest
-          environment-file: envs/testing.yml
+          version: "0.9.6"
           python-version: ${{ matrix.python-version }}
-          activate-environment: testing-env
-          auto-activate-base: false
-          auto-update-conda: false
-      - name: Install sharrow
-        run: |
-          python -m pip install .
-      - name: Conda checkup
+      - name: File contents
         run: |
-          conda info -a
-          conda list
-      - name: Lint with Ruff
+          cat sharrow/example_data.py
+      - name: UV sync
         run: |
-          # code quality check
-          # stop the build if there are Python syntax errors or undefined names
-          ruff check . --select=E9,F63,F7,F82 --no-fix
-          # stop the build for any other configured Ruff linting errors
-          ruff check . --show-fixes --exit-non-zero-on-fix
+          uv self version
+          uv cache clean
+          uv sync --locked
+      - name: Syntax Check with Ruff
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "check . --select=E9,F63,F7,F82 --no-fix"
+      - name: Code Quality Check with Ruff
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "check . --show-fixes --exit-non-zero-on-fix"
       - name: Test with pytest
         run: |
-          python -m pytest
+          uv run --locked pytest
 
   deploy-docs:
     needs: test
@@ -165,6 +148,7 @@ jobs:
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}
+
   activitysim-examples:
     # test that updates to sharrow will not break the activitysim canonical examples
     needs: fmt
@@ -177,17 +161,18 @@ jobs:
           - region: ActivitySim 1-Zone Example (MTC)
             region-org: ActivitySim
             region-repo: activitysim-prototype-mtc
-            region-branch: pandas2
+            region-branch: extended
           - region: ActivitySim 2-Zone Example (SANDAG)
             region-org: ActivitySim
             region-repo: sandag-abm3-example
-            region-branch: pandas2
+            region-branch: main
       fail-fast: false
     defaults:
       run:
         shell: bash -l {0}
     name: ${{ matrix.region }}
     runs-on: ubuntu-latest
+    timeout-minutes: 720 # Sets the timeout to 12 hours
     steps:
       - name: Checkout Sharrow
         uses: actions/checkout@v4
@@ -201,45 +186,37 @@ jobs:
           ref: 'main'
           path: 'activitysim'
 
-      - name: Setup Miniforge
-        uses: conda-incubator/setup-miniconda@v3
+      - name: Setup UV
+        uses: astral-sh/setup-uv@v7
         with:
-          miniforge-version: latest
-          activate-environment: asim-test
+          version: "0.9.6"
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
           python-version: ${{ env.python-version }}
 
       - name: Set cache date for year and month
         run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV
 
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ${{ env.CONDA }}/envs
-            ~/.cache/ActivitySim
-          key: ${{ env.label }}-conda-${{ hashFiles('activitysim/conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }}
-        id: cache
-
-      - name: Update environment
+      - name: Create Virtual Env
         run: |
-          conda env update -n asim-test -f activitysim/conda-environments/github-actions-tests.yml
-        if: steps.cache.outputs.cache-hit != 'true'
-
-      - name: Install sharrow
-        # installing from source
-        run: |
-          python -m pip install ./sharrow
-
-      - name: Install activitysim
-        # installing without dependencies is faster, we trust that all needed dependencies
-        # are in the conda environment defined above.  Also, this avoids pip getting
-        # confused and reinstalling tables (pytables).
-        run: |
-          python -m pip install ./activitysim --no-deps
-
-      - name: Conda checkup
+          uv venv
+          source .venv/bin/activate
+          uv pip install "black==22.12.0" "coveralls==3.3.1" \
+                         "cytoolz==0.12.2" "dask==2023.11.*" "isort==5.12.0" \
+                         "multimethod<2.0" "nbmake==1.4.6" "numba==0.57.*" \
+                         "numpy==1.24.*" "openmatrix==0.3.5.0" "orca==1.8" \
+                         "pandera>=0.15,<0.18.1" "pandas==2.2.*" "platformdirs==3.2.*" \
+                         "psutil==5.9.*" "pyarrow==11.*" "pydantic==2.6.*" "pypyr==5.8.*" \
+                         "tables>=3.9" "pytest==7.2.*" "pytest-cov" "pytest-regressions" \
+                         "pyyaml==6.*" "requests==2.28.*" "ruff" "scikit-learn==1.2.*" \
+                         "sharrow>=2.9.1" "simwrapper>1.7" "sparse" "xarray==2025.01.*" \
+                         "zarr>=2,<3" "zstandard" \
+                         ./sharrow ./activitysim
+
+      - name: UV checkup
         run: |
-          conda info -a
-          conda list
+          source .venv/bin/activate
+          uv pip list
 
       - name: Checkout Example
         uses: actions/checkout@v4
@@ -250,5 +227,6 @@ jobs:
 
       - name: Test ${{ matrix.region }}
         run: |
-          cd ${{ matrix.region-repo }}/test
-          python -m pytest .
+          source .venv/bin/activate
+          cd ${{ matrix.region-repo }}
+          python -m pytest ./test

docs/walkthrough/one-dim.ipynb

@@ -305,8 +305,8 @@
     "\n",
     "Then, it's time to prepare our data.  We'll create a `DataTree`\n",
     "that defines the relationships among all the datasets we're working\n",
-    "with.  This is a tree in the mathematical sense, with nodes referencing\n",
-    "the datasets and edges representing the relationships."
+    "with.  This is a tree roughly in the mathematical sense, with nodes referencing\n",
+    "the dataset dimensions and edges representing the relationships."
    ]
   },
   {
@@ -355,25 +355,41 @@
    "source": [
     "The first named dataset we include, `tour`, is by default the root node of this data tree.\n",
     "We then can define an arbitrary number of other named data nodes.  Here, we add `person`, `hh`,\n",
-    "`odt_skims` and `odt_skims`.  Note that these last two are actually two different names for the\n",
+    "`odt_skims` and `dot_skims`.  Note that these last two are actually two different names for the\n",
     "same underlying dataset, and for each name we will next define a unique set of relationships.\n",
+    "For each of these other data nodes, we will need to define some way to link each dimension of\n",
+    "them back to the root node, so that for any position in the root node's arrays, we can find\n",
+    "one corresponding value in each of the other datasets variables.\n",
     "\n",
     "All data nodes in this tree are stored as `Dataset` objects. We can give a pandas DataFrame\n",
-    "in this contructor instead, but it will be automatically converted into a one-dimension `Dataset`.\n",
+    "in this constructor instead, but it will be automatically converted into a one-dimension `Dataset`.\n",
     "The conversion is no-copy if possible (and it is usually possible) so no additional memory is\n",
     "consumed in the conversion.\n",
     "\n",
     "The `relationships` defines links of the data tree. Each relationship maps a particular variable\n",
     "in a named upstream dataset to a particular dimension of a named downstream dataset.  For example,\n",
     "`\"person.household_id @ hh.HHID\"` tells the tree that the `household_id` variable in the `person` \n",
-    "dataset contains labels (`@`) that map to the `HHID` dimension of the `hh` dataset.\n",
+    "dataset contains labels (`@`) that map to the `HHID` dimension of the `hh` dataset. Similarly,\n",
+    "`\"tour.PERID @ person.PERID\"` tells the tree that the `PERID` variable in the `tour` dataset\n",
+    "contains labels that map to the `PERID` dimension of the `person` dataset.  From this, we can\n",
+    "see that any position in the \"tour\" dataset can be mapped to a position in the \"person\" dataset,\n",
+    "in a many-to-one manner, and from there to a position in the \"hh\" dataset, also in a many-to-one\n",
+    "manner. Unlike tours, persons, and households, the `skims` datasets are multi-dimensional, so we need to\n",
+    "map multiple dimensions.  For the `odt_skims` dataset, we map the origin TAZ dimension (`otaz`)\n",
+    "to the household TAZ (`hh.TAZ`), and the destination TAZ dimension (`dtaz`) to the tour\n",
+    "destination TAZ (`tour.dest_taz_idx`), and the time period dimension (`time_period`) to the\n",
+    "tour outbound time period (`tour.out_time_period`).  This way, even though the skims dataset\n",
+    "is multi-dimensional, we can still find one unique position in the skims dataset for each\n",
+    "position in the tours dataset.  The same is done for the `dot_skims` dataset, which actually\n",
+    "contains the same data as `odt_skims`, but the mapping of the dimensions is different, so a\n",
+    "different unique position in the skims dataset is found for each position in the tours dataset.\n",
     "\n",
     "In addition to mapping by label, we can also map by position, by using the `->` operator in the\n",
     "relationship string instead of `@`.  In the example above, we map the tour destination TAZ's in\n",
     "this manner, as the `dest_taz_idx` variable in the `tours` dataset contains positional references\n",
     "instead of labels.\n",
     "\n",
-    "A special case for the relationship mapping is available when the source varibable\n",
+    "A special case for the relationship mapping is available when the source variable\n",
     "in the upstream dataset is explicitly categorical.  In this case, sharrow checks that\n",
     "the categories exactly match the labels in the referenced downstream dataset dimension,\n",
     "and that there are no missing categorical values. If they do match and there are no\n",
@@ -1450,7 +1466,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "wide_logsums = wide_flow.logit_draws(b, logsums=1, compile_watch=\"simple\")[-1]"
+    "wide_logsums = wide_flow.logit_draws(b, logsums=1, compile_watch=True)[-1]"
    ]
   },
   {
@@ -1460,7 +1476,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%time wide_logsums = wide_flow.logit_draws(b, logsums=1, compile_watch=\"simple\")[-1]\n",
+    "%time wide_logsums = wide_flow.logit_draws(b, logsums=1, compile_watch=True)[-1]\n",
     "wide_logsums"
    ]
   },