From f6dbde71815da3afa854858ab07c28462edcdd4c Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Fri, 7 Nov 2025 17:53:52 +0100 Subject: [PATCH 1/8] Import fileFilter directly instead of via overlay --- libs/opsqueue_python/opsqueue_python.nix | 10 +++-- nix/overlay.nix | 46 ----------------------- nix/util.nix | 48 ++++++++++++++++++++++++ opsqueue/opsqueue.nix | 9 +++-- 4 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 nix/util.nix diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index f024ec1..d392480 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -1,5 +1,5 @@ { - fileFilter, + lib, buildPythonPackage, rustPlatform, perl, @@ -10,14 +10,18 @@ opentelemetry-exporter-otlp, opentelemetry-sdk, }: +let + root = ../../.; + util = import (root + /nix/util.nix) { inherit lib; }; +in buildPythonPackage rec { pname = "opsqueue"; version = "0.1.0"; pyproject = true; - src = fileFilter { + src = util.fileFilter { name = "opsqueue_python"; - src = ../../.; + src = root; # We're copying slightly too much to the Nix store here, # but using the more granular file filter was very error-prone. diff --git a/nix/overlay.nix b/nix/overlay.nix index 25334e8..ddfd3e0 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -5,55 +5,9 @@ self: super: let sources = import ./sources.nix; - lib = super.pkgs.lib; pythonOverlay = import ./python-overlay.nix { inherit sources; }; in { - # A powerful way of filtering the right files for the src attribute of a derivation. - fileFilter = - { - name, - # Regex whitelist of files - srcWhitelist ? [ ".*" ], - # Blacklist of files and directories anywhere in the repository. Use only - # with files that can appear in multiple places in the repository. - srcGlobalBlacklist ? [ ], - # Global whitelist by suffix, only allow the following file extensions - srcGlobalWhitelist ? [ ], - # Global whitelist by regex, only allow the following file extensions - srcGlobalWhitelistRegex ? [ ], - src, - }: - lib.cleanSourceWith rec { - inherit name src; - filter = - path: type: - let - relativePath = lib.removePrefix (toString src + "/") path; - in - (builtins.any (r: builtins.match r relativePath != null) srcWhitelist) - && - # Whitelist these files - ( - (type == "directory") - || ( - (builtins.length srcGlobalWhitelist != 0) - && (builtins.any (suffix: lib.hasSuffix suffix relativePath) srcGlobalWhitelist) - || ( - (builtins.length srcGlobalWhitelistRegex != 0) - && (builtins.any (r: builtins.match r relativePath != null) srcGlobalWhitelistRegex) - ) - ) - ) - && - # Ignore the files from srcGlobalBlacklist anywhere - !(builtins.elem (baseNameOf path) srcGlobalBlacklist) - && - # Discard editor files, git repo stuff, and other cruft. This one is taken - # from Nixpkgs; we don't need to implement it ourselves. - lib.cleanSourceFilter path type; - }; - # Placing the sources in the overlay gives all packages access to the sources, # and it makes it possible to override them in new overlays. sources = if super ? sources then super.sources // sources else sources; diff --git a/nix/util.nix b/nix/util.nix new file mode 100644 index 0000000..fd17252 --- /dev/null +++ b/nix/util.nix @@ -0,0 +1,48 @@ +# Utility functions to use in nix code +{ lib }: +{ + # A powerful way of filtering the right files for the src attribute of a derivation. + fileFilter = + { + name, + # Regex whitelist of files + srcWhitelist ? [ ".*" ], + # Blacklist of files and directories anywhere in the repository. Use only + # with files that can appear in multiple places in the repository. + srcGlobalBlacklist ? [ ], + # Global whitelist by suffix, only allow the following file extensions + srcGlobalWhitelist ? [ ], + # Global whitelist by regex, only allow the following file extensions + srcGlobalWhitelistRegex ? [ ], + src, + }: + lib.cleanSourceWith rec { + inherit name src; + filter = + path: type: + let + relativePath = lib.removePrefix (toString src + "/") path; + in + (builtins.any (r: builtins.match r relativePath != null) srcWhitelist) + && + # Whitelist these files + ( + (type == "directory") + || ( + (builtins.length srcGlobalWhitelist != 0) + && (builtins.any (suffix: lib.hasSuffix suffix relativePath) srcGlobalWhitelist) + || ( + (builtins.length srcGlobalWhitelistRegex != 0) + && (builtins.any (r: builtins.match r relativePath != null) srcGlobalWhitelistRegex) + ) + ) + ) + && + # Ignore the files from srcGlobalBlacklist anywhere + !(builtins.elem (baseNameOf path) srcGlobalBlacklist) + && + # Discard editor files, git repo stuff, and other cruft. This one is taken + # from Nixpkgs; we don't need to implement it ourselves. + lib.cleanSourceFilter path type; + }; +} diff --git a/opsqueue/opsqueue.nix b/opsqueue/opsqueue.nix index 53c0aed..267907a 100644 --- a/opsqueue/opsqueue.nix +++ b/opsqueue/opsqueue.nix @@ -1,6 +1,5 @@ { - fileFilter, - pkgs, + lib, rustPlatform, # Building options buildType ? "release", @@ -11,6 +10,10 @@ perl, git, }: +let + root = ../.; + util = import (root + /nix/util.nix) { inherit lib; }; +in rustPlatform.buildRustPackage { name = "opsqueue"; inherit @@ -20,7 +23,7 @@ rustPlatform.buildRustPackage { useNextest ; - src = fileFilter { + src = util.fileFilter { name = "opsqueue"; src = ./.; From aaaddcffa73dd2941dda4098f13d34fde53ce11e Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Thu, 6 Nov 2025 18:29:37 +0100 Subject: [PATCH 2/8] Fix comment referring to a private repo --- nix/overlay.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nix/overlay.nix b/nix/overlay.nix index ddfd3e0..293bcdb 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,7 +1,4 @@ -# Overlay for Nixpkgs which holds rust-jobs related packages. -# -# Serves as common overlay for this repository. -# +# Overlay for Nixpkgs which holds all opsqueue related packages. self: super: let sources = import ./sources.nix; From 98d9cdbc1837800435edee27471e5ed8c56dc71b Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Thu, 6 Nov 2025 18:46:06 +0100 Subject: [PATCH 3/8] Remove special pre-commit setup Everything in there ultimately relies on Channable-internal tooling, and is only needed if we want to run pre-commit in a separate environment. For this repo it's okay to just put pre-commit and tools we want to run through it in the same `mkShell`. --- .pre-commit-config.yaml | 1 - default.nix | 6 +++++- nix/overlay.nix | 11 ----------- pre-commit/.envrc | 17 ----------------- pre-commit/README.md | 14 -------------- pre-commit/default.nix | 13 ------------- 6 files changed, 5 insertions(+), 57 deletions(-) delete mode 100644 pre-commit/.envrc delete mode 100644 pre-commit/README.md delete mode 100644 pre-commit/default.nix diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03fe6e4..3c136ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,3 @@ -# Pre-commit config as used by ./build.py. repos: # Use local to make sure the dependencies are taken from pre-commit's own environment and # therefore from our nix derivation. diff --git a/default.nix b/default.nix index 3815d8e..a581bcb 100644 --- a/default.nix +++ b/default.nix @@ -29,8 +29,12 @@ let just # For linting and formatting + biome + nixfmt-rfc-style pre-commit - pre-commit-env + python3Packages.pre-commit-hooks + ruff + rust-with-lsp # For compiling the Rust parts rust-with-lsp diff --git a/nix/overlay.nix b/nix/overlay.nix index 293bcdb..c2c5d84 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -25,15 +25,4 @@ in "rustfmt" ]; }; - - pre-commit-env = self.buildEnv { - name = "pre-commit-env"; - paths = [ - super.python3Packages.pre-commit-hooks - super.nixfmt-rfc-style - super.ruff - super.biome - self.rust-with-lsp - ]; - }; } diff --git a/pre-commit/.envrc b/pre-commit/.envrc deleted file mode 100644 index cab5f86..0000000 --- a/pre-commit/.envrc +++ /dev/null @@ -1,17 +0,0 @@ -# shellcheck disable=SC2148 # .envrc files need no shell directive -shopt -s dotglob globstar - -# Decrease logging output -# shellcheck disable=SC2034 # unused variable is still read by direnv. -DIRENV_LOG_FORMAT= -# Install nix-direnv, which has an improved implementation of `use nix` that -# caches the Nix environment. Note that this URL is cached locally, so it -# doesn't fetch the script every time. -if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" -fi - -nix_direnv_watch_file ../nix/**/*.nix ../nix/sources.json - - -use nix default.nix diff --git a/pre-commit/README.md b/pre-commit/README.md deleted file mode 100644 index a6c9b46..0000000 --- a/pre-commit/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# pre-commit - -This folder exists to massage [pre-commit] to work with Nix and direnv. Our pre-commit setup works -as follows: - -- We install a custom pre-commit hook with the `build.py install pre-commit` command; -- This hook uses `direnv` to cache pre-commit (the tool) and it's dependencies. This in order to - keep the dependencies up to date automatically but still keep the hook fast; -- Those dependencies are declared in a separate Nix file (`default.nix` in this folder). We use a - separate Nix file to minimize the dependencies, which means direnv has to reload less often. -- We need a separate `.envrc` file, located in this folder, as one can not specify a file to the - `direnv exec` command, only a folder. - - [pre-commit]: https://pre-commit.com/ diff --git a/pre-commit/default.nix b/pre-commit/default.nix deleted file mode 100644 index feff433..0000000 --- a/pre-commit/default.nix +++ /dev/null @@ -1,13 +0,0 @@ -let - pkgs = import ../nix/nixpkgs-pinned.nix { }; - - defaultEnv = pkgs.buildEnv { - name = "pre-commit-env"; - paths = with pkgs; [ - # Used to execute the pre-commit hook - pre-commit - pre-commit-env - ]; - }; -in -pkgs.mkShell { packages = [ defaultEnv ]; } From 1795986455a4db5a9eacb86135850d1d969323fc Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Thu, 6 Nov 2025 18:44:24 +0100 Subject: [PATCH 4/8] Use the final-prev convention for overlays instead of self-super --- nix/overlay.nix | 10 +++++----- nix/python-overlay.nix | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nix/overlay.nix b/nix/overlay.nix index c2c5d84..c7c08e7 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,5 +1,5 @@ # Overlay for Nixpkgs which holds all opsqueue related packages. -self: super: +final: prev: let sources = import ./sources.nix; pythonOverlay = import ./python-overlay.nix { inherit sources; }; @@ -7,19 +7,19 @@ in { # Placing the sources in the overlay gives all packages access to the sources, # and it makes it possible to override them in new overlays. - sources = if super ? sources then super.sources // sources else sources; + sources = if prev ? sources then prev.sources // sources else sources; - opsqueue = self.callPackage ../opsqueue/opsqueue.nix { }; + opsqueue = final.callPackage ../opsqueue/opsqueue.nix { }; # The explicit choice is made not to override `python312`, as this will cause a rebuild of many # packages when nixpkgs uses python 3.12 as default python environment. # These packages should not be affected, e.g. cachix. This is because of a transitive # dependency on the Python packages that we override. # In our case cachix > ghc > shpinx > Python libraries. - pythonChannable = super.python312.override { packageOverrides = pythonOverlay; }; + pythonChannable = prev.python312.override { packageOverrides = pythonOverlay; }; # We choose a minimal Rust channel to keep the Nix closure size smaller - rust-with-lsp = self.rust-bin.stable.latest.minimal.override { + rust-with-lsp = final.rust-bin.stable.latest.minimal.override { extensions = [ "clippy" "rustfmt" diff --git a/nix/python-overlay.nix b/nix/python-overlay.nix index 6c737da..9e46acb 100644 --- a/nix/python-overlay.nix +++ b/nix/python-overlay.nix @@ -1,4 +1,6 @@ { sources ? import ./sources.nix, }: -self: super: { opsqueue_python = self.callPackage ../libs/opsqueue_python/opsqueue_python.nix { }; } +final: prev: { + opsqueue_python = final.callPackage ../libs/opsqueue_python/opsqueue_python.nix { }; +} From 08904fd0a0d308370929abb96ce1ceadecbe9932 Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Fri, 7 Nov 2025 17:18:38 +0100 Subject: [PATCH 5/8] Move Rust devshell code out of the nixpkgs overlay --- default.nix | 39 +++++++++++++++++++++++---------------- nix/overlay.nix | 7 ------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/default.nix b/default.nix index a581bcb..ce38ff7 100644 --- a/default.nix +++ b/default.nix @@ -22,33 +22,40 @@ let ] ); + # We choose a minimal Rust channel to keep the Nix closure size smaller + rust = pkgs.rust-bin.stable.latest.minimal.override { + extensions = [ + "clippy" + "rustfmt" + ]; + }; + defaultEnv = pkgs.buildEnv { name = "opsqueue-env-default"; - paths = with pkgs; [ + paths = [ # Command runner - just + pkgs.just # For linting and formatting - biome - nixfmt-rfc-style - pre-commit - python3Packages.pre-commit-hooks - ruff - rust-with-lsp + pkgs.biome + pkgs.nixfmt-rfc-style + pkgs.pre-commit + pkgs.python3Packages.pre-commit-hooks + pkgs.ruff # For compiling the Rust parts - rust-with-lsp - sqlx-cli + rust + pkgs.sqlx-cli # Manage nix pins - niv - nvd + pkgs.niv + pkgs.nvd # Rust build tools - cargo-audit - cargo-edit - cargo-nextest - maturin + pkgs.cargo-audit + pkgs.cargo-edit + pkgs.cargo-nextest + pkgs.maturin ]; }; environments = { diff --git a/nix/overlay.nix b/nix/overlay.nix index c7c08e7..c28ba8a 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -18,11 +18,4 @@ in # In our case cachix > ghc > shpinx > Python libraries. pythonChannable = prev.python312.override { packageOverrides = pythonOverlay; }; - # We choose a minimal Rust channel to keep the Nix closure size smaller - rust-with-lsp = final.rust-bin.stable.latest.minimal.override { - extensions = [ - "clippy" - "rustfmt" - ]; - }; } From 20867fdff6f7e9ecef6548b1a2d2bbd586e58f9f Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Fri, 7 Nov 2025 17:30:39 +0100 Subject: [PATCH 6/8] Remove unnecessary passing around of sources --- nix/overlay.nix | 7 +------ nix/python-overlay.nix | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/nix/overlay.nix b/nix/overlay.nix index c28ba8a..3f8c455 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,14 +1,9 @@ # Overlay for Nixpkgs which holds all opsqueue related packages. final: prev: let - sources = import ./sources.nix; - pythonOverlay = import ./python-overlay.nix { inherit sources; }; + pythonOverlay = import ./python-overlay.nix; in { - # Placing the sources in the overlay gives all packages access to the sources, - # and it makes it possible to override them in new overlays. - sources = if prev ? sources then prev.sources // sources else sources; - opsqueue = final.callPackage ../opsqueue/opsqueue.nix { }; # The explicit choice is made not to override `python312`, as this will cause a rebuild of many diff --git a/nix/python-overlay.nix b/nix/python-overlay.nix index 9e46acb..ad4aba1 100644 --- a/nix/python-overlay.nix +++ b/nix/python-overlay.nix @@ -1,6 +1,3 @@ -{ - sources ? import ./sources.nix, -}: final: prev: { opsqueue_python = final.callPackage ../libs/opsqueue_python/opsqueue_python.nix { }; } From 07e2adb5145c2c8e9ef2230cca6a32d890491e19 Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Fri, 7 Nov 2025 17:59:06 +0100 Subject: [PATCH 7/8] Simplify .envrc --- .envrc | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/.envrc b/.envrc index 78eaec2..6fce732 100644 --- a/.envrc +++ b/.envrc @@ -1,27 +1,16 @@ # shellcheck disable=SC2148 # .envrc files need no shell directive shopt -s dotglob globstar -# NOTE: This file is based on https://github.com/channable/repository-template/blob/master/.envrc -# When making edits that apply to other repositories, please update the file there. - -# Add possibility to run a custom envrc that completely overrides the behavior of this envrc. -CUSTOM_ENVRC=.customenvrc -if [ -f "$CUSTOM_ENVRC" ]; then - echo "Using .customenvrc file" - source_env $CUSTOM_ENVRC -else - # Decrease logging output - # shellcheck disable=SC2034 # unused variable is still read by direnv. - DIRENV_LOG_FORMAT= - # Install nix-direnv, which has an improved implementation of `use nix` that - # caches the Nix environment. Note that this URL is cached locally, so it - # doesn't fetch the script every time. - if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then +# Decrease logging output +# shellcheck disable=SC2034 # unused variable is still read by direnv. +DIRENV_LOG_FORMAT= +# Install nix-direnv, which has an improved implementation of `use nix` that +# caches the Nix environment. Note that this URL is cached locally, so it +# doesn't fetch the script every time. +if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" - fi - - watch_file nix/sources.json nix/sources.nix nix/overlay.nix nix/python-overlay.nix opsqueue/opsqueue.nix /libs/opsqueue_python/opsqueue_python.nix +fi +watch_file ./nix/ ./**/*.nix - use nix default.nix --argstr environment shell -fi +use nix default.nix --argstr environment shell From 30ba761d62d42c3fd21e76076f786fcaf8b3667d Mon Sep 17 00:00:00 2001 From: Daan Rijks Date: Fri, 7 Nov 2025 18:00:02 +0100 Subject: [PATCH 8/8] Remove vendored Nix install script No way that'll actually be kept up to date. --- nix/install | 94 ----------------------------------------------------- 1 file changed, 94 deletions(-) delete mode 100755 nix/install diff --git a/nix/install b/nix/install deleted file mode 100755 index c3cf49d..0000000 --- a/nix/install +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -# Vendored Nix installation script. -# Retrieved 2023-05-01 from https://nixos.org/nix/install -# Updated curl call to include 3 retries. -# Removed hashes for unsupported systems. - -# This script installs the Nix package manager on your system by -# downloading a binary distribution and running its installer script -# (which in turn creates and populates /nix). - -{ # Prevent execution if this script was only partially downloaded -oops() { - echo "$0:" "$@" >&2 - exit 1 -} - -umask 0022 - -tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ - oops "Can't create temporary directory for downloading the Nix binary tarball")" -cleanup() { - rm -rf "$tmpDir" -} -trap cleanup EXIT INT QUIT TERM - -require_util() { - command -v "$1" > /dev/null 2>&1 || - oops "you do not have '$1' installed, which I need to $2" -} - -case "$(uname -s).$(uname -m)" in - Linux.x86_64) - hash=1bdf98f951ce82ad1a12b9e6874f5858cb2aa2e402907e8c3079d8482cb8430b - path=dw0gkcqnig9qwyhkrq0sjrzai63zi6wy/nix-2.15.0-x86_64-linux.tar.xz - system=x86_64-linux - ;; - *) oops "sorry, there is no binary distribution of Nix for your platform";; -esac - -# Use this command-line option to fetch the tarballs using nar-serve or Cachix -if [ "${1:-}" = "--tarball-url-prefix" ]; then - if [ -z "${2:-}" ]; then - oops "missing argument for --tarball-url-prefix" - fi - url=${2}/${path} - shift 2 -else - url=https://releases.nixos.org/nix/nix-2.15.0/nix-2.15.0-$system.tar.xz -fi - -tarball=$tmpDir/nix-2.15.0-$system.tar.xz - -require_util tar "unpack the binary tarball" -if [ "$(uname -s)" != "Darwin" ]; then - require_util xz "unpack the binary tarball" -fi - -if command -v curl --retry 3 > /dev/null 2>&1; then - fetch() { curl --fail -L "$1" -o "$2"; } -elif command -v wget > /dev/null 2>&1; then - fetch() { wget "$1" -O "$2"; } -else - oops "you don't have wget or curl installed, which I need to download the binary tarball" -fi - -echo "downloading Nix 2.15.0 binary tarball for $system from '$url' to '$tmpDir'..." -fetch "$url" "$tarball" || oops "failed to download '$url'" - -if command -v sha256sum > /dev/null 2>&1; then - hash2="$(sha256sum -b "$tarball" | cut -c1-64)" -elif command -v shasum > /dev/null 2>&1; then - hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" -elif command -v openssl > /dev/null 2>&1; then - hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" -else - oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" -fi - -if [ "$hash" != "$hash2" ]; then - oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" -fi - -unpack=$tmpDir/unpack -mkdir -p "$unpack" -tar -xJf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" - -script=$(echo "$unpack"/*/install) - -[ -e "$script" ] || oops "installation script is missing from the binary tarball!" -export INVOKED_FROM_INSTALL_IN=1 -"$script" "$@" - -} # End of wrapping