Latest internal updates

libs/opsqueue_python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "opsqueue_python"
-version = "0.30.2"
+version = "0.30.4"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

libs/opsqueue_python/opsqueue_python.nix

@@ -30,6 +30,7 @@ buildPythonPackage rec {
       ".toml"
       ".lock"
       ".db"
+      ".md"
     ];
   };
 

opsqueue/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "opsqueue"
-version = "0.30.1"
+version = "0.30.4"
 edition = "2021"
 description = "lightweight batch processing queue for heavy loads"
 repository = "https://github.com/channable/opsqueue"

opsqueue/migrations/20250416081453_track_chunk_size.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE submissions DROP COLUMN chunk_size;
+ALTER TABLE submissions_completed DROP COLUMN chunk_size;
+ALTER TABLE submissions_failed DROP COLUMN chunk_size;

opsqueue/migrations/20250803174028_better_random_order_formula.down.sql

@@ -0,0 +1,38 @@
+-- 1. Drop all indexes and finally the column itself
+DROP INDEX random_chunks_order;
+DROP INDEX random_chunks_metadata_order;
+DROP INDEX random_chunks_metadata_order2;
+ALTER TABLE chunks DROP COLUMN random_order;
+ALTER TABLE chunks_metadata DROP COLUMN random_order;
+
+-- 2. Recreate the column with its **old, bad** definition
+ALTER TABLE chunks ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS (
+    (((submission_id + chunk_index) % 65536) * 40503) % 65536
+    ) VIRTUAL;
+ALTER TABLE chunks_metadata ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS (
+    (((submission_id + chunk_index) % 65536) * 40503) % 65536
+) VIRTUAL;
+
+
+-- 3. Recreate all dropped indexes
+CREATE INDEX random_chunks_order ON chunks (
+      random_order
+    , submission_id
+    , chunk_index
+);
+
+CREATE INDEX random_chunks_metadata_order ON chunks_metadata (
+      metadata_key
+    , metadata_value
+    , random_order
+    , submission_id
+    , chunk_index
+);
+
+CREATE INDEX random_chunks_metadata_order2 ON chunks_metadata (
+      metadata_key
+    , random_order
+    , metadata_value
+    , submission_id
+    , chunk_index
+);

opsqueue/migrations/20250803174028_better_random_order_formula.up.sql

@@ -0,0 +1,43 @@
+-- 1. Drop all indexes and finally the column itself
+DROP INDEX random_chunks_order;
+DROP INDEX random_chunks_metadata_order;
+DROP INDEX random_chunks_metadata_order2;
+ALTER TABLE chunks DROP COLUMN random_order;
+ALTER TABLE chunks_metadata DROP COLUMN random_order;
+
+-- 2. Recreate the column with its new, proper definition
+--
+-- Compared to the OG definition, we ensure that the top 42 bits of `submission_id`
+-- which contain the timestamp part of the snowflake,
+-- always participate in the `random_order`,
+-- since the lower 22 bits are likely to be `0` except when under peak load.
+ALTER TABLE chunks ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS (
+    (((submission_id + (submission_id >> 22) + chunk_index) % 65536) * 40503) % 65536
+    ) VIRTUAL;
+ALTER TABLE chunks_metadata ADD COLUMN random_order INTEGER NOT NULL GENERATED ALWAYS AS (
+    (((submission_id + (submission_id >> 22) + chunk_index) % 65536) * 40503) % 65536
+) VIRTUAL;
+
+
+-- 3. Recreate all dropped indexes
+CREATE INDEX random_chunks_order ON chunks (
+      random_order
+    , submission_id
+    , chunk_index
+);
+
+CREATE INDEX random_chunks_metadata_order ON chunks_metadata (
+      metadata_key
+    , metadata_value
+    , random_order
+    , submission_id
+    , chunk_index
+);
+
+CREATE INDEX random_chunks_metadata_order2 ON chunks_metadata (
+      metadata_key
+    , random_order
+    , metadata_value
+    , submission_id
+    , chunk_index
+);