Add some simple module-documentation

Author: Qqwy

opsqueue/src/common/chunk.rs

@@ -1,3 +1,8 @@
+//! Dealing with `Chunk``s, the most fine-grained datatype the queue itself works with.
+//!
+//! While the smallest datatype is the 'Operation', the Opsqueue queue binary itself
+//! only deals with _chunks_ of them. The operations inside of them are hidden and only read/written
+//! from/to object store all the way in the client.
 use chrono::{DateTime, Utc};
 
 use serde::{Deserialize, Serialize};

opsqueue/src/common/errors.rs

@@ -1,3 +1,11 @@
+//! Common error types
+//!
+//! You might notice in many places in Opsqueue that we use very fine-grained error types,
+//! and combine them together using the `E` helper.
+//!
+//! This is a conscious choice: While it makes some function signatures more complex,
+//! it allows us to be super precise in what kind of errors can and cannot occur
+//! in certain API calls.
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 

opsqueue/src/common/mod.rs

@@ -1,3 +1,4 @@
+//! Common datatypes and errors shared across all parts of Opsqueue
 use rustc_hash::FxHashMap;
 
 pub mod chunk;

opsqueue/src/common/submission.rs

@@ -1,3 +1,4 @@
+//! Dealing with `Submission`s: Collections of (`Chunks`s of) operations.
 use std::fmt::Display;
 use std::time::Duration;
 

opsqueue/src/config.rs

@@ -1,3 +1,7 @@
+//! Defines the source of truth for configuring the Opsqueue queue
+//!
+//! We make use of the excellent `clap` crate to make customizing the configuration
+//! with command-line args easier.
 use std::num::NonZero;
 
 use clap::Parser;

opsqueue/src/consumer/mod.rs

@@ -1,3 +1,4 @@
+//! The Consumer side: Interface to reserve, work on, and complete/fail individual Chunks
 pub mod common;
 pub mod strategy;
 

opsqueue/src/lib.rs

@@ -1,3 +1,24 @@
+//! Opsqueue: A lightweight batch processing queue for heavy loads
+//!
+//! Simple 'getting started' instructions can be found [in the repository README](https://github.com/channable/opsqueue/).
+//!
+//! The Rust codebase defines both the 'server', which makes up the bulk of the Opsqueue binary itself,
+//! and the 'client', which is the common part of functionality that clients written in different other programming languages
+//! can all use.
+//!
+//! Many datatypes are shared between this server and client, and therefore their code lives together in the same crate.
+//! Instead, we use feature-flags (`client-logic` and `server-logic`) to decide what concrete parts to include when building.
+//! The set of dependencies is based on these same feature-flags.
+//! Most interestingly, in the test suite we enable both feature-flags so we're able to do a bunch of round-trip testing
+//! immediately in Rust code.
+//!
+//! # Module setup
+//! - The basic logic is divided in the `producer` and `consumer` modules. These both have their own `db` submodule.
+//! - Common functionality and datatypes exists in the `common` module
+//! - Common database helpers live in the `db` module.
+//! - Reading/writing to object stores like GCS or S3 is abstracted in the `object_store` module.
+//! - Finally, extra modules to have a single source of truth for configuration of the queue, and to nicely do tracing and expose metrics exist.
+
 pub mod common;
 pub mod consumer;
 pub mod producer;

opsqueue/src/producer/mod.rs

@@ -1,3 +1,4 @@
+//! The Producer side: Interface to create and read submissions
 #[cfg(feature = "client-logic")]
 pub mod client;
 pub mod common;

opsqueue/src/prometheus.rs

@@ -1,3 +1,9 @@
+//! Define common metrics exposed via a Prometheus endpoint
+//!
+//! This allows inspection of how well the queue is performing under production load.
+//!
+//! Note that we explicitly have a separate endpoint to check the queue health,
+//! which is more fine-grained than Prometheus' way to check whether a service is 'up'.
 use axum_prometheus::{
     metrics::{describe_counter, describe_gauge, describe_histogram, gauge, Unit},
     metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle},

opsqueue/src/server.rs

@@ -1,3 +1,4 @@
+//! Defines the HTTP endpoints that are used by both the `producer` and `consumer` APIs
 use std::{
     any::Any,
     sync::{atomic::AtomicBool, Arc},