Add initial analytics

Cargo.toml

@@ -60,7 +60,7 @@ tokio = "1.28.2"
 serial_test = "2.0.0"
 
 [dev-dependencies]
-insta = { version = "1.1.0" }
+insta = { version = "1.29.0", features = ["yaml"] }
 
 [features]
 vendored-libgit2 = ["git2/vendored-libgit2"]

src/resolution.rs

@@ -326,20 +326,22 @@ mod tests {
         package_name::PackageName, package_source::InMemoryRegistry, test_package::PackageBuilder,
     };
 
-    fn test_project(registry: InMemoryRegistry, package: PackageBuilder) -> anyhow::Result<()> {
+    fn test_project(
+        registry: InMemoryRegistry,
+        package: PackageBuilder,
+    ) -> anyhow::Result<Resolve> {
         let package_sources = PackageSourceMap::new(Box::new(registry.source()));
         let manifest = package.into_manifest();
         let resolve = resolve(&manifest, &Default::default(), &package_sources)?;
-        insta::assert_yaml_snapshot!(resolve);
-        Ok(())
+        Ok(resolve)
     }
 
     #[test]
     fn minimal() -> anyhow::Result<()> {
         let registry = InMemoryRegistry::new();
-
         let root = PackageBuilder::new("biff/minimal@0.1.0");
-        test_project(registry, root)
+
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     #[test]
@@ -350,7 +352,8 @@ mod tests {
 
         let root = PackageBuilder::new("biff/one-dependency@0.1.0")
             .with_dep("Minimal", "biff/minimal@0.1.0");
-        test_project(registry, root)
+
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     #[test]
@@ -364,7 +367,8 @@ mod tests {
 
         let root = PackageBuilder::new("biff/transitive-dependency@0.1.0")
             .with_dep("OneDependency", "biff/one-dependency@0.1.0");
-        test_project(registry, root)
+
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     /// When there are shared dependencies, Wally should select the same
@@ -380,7 +384,7 @@ mod tests {
             .with_dep("B", "biff/b@1.0.0")
             .with_dep("C", "biff/c@1.0.0");
 
-        test_project(registry, root)
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     /// Server dependencies are allowed to depend on shared dependencies. If a
@@ -399,7 +403,7 @@ mod tests {
         let root =
             PackageBuilder::new("biff/root@1.0.0").with_server_dep("Server", "biff/server@1.0.0");
 
-        test_project(registry, root)
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     /// but... if that shared dependency is required by another shared dependency,
@@ -418,7 +422,7 @@ mod tests {
             .with_server_dep("Server", "biff/server@1.0.0")
             .with_dep("Shared", "biff/shared@1.0.0");
 
-        test_project(registry, root)
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     /// Shared dependencies are allowed to depend on server dependencies. Server
@@ -431,7 +435,7 @@ mod tests {
         let root =
             PackageBuilder::new("biff/root@1.0.0").with_server_dep("Server", "biff/server@1.0.0");
 
-        test_project(registry, root)
+        Ok(insta::assert_yaml_snapshot!(test_project(registry, root)?))
     }
 
     #[test]

tests/integration/install.rs

@@ -7,42 +7,42 @@ use walkdir::WalkDir;
 
 #[test]
 fn minimal() {
-    run_test("minimal");
+    assert_project_snapshot!(run_test("minimal"));
 }
 
 #[test]
 fn one_dependency() {
-    run_test("one-dependency");
+    assert_project_snapshot!(run_test("one-dependency"));
 }
 
 #[test]
 fn transitive_dependency() {
-    run_test("transitive-dependency");
+    assert_project_snapshot!(run_test("transitive-dependency"));
 }
 
 #[test]
 fn private_with_public_dependency() {
-    run_test("private-with-public-dependency");
+    assert_project_snapshot!(run_test("private-with-public-dependency"));
 }
 
 #[test]
 fn dev_dependency() {
-    run_test("dev-dependency");
+    assert_project_snapshot!(run_test("dev-dependency"));
 }
 
 #[test]
 fn dev_dependency_also_required_as_non_dev() {
-    run_test("dev-dependency-also-required-as-non-dev");
+    assert_project_snapshot!(run_test("dev-dependency-also-required-as-non-dev"));
 }
 
 #[test]
 fn cross_realm_dependency() {
-    run_test("cross-realm-dependency");
+    assert_project_snapshot!(run_test("cross-realm-dependency"));
 }
 
 #[test]
 fn cross_realm_explicit_dependency() {
-    run_test("cross-realm-explicit-dependency");
+    assert_project_snapshot!(run_test("cross-realm-explicit-dependency"));
 }
 
 fn run_test(name: &str) -> TempProject {
@@ -63,7 +63,6 @@ fn run_test(name: &str) -> TempProject {
 
     args.run().unwrap();
 
-    assert_dir_snapshot!(project.path());
     project
 }
 

tests/integration/util.rs

@@ -4,9 +4,9 @@ use std::path::Path;
 use serde::{Deserialize, Serialize};
 
 #[macro_export]
-macro_rules! assert_dir_snapshot {
-    ( $path:expr ) => {
-        let result = crate::util::read_path($path).unwrap();
+macro_rules! assert_project_snapshot {
+    ( $project:expr ) => {
+        let result = crate::util::read_path($project.path()).unwrap();
         insta::assert_yaml_snapshot!(result);
     };
 }

wally-registry-backend/Cargo.toml

@@ -10,6 +10,7 @@ publish = false
 [features]
 default = []
 s3-storage = ["dep:rusoto_core", "dep:rusoto_s3"]
+postgres-analytics = ["dep:sqlx"]
 
 [dependencies]
 wally = { path = ".." }
@@ -37,6 +38,8 @@ url = { version = "2.2.1", features = ["serde"] }
 walkdir = "2.3.1"
 zip = "0.5.11"
 moka = "0.11.1"
+sqlx = { version = "0.6", features = [ "runtime-tokio-rustls", "postgres", "chrono" ], optional = true }
+chrono = "0.4.26"
 
 [dev-dependencies]
 tempfile = "3.1.0"

wally-registry-backend/Rocket.toml

@@ -28,5 +28,10 @@ index_url = "https://github.com/UpliftGames/wally-test-index"
 # Here's the production config:
 # index_url = "https://github.com/UpliftGames/wally-index"
 
+# With the postgres-analytics feature enabled we can enable analytics with:
+# analytics = { type = "postgres", database-url = "...", downloads-table-name = "package_downloads" }
+# 
+# By default there are no package analytics
+
 [release]
 log_level = "normal"

wally-registry-backend/src/analytics/mod.rs

@@ -0,0 +1,48 @@
+#[cfg(feature = "postgres-analytics")]
+mod postgres;
+
+use libwally::package_id::PackageId;
+use serde::{Deserialize, Serialize};
+
+#[cfg(feature = "postgres-analytics")]
+pub use postgres::PostgresAnalytics;
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(tag = "type", rename_all = "kebab-case")]
+pub enum AnalyticsMode {
+    #[cfg(feature = "postgres-analytics")]
+    #[serde(rename_all = "kebab-case")]
+    Postgres {
+        database_url: String,
+        downloads_table_name: String,
+    },
+}
+
+#[derive(Clone)]
+pub enum AnalyticsBackend {
+    #[cfg(feature = "postgres-analytics")]
+    Postgres(PostgresAnalytics),
+}
+
+#[async_trait]
+pub trait AnalyticsBackendProvider: Send + Sync + Clone + 'static {
+    async fn record_download(self, package_id: PackageId) -> anyhow::Result<()>;
+    async fn ensure_initialized(&self) -> anyhow::Result<()>;
+}
+
+#[async_trait]
+impl AnalyticsBackendProvider for AnalyticsBackend {
+    async fn record_download(self, package_id: PackageId) -> anyhow::Result<()> {
+        match self {
+            #[cfg(feature = "postgres-analytics")]
+            AnalyticsBackend::Postgres(backend) => backend.record_download(package_id).await,
+        }
+    }
+
+    async fn ensure_initialized(&self) -> anyhow::Result<()> {
+        match *self {
+            #[cfg(feature = "postgres-analytics")]
+            AnalyticsBackend::Postgres(backend) => backend.ensure_initialized().await,
+        }
+    }
+}

wally-registry-backend/src/analytics/postgres.rs

@@ -0,0 +1,50 @@
+use super::AnalyticsBackendProvider;
+use sqlx::{Pool, Postgres};
+
+#[derive(Clone)]
+pub struct PostgresAnalytics {
+    pool: Pool<Postgres>,
+    table_name: String,
+}
+
+impl PostgresAnalytics {
+    pub fn new(pool: Pool<Postgres>, table_name: String) -> Self {
+        Self { pool, table_name }
+    }
+}
+
+#[async_trait]
+impl AnalyticsBackendProvider for PostgresAnalytics {
+    async fn record_download(
+        self,
+        package_id: libwally::package_id::PackageId,
+    ) -> anyhow::Result<()> {
+        sqlx::query(&format!(
+            "INSERT INTO {} VALUES (NOW(), $1, $2, $3);",
+            self.table_name
+        ))
+        .bind(package_id.name().scope())
+        .bind(package_id.name().name())
+        .bind(package_id.version().to_string())
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(())
+    }
+
+    async fn ensure_initialized(&self) -> anyhow::Result<()> {
+        sqlx::query(&format!(
+            "CREATE TABLE IF NOT EXISTS {} (
+            time TIMESTAMP NOT NULL,
+            package_scope VARCHAR ( 50 ) NOT NULL,
+            package_name VARCHAR ( 50 ) NOT NULL,
+            package_version VARCHAR ( 50 ) NOT NULL
+         );",
+            self.table_name
+        ))
+        .execute(&self.pool)
+        .await?;
+
+        Ok(())
+    }
+}

wally-registry-backend/src/config.rs

@@ -2,7 +2,7 @@ use semver::Version;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
-use crate::{auth::AuthMode, storage::StorageMode};
+use crate::{analytics::AnalyticsMode, auth::AuthMode, storage::StorageMode};
 
 #[derive(Deserialize, Serialize)]
 pub struct Config {
@@ -22,4 +22,7 @@ pub struct Config {
 
     /// The minimum wally cli version required to publish to the registry
     pub minimum_wally_version: Option<Version>,
+
+    /// What analytics backend should be used, currently the only option is Postgres
+    pub analytics: Option<AnalyticsMode>,
 }