feat: Opt-in weekly sending of statistics

This way, the statistics / self-reporting bot will be made into an opt-in regular sending of statistics, where you enable the setting once and then they will be sent automatically. The statistics will be sent to a bot, so that the user can see exactly which data is being sent, and how often. The chat will be archived and muted by default, so that it doesn't disturb the user.

Author: Hocuri

assets/self-reporting-bot.vcf renamed to assets/statistics-bot.vcf

@@ -383,11 +383,6 @@ impl CommandApi {
         Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path())
     }
 
-    async fn draft_self_report(&self, account_id: u32) -> Result<u32> {
-        let ctx = self.get_context(account_id).await?;
-        Ok(ctx.draft_self_report().await?.to_u32())
-    }
-
     /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
@@ -877,12 +872,22 @@ impl CommandApi {
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
     ///     to `check_qr()`.
     ///
+    /// **source** and **uipath** are for statistics-sending,
+    /// if the user enabled it in the settings;
+    /// if you don't have statistics-sending implemented, just pass `None` here.
+    ///
     /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat.
     ///         A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified.
     ///
-    async fn secure_join(&self, account_id: u32, qr: String) -> Result<u32> {
+    async fn secure_join(
+        &self,
+        account_id: u32,
+        qr: String,
+        source: Option<u32>,
+        uipath: Option<u32>,
+    ) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let chat_id = securejoin::join_securejoin(&ctx, &qr).await?;
+        let chat_id = securejoin::join_securejoin_with_source(&ctx, &qr, source, uipath).await?;
         Ok(chat_id.to_u32())
     }
 

deltachat-jsonrpc/src/api.rs

@@ -1,7 +1,7 @@
 use std::time::{Duration, SystemTime};
 
-use anyhow::{bail, Context as _, Result};
-use deltachat::chat::{self, get_chat_contacts, get_past_chat_contacts, ChatVisibility};
+use anyhow::{Context as _, Result, bail};
+use deltachat::chat::{self, ChatVisibility, get_chat_contacts, get_past_chat_contacts};
 use deltachat::chat::{Chat, ChatId};
 use deltachat::constants::Chattype;
 use deltachat::contact::{Contact, ContactId};

deltachat-jsonrpc/src/api/types/chat.rs

@@ -4,7 +4,7 @@ use deltachat::chatlist::get_last_message_for_chat;
 use deltachat::constants::*;
 use deltachat::contact::{Contact, ContactId};
 use deltachat::{
-    chat::{get_chat_contacts, ChatVisibility},
+    chat::{ChatVisibility, get_chat_contacts},
     chatlist::Chatlist,
 };
 use num_traits::cast::ToPrimitive;

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -16,7 +16,7 @@ pub struct HttpResponse {
 
 impl From<CoreHttpResponse> for HttpResponse {
     fn from(response: CoreHttpResponse) -> Self {
-        use base64::{engine::general_purpose, Engine as _};
+        use base64::{Engine as _, engine::general_purpose};
         let blob = general_purpose::STANDARD_NO_PAD.encode(response.blob);
         let mimetype = response.mimetype;
         let encoding = response.encoding;

deltachat-jsonrpc/src/api/types/http.rs

@@ -431,9 +431,26 @@ pub enum Config {
     /// used for signatures, encryption to self and included in `Autocrypt` header.
     KeyId,
 
-    /// This key is sent to the self_reporting bot so that the bot can recognize the user
+    /// Send statistics to Delta Chat's developers.
+    /// Can be exposed to the user as a setting.
+    SendStatistics,
+
+    /// Last time statistics were sent to Delta Chat's developers
+    LastStatisticsSent,
+
+    /// This key is sent to the statistics bot so that the bot can recognize the user
     /// without storing the email address
-    SelfReportingId,
+    StatisticsId,
+
+    /// The last message id that was already included in the previously sent statistics,
+    /// or that already existed before the user opted in.
+    /// Only messages with an id larger than this
+    /// will be counted in the next statistics.
+    StatsLastExcludedMsgId,
+
+    /// The last contact id that already existed when statistics-sending was enabled.
+    /// All newer contacts get the `"new": true` attribute.
+    StatsLastOldContactId,
 
     /// MsgId of webxdc map integration.
     WebxdcIntegration,
@@ -827,6 +844,11 @@ impl Context {
                     .await?;
                 }
             }
+            Config::SendStatistics => {
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+                crate::statistics::set_last_excluded_msg_id(self).await?;
+                crate::statistics::set_last_old_contact_id(self).await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }

src/config.rs

@@ -10,35 +10,30 @@ use std::time::Duration;
 
 use anyhow::{Context as _, Result, bail, ensure};
 use async_channel::{self as channel, Receiver, Sender};
-use pgp::types::PublicKeyTrait;
 use ratelimit::Ratelimit;
 use tokio::sync::{Mutex, Notify, RwLock};
 
-use crate::chat::{ChatId, ProtectionStatus, get_chat_cnt};
+use crate::chat::{ChatId, get_chat_cnt};
 use crate::chatlist_events;
 use crate::config::Config;
-use crate::constants::{
-    self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR,
-};
-use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified};
+use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR};
+use crate::contact::{Contact, ContactId};
 use crate::debug_logging::DebugLogging;
-use crate::download::DownloadState;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::imap::{FolderMeaning, Imap, ServerMetadata};
-use crate::key::{load_self_secret_key, self_fingerprint};
+use crate::key::self_fingerprint;
 use crate::log::{info, warn};
 use crate::logged_debug_assert;
 use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam};
-use crate::message::{self, Message, MessageState, MsgId};
-use crate::param::{Param, Params};
+use crate::message::{self, MessageState, MsgId};
 use crate::peer_channels::Iroh;
 use crate::push::PushSubscriber;
 use crate::quota::QuotaInfo;
 use crate::scheduler::{SchedulerState, convert_folder_meaning};
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
-use crate::tools::{self, create_id, duration_to_str, time, time_elapsed};
+use crate::tools::{self, duration_to_str, time, time_elapsed};
 
 /// Builder for the [`Context`].
 ///
@@ -1076,154 +1071,31 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
+        res.insert(
+            "statistics_id",
+            self.get_config_bool(Config::StatisticsId)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "send_statistics",
+            self.get_config_bool(Config::SendStatistics)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "last_statistics_sent",
+            self.get_config_i64(Config::LastStatisticsSent)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = time_elapsed(&self.creation_time);
         res.insert("uptime", duration_to_str(elapsed));
 
         Ok(res)
     }
 
-    async fn get_self_report(&self) -> Result<String> {
-        #[derive(Default)]
-        struct ChatNumbers {
-            protected: u32,
-            opportunistic_dc: u32,
-            opportunistic_mua: u32,
-            unencrypted_dc: u32,
-            unencrypted_mua: u32,
-        }
-
-        let mut res = String::new();
-        res += &format!("core_version {}\n", get_version_str());
-
-        let num_msgs: u32 = self
-            .sql
-            .query_get_value(
-                "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?",
-                (DC_CHAT_ID_TRASH,),
-            )
-            .await?
-            .unwrap_or_default();
-        res += &format!("num_msgs {num_msgs}\n");
-
-        let num_chats: u32 = self
-            .sql
-            .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ())
-            .await?
-            .unwrap_or_default();
-        res += &format!("num_chats {num_chats}\n");
-
-        let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len();
-        res += &format!("db_size_bytes {db_size}\n");
-
-        let secret_key = &load_self_secret_key(self).await?.primary_key;
-        let key_created = secret_key.public_key().created_at().timestamp();
-        res += &format!("key_created {key_created}\n");
-
-        // how many of the chats active in the last months are:
-        // - protected
-        // - opportunistic-encrypted and the contact uses Delta Chat
-        // - opportunistic-encrypted and the contact uses a classical MUA
-        // - unencrypted and the contact uses Delta Chat
-        // - unencrypted and the contact uses a classical MUA
-        let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3);
-        let chats = self
-            .sql
-            .query_map(
-                "SELECT c.protected, m.param, m.msgrmsg
-                    FROM chats c
-                    JOIN msgs m
-                        ON c.id=m.chat_id
-                        AND m.id=(
-                                SELECT id
-                                FROM msgs
-                                WHERE chat_id=c.id
-                                AND hidden=0
-                                AND download_state=?
-                                AND to_id!=?
-                                ORDER BY timestamp DESC, id DESC LIMIT 1)
-                    WHERE c.id>9
-                    AND (c.blocked=0 OR c.blocked=2)
-                    AND IFNULL(m.timestamp,c.created_timestamp) > ?
-                    GROUP BY c.id",
-                (DownloadState::Done, ContactId::INFO, three_months_ago),
-                |row| {
-                    let protected: ProtectionStatus = row.get(0)?;
-                    let message_param: Params =
-                        row.get::<_, String>(1)?.parse().unwrap_or_default();
-                    let is_dc_message: bool = row.get(2)?;
-                    Ok((protected, message_param, is_dc_message))
-                },
-                |rows| {
-                    let mut chats = ChatNumbers::default();
-                    for row in rows {
-                        let (protected, message_param, is_dc_message) = row?;
-                        let encrypted = message_param
-                            .get_bool(Param::GuaranteeE2ee)
-                            .unwrap_or(false);
-
-                        if protected == ProtectionStatus::Protected {
-                            chats.protected += 1;
-                        } else if encrypted {
-                            if is_dc_message {
-                                chats.opportunistic_dc += 1;
-                            } else {
-                                chats.opportunistic_mua += 1;
-                            }
-                        } else if is_dc_message {
-                            chats.unencrypted_dc += 1;
-                        } else {
-                            chats.unencrypted_mua += 1;
-                        }
-                    }
-                    Ok(chats)
-                },
-            )
-            .await?;
-        res += &format!("chats_protected {}\n", chats.protected);
-        res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc);
-        res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua);
-        res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc);
-        res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua);
-
-        let self_reporting_id = match self.get_config(Config::SelfReportingId).await? {
-            Some(id) => id,
-            None => {
-                let id = create_id();
-                self.set_config(Config::SelfReportingId, Some(&id)).await?;
-                id
-            }
-        };
-        res += &format!("self_reporting_id {self_reporting_id}");
-
-        Ok(res)
-    }
-
-    /// Drafts a message with statistics about the usage of Delta Chat.
-    /// The user can inspect the message if they want, and then hit "Send".
-    ///
-    /// On the other end, a bot will receive the message and make it available
-    /// to Delta Chat's developers.
-    pub async fn draft_self_report(&self) -> Result<ChatId> {
-        const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf");
-        let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD)
-            .await?
-            .first()
-            .context("Self reporting bot vCard does not contain a contact")?;
-        mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?;
-
-        let chat_id = ChatId::create_for_contact(self, contact_id).await?;
-        chat_id
-            .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id))
-            .await?;
-
-        let mut msg = Message::new_text(self.get_self_report().await?);
-
-        chat_id.set_draft(self, Some(&mut msg)).await?;
-
-        Ok(chat_id)
-    }
-
     /// Get a list of fresh, unmuted messages in unblocked chats.
     ///
     /// The list starts with the most recent message

src/context.rs

@@ -6,9 +6,9 @@ use super::*;
 use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted};
 use crate::chatlist::Chatlist;
 use crate::constants::Chattype;
-use crate::mimeparser::SystemMessage;
+use crate::message::Message;
 use crate::receive_imf::receive_imf;
-use crate::test_utils::{E2EE_INFO_MSGS, TestContext, get_chat_msg};
+use crate::test_utils::{E2EE_INFO_MSGS, TestContext};
 use crate::tools::{SystemTime, create_outgoing_rfc724_mid};
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -276,7 +276,6 @@ async fn test_get_info_completeness() {
         "mail_port",
         "mail_security",
         "notify_about_wrong_pw",
-        "self_reporting_id",
         "selfstatus",
         "send_server",
         "send_user",
@@ -296,6 +295,8 @@ async fn test_get_info_completeness() {
         "webxdc_integration",
         "device_token",
         "encrypted_device_token",
+        "stats_last_excluded_msg_id",
+        "stats_last_old_contact_id",
     ];
     let t = TestContext::new().await;
     let info = t.get_info().await.unwrap();
@@ -598,26 +599,6 @@ async fn test_get_next_msgs() -> Result<()> {
     Ok(())
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_draft_self_report() -> Result<()> {
-    let alice = TestContext::new_alice().await;
-
-    let chat_id = alice.draft_self_report().await?;
-    let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
-    assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
-
-    let chat = Chat::load_from_db(&alice, chat_id).await?;
-    assert!(chat.is_protected());
-
-    let mut draft = chat_id.get_draft(&alice).await?.unwrap();
-    assert!(draft.text.starts_with("core_version"));
-
-    // Test that sending into the protected chat works:
-    let _sent = alice.send_msg(chat_id, &mut draft).await;
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_cache_is_cleared_when_io_is_started() -> Result<()> {
     let alice = TestContext::new_alice().await;

src/context/context_tests.rs

@@ -98,6 +98,7 @@ pub mod html;
 pub mod net;
 pub mod plaintext;
 mod push;
+pub mod statistics;
 pub mod summary;
 
 mod debug_logging;