diff --git a/rs/tests/driver/src/util/delegations.rs b/rs/tests/driver/src/util/delegations.rs index 8ceea9ee6b35..0433dcbdb6c9 100644 --- a/rs/tests/driver/src/util/delegations.rs +++ b/rs/tests/driver/src/util/delegations.rs @@ -177,17 +177,39 @@ struct CaptchaConfig { pub captcha_trigger: CaptchaTrigger, } +#[derive(CandidType, Serialize)] +pub struct DummyAuthConfig { + pub prompt_for_index: bool, +} + #[derive(CandidType, Serialize)] struct InternetIdentityInit { pub captcha_config: Option, + pub dummy_auth: Option>, +} + +pub fn build_dummy_internet_identity_backend_install_arg() -> Vec { + build_ii_backend_install_arg_inner(true) } pub fn build_internet_identity_backend_install_arg() -> Vec { + build_ii_backend_install_arg_inner(false) +} + +fn build_ii_backend_install_arg_inner(dummy_auth: bool) -> Vec { + let dummy_auth = if dummy_auth { + Some(Some(DummyAuthConfig { + prompt_for_index: true, + })) + } else { + None + }; candid::encode_one(&InternetIdentityInit { captcha_config: Some(CaptchaConfig { max_unsolved_captchas: 50, captcha_trigger: CaptchaTrigger::Static(StaticCaptchaTrigger::CaptchaDisabled), }), + dummy_auth, }) .unwrap() } diff --git a/rs/tests/nns/nns_dapp/nns_dapp.rs b/rs/tests/nns/nns_dapp/nns_dapp.rs index 46bf5bfee3dd..76207a4d17de 100644 --- a/rs/tests/nns/nns_dapp/nns_dapp.rs +++ b/rs/tests/nns/nns_dapp/nns_dapp.rs @@ -18,7 +18,10 @@ use ic_system_test_driver::driver::{ }; use ic_system_test_driver::nns::set_authorized_subnetwork_list; use ic_system_test_driver::sns_client::add_subnet_to_sns_deploy_whitelist; -use ic_system_test_driver::util::delegations::build_internet_identity_backend_install_arg; +use ic_system_test_driver::util::delegations::{ + DummyAuthConfig, build_dummy_internet_identity_backend_install_arg, + build_internet_identity_backend_install_arg, +}; use ic_system_test_driver::util::{block_on, create_canister, install_canister, runtime_from_url}; use icp_ledger::AccountIdentifier; use serde::{Deserialize, Serialize}; @@ -117,16 +120,39 @@ struct InternetIdentityFrontendInitArgs { related_origins: Option>, fetch_root_key: Option, dev_csp: Option, + dummy_auth: Option>, +} + +pub fn install_ii_canisters( + env: &TestEnv, + node: &IcNodeSnapshot, + ic_gateway_domain: &str, +) -> (Principal, Principal) { + install_ii_canisters_inner(env, node, ic_gateway_domain, false) +} + +pub fn install_dummy_ii_canisters( + env: &TestEnv, + node: &IcNodeSnapshot, + ic_gateway_domain: &str, +) -> (Principal, Principal) { + install_ii_canisters_inner(env, node, ic_gateway_domain, true) } -fn install_ii_canisters( +fn install_ii_canisters_inner( env: &TestEnv, node: &IcNodeSnapshot, ic_gateway_domain: &str, + dummy_auth: bool, ) -> (Principal, Principal) { + let backend_install_arg = if dummy_auth { + build_dummy_internet_identity_backend_install_arg() + } else { + build_internet_identity_backend_install_arg() + }; let backend_canister_id = node.create_and_install_canister_with_arg( &env::var("II_BACKEND_WASM_PATH").expect("II_BACKEND_WASM_PATH not set"), - Some(build_internet_identity_backend_install_arg()), + Some(backend_install_arg), ); // Create the II frontend canister first to get its canister ID @@ -134,6 +160,14 @@ fn install_ii_canisters( let frontend_canister_id = block_on(async { create_canister(&agent, node.effective_canister_id()).await }); + let dummy_auth_config = if dummy_auth { + Some(Some(DummyAuthConfig { + prompt_for_index: true, + })) + } else { + None + }; + // Install code into the II frontend canister. let frontend_wasm = load_wasm(env::var("II_FRONTEND_WASM_PATH").expect("II_FRONTEND_WASM_PATH not set")); @@ -145,6 +179,7 @@ fn install_ii_canisters( )]), fetch_root_key: Some(true), dev_csp: Some(true), + dummy_auth: dummy_auth_config, }) .unwrap(); let logger = env.logger(); @@ -176,13 +211,41 @@ pub fn install_ii_nns_dapp_and_subnet_rental( env: &TestEnv, ic_gateway_url: &Url, sns_aggregator_canister_id: Option, +) -> (Principal, Principal) { + install_ii_nns_dapp_and_subnet_rental_impl( + env, + ic_gateway_url, + sns_aggregator_canister_id, + false, + ) +} + +pub fn install_ii_nns_dapp_and_subnet_rental_with_dummy_auth( + env: &TestEnv, + ic_gateway_url: &Url, + sns_aggregator_canister_id: Option, +) -> (Principal, Principal) { + install_ii_nns_dapp_and_subnet_rental_impl( + env, + ic_gateway_url, + sns_aggregator_canister_id, + true, + ) +} + +fn install_ii_nns_dapp_and_subnet_rental_impl( + env: &TestEnv, + ic_gateway_url: &Url, + sns_aggregator_canister_id: Option, + dummy_auth: bool, ) -> (Principal, Principal) { let ic_gateway_domain = ic_gateway_url.domain().unwrap().to_string(); // deploy the II canister let topology = env.topology_snapshot(); let nns_node = topology.root_subnet().nodes().next().unwrap(); - let (_, ii_canister_id) = install_ii_canisters(env, &nns_node, &ic_gateway_domain); + let (_, ii_canister_id) = + install_ii_canisters_inner(env, &nns_node, &ic_gateway_domain, dummy_auth); // create the NNS dapp canister so that its canister ID is allocated // and the Subnet Rental Canister gets its mainnet canister ID in the next step diff --git a/rs/tests/testnets/cloud_engine.rs b/rs/tests/testnets/cloud_engine.rs index 9fe0317176c6..ef1e7028bd7f 100644 --- a/rs/tests/testnets/cloud_engine.rs +++ b/rs/tests/testnets/cloud_engine.rs @@ -1,8 +1,13 @@ // Set up a testnet containing: -// one 4-node System/NNS subnet, one 4-node CloudEngine subnet (1 node per DC in 4 datacenters), -// 4 unassigned nodes (1 per DC), one API boundary node, one ic-gateway, and a p8s (with grafana) VM. +// one 1-node System/NNS subnet, by default 20 unassigned nodes distributed +// round-robin across 30 datacenters (so each of the first 20 DCs gets 1 node), +// one API boundary node, one ic-gateway, and a p8s (with grafana) VM. // All replica nodes use the following resources: 6 vCPUs, 24GiB of RAM, and 50 GiB disk. // +// The number of unassigned nodes can be overridden via the NUM_UNASSIGNED_NODES +// env var (e.g. NUM_UNASSIGNED_NODES=60 will spin up 60 unassigned nodes, +// distributed round-robin across the 30 DCs, yielding 2 nodes per DC). +// // You can setup this testnet by executing the following commands: // // $ ./ci/tools/docker-run @@ -38,7 +43,10 @@ use anyhow::Result; use ic_consensus_system_test_utils::rw_message::install_nns_with_customizations_and_check_progress; -use ic_protobuf::registry::dc::v1::{DataCenterRecord, Gps}; +use ic_protobuf::registry::{ + dc::v1::{DataCenterRecord, Gps}, + node::v1::NodeRewardType, +}; use ic_registry_subnet_type::SubnetType; use ic_system_test_driver::driver::{ farm::HostFeature, @@ -48,9 +56,10 @@ use ic_system_test_driver::driver::{ test_env::TestEnv, test_env_api::HasTopologySnapshot, }; -use ic_types::PrincipalId; +use ic_types::{Height, PrincipalId}; use nns_dapp::{ - install_ii_nns_dapp_and_subnet_rental, nns_dapp_customizations, set_authorized_subnets, + install_ii_nns_dapp_and_subnet_rental_with_dummy_auth, nns_dapp_customizations, + set_authorized_subnets, }; use std::collections::BTreeMap; use std::net::Ipv4Addr; @@ -61,12 +70,43 @@ const DM1_DMZ_NETWORK: Ipv4Addr = Ipv4Addr::new(23, 142, 184, 224); const DM1_DMZ_PREFIX: u8 = 28; const DM1_DMZ_GATEWAY: Ipv4Addr = Ipv4Addr::new(23, 142, 184, 238); +/// Node providers used in this testnet. Each data center is owned by exactly +/// one node provider (1 node provider per DC). Providers can own multiple DCs +/// and do not need to own the same number of DCs / nodes. +#[derive(Clone, Copy, Debug)] +enum NodeProvider { + // Required because of DFINITY-capitalization-check pre-commit + #[allow(clippy::upper_case_acronyms)] + DFINITY, + Alusion, + OneSixtyTwoDigitalCapital, + DecentralizedEntitiesFoundation, +} + +impl NodeProvider { + /// Stable test principal id for each provider. These are deterministic and + /// only intended for the testnet setup. + fn principal_id(&self) -> PrincipalId { + // Use a separate id range (3000+) from node operators (1000+) so that + // the principals don't overlap. + match self { + NodeProvider::DFINITY => PrincipalId::new_user_test_id(3000), + NodeProvider::Alusion => PrincipalId::new_user_test_id(3001), + NodeProvider::OneSixtyTwoDigitalCapital => PrincipalId::new_user_test_id(3002), + NodeProvider::DecentralizedEntitiesFoundation => PrincipalId::new_user_test_id(3003), + } + } +} + struct DcConfig { id: &'static str, region: &'static str, owner: &'static str, latitude: f32, longitude: f32, + /// The node provider that owns this data center. Every DC has exactly one + /// node provider, but one node provider may own many DCs. + node_provider: NodeProvider, } const DATA_CENTERS: &[DcConfig] = &[ @@ -76,6 +116,7 @@ const DATA_CENTERS: &[DcConfig] = &[ owner: "Hurricane Electric", latitude: 37.549, longitude: -121.989, + node_provider: NodeProvider::DFINITY, }, DcConfig { id: "Brussels", @@ -83,6 +124,7 @@ const DATA_CENTERS: &[DcConfig] = &[ owner: "Digital Realty", latitude: 50.839, longitude: 4.348, + node_provider: NodeProvider::DFINITY, }, DcConfig { id: "HongKong 1", @@ -90,6 +132,7 @@ const DATA_CENTERS: &[DcConfig] = &[ owner: "Unicom", latitude: 22.284, longitude: 114.269, + node_provider: NodeProvider::DFINITY, }, DcConfig { id: "Sterling", @@ -97,6 +140,215 @@ const DATA_CENTERS: &[DcConfig] = &[ owner: "CyrusOne", latitude: 39.004, longitude: -77.408, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "Tokyo", + region: "Asia,JP,Tokyo", + owner: "Equinix", + latitude: 35.682, + longitude: 139.692, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "London", + region: "Europe,GB,London", + owner: "Telehouse", + latitude: 51.508, + longitude: -0.076, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "Frankfurt", + region: "Europe,DE,Hessen", + owner: "Interxion", + latitude: 50.110, + longitude: 8.682, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "Singapore", + region: "Asia,SG,Singapore", + owner: "Equinix", + latitude: 1.290, + longitude: 103.851, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "Sao Paulo", + region: "South America,BR,Sao Paulo", + owner: "Ascenty", + latitude: -23.550, + longitude: -46.633, + node_provider: NodeProvider::DFINITY, + }, + DcConfig { + id: "Sydney", + region: "Oceania,AU,New South Wales", + owner: "Equinix", + latitude: -33.868, + longitude: 151.207, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Toronto", + region: "North America,CA,Ontario", + owner: "eStruxture", + latitude: 43.651, + longitude: -79.347, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Mumbai", + region: "Asia,IN,Maharashtra", + owner: "Nxtra", + latitude: 19.076, + longitude: 72.878, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Seoul", + region: "Asia,KR,Seoul", + owner: "KINX", + latitude: 37.566, + longitude: 126.978, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Amsterdam", + region: "Europe,NL,North Holland", + owner: "Equinix", + latitude: 52.370, + longitude: 4.895, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Paris", + region: "Europe,FR,Ile-de-France", + owner: "Interxion", + latitude: 48.864, + longitude: 2.349, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Stockholm", + region: "Europe,SE,Stockholm", + owner: "Interxion", + latitude: 59.330, + longitude: 18.069, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Zurich", + region: "Europe,CH,Zurich", + owner: "Green", + latitude: 47.376, + longitude: 8.540, + node_provider: NodeProvider::Alusion, + }, + DcConfig { + id: "Dublin", + region: "Europe,IE,Dublin", + owner: "Equinix", + latitude: 53.350, + longitude: -6.260, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Chicago", + region: "North America,US,Illinois", + owner: "Equinix", + latitude: 41.878, + longitude: -87.630, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Dallas", + region: "North America,US,Texas", + owner: "DataBank", + latitude: 32.777, + longitude: -96.797, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Los Angeles", + region: "North America,US,California", + owner: "CoreSite", + latitude: 34.052, + longitude: -118.244, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Miami", + region: "North America,US,Florida", + owner: "Equinix", + latitude: 25.762, + longitude: -80.192, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Bogota", + region: "South America,CO,Bogota", + owner: "Equinix", + latitude: 4.711, + longitude: -74.072, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Cape Town", + region: "Africa,ZA,Western Cape", + owner: "Teraco", + latitude: -33.925, + longitude: 18.424, + node_provider: NodeProvider::OneSixtyTwoDigitalCapital, + }, + DcConfig { + id: "Nairobi", + region: "Africa,KE,Nairobi", + owner: "PAIX", + latitude: -1.286, + longitude: 36.817, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, + }, + DcConfig { + id: "Warsaw", + region: "Europe,PL,Masovia", + owner: "Equinix", + latitude: 52.230, + longitude: 21.012, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, + }, + DcConfig { + id: "Madrid", + region: "Europe,ES,Madrid", + owner: "Interxion", + latitude: 40.417, + longitude: -3.704, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, + }, + DcConfig { + id: "Milan", + region: "Europe,IT,Lombardy", + owner: "Equinix", + latitude: 45.464, + longitude: 9.190, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, + }, + DcConfig { + id: "Osaka", + region: "Asia,JP,Osaka", + owner: "Equinix", + latitude: 34.694, + longitude: 135.502, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, + }, + DcConfig { + id: "Jakarta", + region: "Asia,ID,Jakarta", + owner: "DCI", + latitude: -6.175, + longitude: 106.845, + node_provider: NodeProvider::DecentralizedEntitiesFoundation, }, ]; @@ -112,14 +364,65 @@ pub fn setup(env: TestEnv) { let mut ic = InternetComputer::new() .with_required_host_features(dm1_dmz_features.clone()) - .add_subnet(Subnet::new(SubnetType::System).add_nodes(1)); + .add_subnet( + Subnet::new(SubnetType::System) + .add_nodes(1) + // To speed up subnet creation + .with_dkg_interval_length(Height::from(10)), + ); + + // Build unassigned nodes distributed across the 30 datacenters. + // Each datacenter gets its own node operator. The total number of unassigned + // nodes defaults to 20 and can be overridden via the NUM_UNASSIGNED_NODES + // env var. Nodes are distributed round-robin across DATA_CENTERS in order: + // node index i is placed in DC (i % NUM_DCS). This means with the default + // of 20 nodes the first 20 DCs each get 1 node, and with 60 nodes each DC + // gets 2 nodes. + // + // Reward types are assigned in a circular rotation across all nodes globally: + // node index 0 -> type4.1, 1 -> type4.2, 2 -> type4.3, 3 -> type4.1, ... + // With a total that is a multiple of 3 this yields equal counts of each + // reward type. + const CLOUD_ENGINE_REWARD_TYPES: &[(NodeRewardType, &str)] = &[ + (NodeRewardType::Type4dot1, "type4.1"), + (NodeRewardType::Type4dot2, "type4.2"), + (NodeRewardType::Type4dot3, "type4.3"), + ]; + const DEFAULT_NUM_UNASSIGNED_NODES: usize = 20; - // Build CloudEngine subnet and unassigned nodes distributed across 4 datacenters. - // Each datacenter gets its own node operator with 1 CloudEngine node + 1 unassigned node. - let mut cloud_engine_subnet = Subnet::new(SubnetType::CloudEngine); + let num_unassigned_nodes: usize = match std::env::var("NUM_UNASSIGNED_NODES") { + Ok(v) => v + .parse() + .unwrap_or_else(|e| panic!("invalid NUM_UNASSIGNED_NODES value '{v}': {e}")), + Err(_) => DEFAULT_NUM_UNASSIGNED_NODES, + }; + + // Compute, for each DC, the list of (node_index, reward_type) pairs that + // belong to that DC after the round-robin distribution. The global node + // index is preserved so that the reward-type rotation is identical to a + // simple sequential walk over all nodes. + let mut nodes_per_dc: Vec> = + vec![Vec::new(); DATA_CENTERS.len()]; + for node_idx in 0..num_unassigned_nodes { + let dc_idx = node_idx % DATA_CENTERS.len(); + let reward = CLOUD_ENGINE_REWARD_TYPES[node_idx % CLOUD_ENGINE_REWARD_TYPES.len()]; + nodes_per_dc[dc_idx].push((node_idx, reward)); + } + + // let mut cloud_engine_subnet = Subnet::new(SubnetType::CloudEngine); for (i, dc) in DATA_CENTERS.iter().enumerate() { + // Each DC has its own node operator (1 node operator per DC), but the + // node provider is shared across all DCs owned by that provider. let operator_principal = PrincipalId::new_user_test_id(1000 + i as u64); - let provider_principal = PrincipalId::new_user_test_id(2000 + i as u64); + let provider_principal = dc.node_provider.principal_id(); + + let dc_nodes = &nodes_per_dc[i]; + + // Aggregate rewardable_nodes counts per type string for this operator. + let mut rewardable_nodes: BTreeMap = BTreeMap::new(); + for (_, (_, type_str)) in dc_nodes { + *rewardable_nodes.entry(type_str.to_string()).or_insert(0) += 1; + } ic = ic .add_data_center(DataCenterRecord { @@ -135,22 +438,23 @@ pub fn setup(env: TestEnv) { name: format!("operator_{}", dc.id), principal_id: operator_principal, node_provider_principal_id: Some(provider_principal), - node_allowance: 2, + node_allowance: dc_nodes.len() as u64, dc_id: dc.id.to_string(), - rewardable_nodes: BTreeMap::from([("type4.1".to_string(), 2)]), + rewardable_nodes, }); - // 1 CloudEngine node per DC - cloud_engine_subnet = cloud_engine_subnet - .add_node(Node::new().with_node_operator_principal_id(operator_principal)); - - // 1 unassigned node per DC - ic = ic - .with_unassigned_node(Node::new().with_node_operator_principal_id(operator_principal)); + // Add unassigned nodes for this DC using the circularly-assigned types. + for (_, (reward_type, _)) in dc_nodes { + ic = ic.with_unassigned_node( + Node::new() + .with_node_operator_principal_id(operator_principal) + .with_node_reward_type(*reward_type), + ); + } } ic = ic - .add_subnet(cloud_engine_subnet) + //.add_subnet(cloud_engine_subnet) .with_api_boundary_nodes_playnet(1); ic.setup_and_start(&env) @@ -185,5 +489,5 @@ pub fn setup(env: TestEnv) { set_authorized_subnets(&env); // install II, NNS dapp, and Subnet Rental Canister - install_ii_nns_dapp_and_subnet_rental(&env, &ic_gateway_url, None); + install_ii_nns_dapp_and_subnet_rental_with_dummy_auth(&env, &ic_gateway_url, None); }