diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 5a50b1a..9bc4c1c 100644 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -6,7 +6,7 @@ env: CARGO_TERM_COLOR: always CARGO_TERM_VERBOSE: true CARGOFLAGS: --workspace --all-targets --all-features - RUST_LOG: citrea-e2e=trace,debug + RUST_LOG: citrea-e2e=trace,info RISC0_DEV_MODE: 1 jobs: @@ -57,8 +57,8 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install protoc - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v3 - name: Set up Rust uses: actions-rs/toolchain@v1 @@ -72,12 +72,17 @@ jobs: - name: Download Clementine run: | - wget https://github.com/chainwayxyz/clementine/releases/download/v0.3.26/clementine-core-v0.3.26-with-automation-linux-amd64 -O /tmp/clementine-core + wget https://github.com/chainwayxyz/clementine/releases/download/v0.4.1/clementine-core-v0.4.1-with-automation-linux-amd64 -O /tmp/clementine-core chmod +x /tmp/clementine-core + - name: Copy Docker binary + run: | + mkdir -p resources/docker + cp /usr/bin/docker ./resources/docker/docker-linux-amd64 + - name: Run Cargo test env: CLEMENTINE_E2E_TEST_BINARY: /tmp/clementine-core BITVM_CACHE_PATH: /tmp/bitvm_cache_dev.bin RISC0_DEV_MODE: 1 - run: cargo test + run: cargo test --all-features diff --git a/.gitignore b/.gitignore index a20ab64..b43935d 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ Cargo.lock # Autogenerated/downloaded files /resources/clementine/certs/ /resources/clementine/bitvm_cache.bin +/resources/docker diff --git a/Cargo.toml b/Cargo.toml index c2d1c8b..cb1a532 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ async-trait = { version = "0.1.71" } bitcoin = { version = "0.32.2", default-features = false, features = ["serde", "rand"] } bitcoincore-rpc = { version = "0.18.0", default-features = false } bollard = { version = "0.19.1" } +fs2 = { version = "0.4", optional = true } futures = { version = "0.3", default-features = false } hex = { version = "0.4.3", default-features = false, features = ["serde"] } jsonrpsee = { version = "0.24.2", default-features = false, features = ["http-client"] } @@ -32,7 +33,7 @@ tonic-build = { version = "0.12", optional = true } [features] default = [] -clementine = ["prost", "tonic", "prost-build", "tonic-build"] +clementine = ["prost", "tonic", "prost-build", "tonic-build", "fs2"] [patch.crates-io] bitcoincore-rpc = { version = "0.18.0", git = "https://github.com/chainwayxyz/rust-bitcoincore-rpc.git", rev = "5ce1bed" } diff --git a/proto/clementine.proto b/proto/clementine.proto index fd05ec2..bc4aa7a 100644 --- a/proto/clementine.proto +++ b/proto/clementine.proto @@ -12,10 +12,6 @@ message Outpoint { uint32 vout = 2; } -message XonlyPublicKey { - bytes xonly_pk = 1; -} - message NofnResponse { bytes nofn_xonly_pk = 1; uint32 num_verifiers = 2; @@ -278,14 +274,14 @@ message EntityError { message EntityStatus { bool automation = 1; - string wallet_balance = 2; - uint32 tx_sender_synced_height = 3; - uint32 finalized_synced_height = 4; - uint32 hcp_last_proven_height = 5; + optional string wallet_balance = 2; + optional uint32 tx_sender_synced_height = 3; + optional uint32 finalized_synced_height = 4; + optional uint32 hcp_last_proven_height = 5; StoppedTasks stopped_tasks = 6; - uint32 rpc_tip_height = 7; - uint32 bitcoin_syncer_synced_height = 8; - uint32 state_manager_next_height = 9; + optional uint32 rpc_tip_height = 7; + optional uint32 bitcoin_syncer_synced_height = 8; + optional uint32 state_manager_next_height = 9; } enum EntityType { @@ -356,11 +352,31 @@ service ClementineOperator { // Restarts the background tasks for the operator. rpc RestartBackgroundTasks(Empty) returns (Empty) {} - // Prepares a withdrawal if it's profitable and the withdrawal is correct and registered in Citrea bridge contract/ + // Prepares a withdrawal if it's profitable and the withdrawal is correct and registered in Citrea bridge contract. // If withdrawal is accepted, the payout tx will be added to the TxSender and success is returned, otherwise an error is returned. // If automation is disabled, the withdrawal will not be accepted and an error will be returned. - rpc Withdraw(WithdrawParams) - returns (Empty) {} + // Note: This is intended for operator's own use, so it doesn't include a signature from aggregator. + rpc InternalWithdraw(WithdrawParams) + returns (RawSignedTx) {} + + + // First, if verification address in operator's config is set, the signature in rpc is checked to see if it was signed by the verification address. + // Then prepares a withdrawal if it's profitable and the withdrawal is correct and registered in Citrea bridge contract. + // If withdrawal is accepted, the payout tx will be added to the TxSender and success is returned, otherwise an error is returned. + // If automation is disabled, the withdrawal will not be accepted and an error will be returned. + rpc Withdraw(WithdrawParamsWithSig) + returns (RawSignedTx) {} + + // For a given deposit outpoint, determines the next step in the kickoff process the operator is in, + // and returns the raw signed txs that the operator needs to send next, for enabling reimbursement process + // without automation. + // + // # Parameters + // - deposit_outpoint: Deposit outpoint to create the kickoff for + // + // # Returns + // - Raw signed txs that the operator needs to send next + rpc GetReimbursementTxs(Outpoint) returns (SignedTxsWithType) {} // Signs all tx's it can according to given transaction type (use it with AllNeededForDeposit to get almost all tx's) // Creates the transactions denoted by the deposit and operator_idx, round_idx, and kickoff_idx. @@ -411,7 +427,8 @@ message NonceGenRequest { message NonceGenFirstResponse { // ID of the nonce session (used to store nonces in verifier's memory) - uint32 id = 1; + // The id is string representation of a u128 number + string id = 1; // Number of nonces to generate uint32 num_nonces = 2; } @@ -422,8 +439,34 @@ message NonceGenResponse { } } -message OptimisticPayoutParams { +message OptimisticWithdrawParams { WithdrawParams withdrawal = 1; + // An ECDSA signature (of citrea/aggregator) over the withdrawal params + // to authenticate the withdrawal params. This will be signed manually by citrea + // after manual verification of the optimistic payout. + optional string verification_signature = 2; +} + +message WithdrawParamsWithSig { + WithdrawParams withdrawal = 1; + // An ECDSA signature (of citrea/aggregator) over the withdrawal params + // to authenticate the withdrawal params. This will be signed manually by citrea + // after manual verification of the optimistic payout. + // This message contains same data as the one in Optimistic Payout signature, but with a different message name, + // so that the same signature can't be used for both optimistic payout and normal withdrawal. + optional string verification_signature = 2; +} + +// Input of the aggregator's withdraw function. +// It contains the withdrawal params along with the verification signature that signs the withdrawal params. +// It also contains the operator's xonly public keys that the withdrawal request should be sent to. If the list is empty, the withdrawal will be sent to all operators. +message AggregatorWithdrawalInput { + WithdrawParamsWithSig withdrawal = 1; + repeated XOnlyPublicKeyRpc operator_xonly_pks = 2; +} + +message OptimisticPayoutParams { + OptimisticWithdrawParams opt_withdrawal = 1; NonceGenFirstResponse nonce_gen = 2; bytes agg_nonce = 3; } @@ -473,7 +516,7 @@ message TxMetadata { // Optional outpoint of the deposit transaction Outpoint deposit_outpoint = 1; // Deposit identification - XonlyPublicKey operator_xonly_pk = 2; + XOnlyPublicKeyRpc operator_xonly_pk = 2; uint32 round_idx = 4; uint32 kickoff_idx = 5; // Transaction ID @@ -616,9 +659,13 @@ message AggregatorWithdrawResponse { repeated string withdraw_responses = 1; } -message CreateEmergencyStopTxRequest { +message GetEmergencyStopTxRequest { + repeated Txid txids = 1; +} + +message GetEmergencyStopTxResponse { repeated Txid txids = 1; - bool add_anchor = 2; + repeated bytes encrypted_emergency_stop_txs = 2; } message SendMoveTxRequest { @@ -655,11 +702,13 @@ service ClementineAggregator { // Call's withdraw on all operators // Used by the clementine-backend service to initiate a withdrawal - rpc Withdraw(WithdrawParams) + // If the operator's xonly public keys list is empty, the withdrawal will be sent to all operators. + // If not, only the operators in the list will be sent the withdrawal request. + rpc Withdraw(AggregatorWithdrawalInput) returns (AggregatorWithdrawResponse) {} // Perform an optimistic payout to reimburse a peg-out from Citrea - rpc OptimisticPayout(WithdrawParams) returns (RawSignedTx) {} + rpc OptimisticPayout(OptimisticWithdrawParams) returns (RawSignedTx) {} // Send a pre-signed tx to the network rpc InternalSendTx(SendTxRequest) returns (Empty) {} @@ -673,7 +722,7 @@ service ClementineAggregator { // Creates an emergency stop tx that won't be broadcasted. // Tx will have around 3 sats/vbyte fee. // Set add_anchor to true to add an anchor output for cpfp.. - rpc InternalCreateEmergencyStopTx(CreateEmergencyStopTxRequest) returns (SignedTxWithType) {} + rpc InternalGetEmergencyStopTx(GetEmergencyStopTxRequest) returns (GetEmergencyStopTxResponse) {} rpc Vergen(Empty) returns (VergenResponse) {} -} \ No newline at end of file +} diff --git a/resources/clementine/generate-certs.sh b/resources/clementine/generate-certs.sh index 8fb1d6a..3c5ee49 100644 --- a/resources/clementine/generate-certs.sh +++ b/resources/clementine/generate-certs.sh @@ -15,7 +15,7 @@ mkdir -p $CA_DIR $SERVER_DIR $CLIENT_DIR $AGGREGATOR_DIR echo "Generating certificates for mutual TLS..." # Create openssl config file for x509v3 extensions -cat > openssl_x509.cnf << EOF +cat > $CERT_DIR/openssl_x509.cnf << EOF [req] distinguished_name = req_distinguished_name req_extensions = v3_req @@ -37,6 +37,7 @@ basicConstraints = CA:true [alt_names] DNS.1 = localhost DNS.2 = host.docker.internal +DNS.3 = *.e2e.internal IP.1 = 127.0.0.1 EOF @@ -45,46 +46,46 @@ echo "Generating CA certificate..." openssl genrsa -out $CA_DIR/ca.key 4096 openssl req -new -x509 -sha256 -days 365 -key $CA_DIR/ca.key -out $CA_DIR/ca.pem \ -subj "/C=US/ST=California/L=San Francisco/O=Clementine/OU=CA/CN=clementine-ca" \ - -extensions v3_ca -config openssl_x509.cnf + -extensions v3_ca -config $CERT_DIR/openssl_x509.cnf # Generate server key and CSR echo "Generating server certificate..." openssl genrsa -out $SERVER_DIR/server.key 2048 openssl req -new -key $SERVER_DIR/server.key -out $SERVER_DIR/server.csr \ -subj "/C=US/ST=California/L=San Francisco/O=Clementine/OU=Server/CN=localhost" \ - -config openssl_x509.cnf + -config $CERT_DIR/openssl_x509.cnf # Sign server certificate with CA openssl x509 -req -sha256 -days 365 -in $SERVER_DIR/server.csr \ -CA $CA_DIR/ca.pem -CAkey $CA_DIR/ca.key -CAcreateserial \ -out $SERVER_DIR/server.pem \ - -extfile openssl_x509.cnf -extensions v3_req + -extfile $CERT_DIR/openssl_x509.cnf -extensions v3_req # Generate client key and CSR echo "Generating client certificate..." openssl genrsa -out $CLIENT_DIR/client.key 2048 openssl req -new -key $CLIENT_DIR/client.key -out $CLIENT_DIR/client.csr \ -subj "/C=US/ST=California/L=San Francisco/O=Clementine/OU=Client/CN=clementine-client" \ - -config openssl_x509.cnf + -config $CERT_DIR/openssl_x509.cnf # Sign client certificate with CA openssl x509 -req -sha256 -days 365 -in $CLIENT_DIR/client.csr \ -CA $CA_DIR/ca.pem -CAkey $CA_DIR/ca.key -CAcreateserial \ -out $CLIENT_DIR/client.pem \ - -extfile openssl_x509.cnf -extensions v3_req + -extfile $CERT_DIR/openssl_x509.cnf -extensions v3_req # Generate aggregator key and CSR echo "Generating aggregator certificate..." openssl genrsa -out $AGGREGATOR_DIR/aggregator.key 2048 openssl req -new -key $AGGREGATOR_DIR/aggregator.key -out $AGGREGATOR_DIR/aggregator.csr \ -subj "/C=US/ST=California/L=San Francisco/O=Clementine/OU=Aggregator/CN=clementine-aggregator" \ - -config openssl_x509.cnf + -config $CERT_DIR/openssl_x509.cnf # Sign client certificate with CA openssl x509 -req -sha256 -days 365 -in $AGGREGATOR_DIR/aggregator.csr \ -CA $CA_DIR/ca.pem -CAkey $CA_DIR/ca.key -CAcreateserial \ -out $AGGREGATOR_DIR/aggregator.pem \ - -extfile openssl_x509.cnf -extensions v3_req + -extfile $CERT_DIR/openssl_x509.cnf -extensions v3_req # Copy CA certificate to both directories for convenience cp $CA_DIR/ca.pem $SERVER_DIR/ @@ -92,7 +93,7 @@ cp $CA_DIR/ca.pem $CLIENT_DIR/ cp $CA_DIR/ca.pem $AGGREGATOR_DIR/ # Clean up temporary files -rm -f openssl_x509.cnf +rm -f $CERT_DIR/openssl_x509.cnf rm -f $SERVER_DIR/server.csr rm -f $CLIENT_DIR/client.csr rm -f $AGGREGATOR_DIR/aggregator.csr diff --git a/resources/clementine/regtest_paramset.toml b/resources/clementine/regtest_paramset.toml index 2e7396a..1532fdc 100644 --- a/resources/clementine/regtest_paramset.toml +++ b/resources/clementine/regtest_paramset.toml @@ -19,7 +19,7 @@ watchtower_challenge_timeout_timelock = 288 # BLOCKS_PER_DAY * 2 time_to_send_watchtower_challenge = 216 # BLOCKS_PER_DAY * 3 / 2 latest_blockhash_timeout_timelock = 360 # BLOCKS_PER_DAY * 5 / 2 finality_depth = 1 -start_height = 0 +start_height = 2 genesis_height = 0 genesis_chain_state_hash = [ 95, diff --git a/src/clementine/client.rs b/src/clementine/client.rs index f5e491a..95df68e 100644 --- a/src/clementine/client.rs +++ b/src/clementine/client.rs @@ -82,6 +82,10 @@ impl ClementineAggregatorTestClient { Ok(Self { client }) } + pub fn inner_mut(&mut self) -> &mut ClementineAggregatorClient { + &mut self.client + } + pub async fn new_deposit(&mut self, deposit: Deposit) -> Result { let request = Request::new(deposit); let response = self @@ -92,7 +96,10 @@ impl ClementineAggregatorTestClient { Ok(response.into_inner()) } - pub async fn withdraw(&mut self, params: WithdrawParams) -> Result { + pub async fn withdraw( + &mut self, + params: AggregatorWithdrawalInput, + ) -> Result { let request = Request::new(params); let response = self .client @@ -102,7 +109,10 @@ impl ClementineAggregatorTestClient { Ok(response.into_inner()) } - pub async fn optimistic_payout(&mut self, params: WithdrawParams) -> Result { + pub async fn optimistic_payout( + &mut self, + params: OptimisticWithdrawParams, + ) -> Result { let request = Request::new(params); let response = self .client @@ -155,6 +165,10 @@ impl ClementineOperatorTestClient { Ok(Self { client }) } + pub fn inner_mut(&mut self) -> &mut ClementineOperatorClient { + &mut self.client + } + pub async fn get_x_only_public_key(&mut self) -> Result { let request = Request::new(Empty {}); let response = self @@ -185,7 +199,7 @@ impl ClementineOperatorTestClient { Ok(response.into_inner()) } - pub async fn withdraw(&mut self, params: WithdrawParams) -> Result<()> { + pub async fn withdraw(&mut self, params: WithdrawParamsWithSig) -> Result<()> { let request = Request::new(params); self.client .withdraw(request) @@ -216,6 +230,10 @@ impl ClementineVerifierTestClient { Ok(Self { client }) } + pub fn inner_mut(&mut self) -> &mut ClementineVerifierClient { + &mut self.client + } + pub async fn get_params(&mut self) -> Result { let request = Request::new(Empty {}); let response = self diff --git a/src/clementine/mod.rs b/src/clementine/mod.rs index cfa7f98..dd6145a 100644 --- a/src/clementine/mod.rs +++ b/src/clementine/mod.rs @@ -67,6 +67,7 @@ impl ClementineIntegration { bitcoin_config: crate::config::BitcoinConfig, full_node_rpc: crate::config::RpcConfig, light_client_rpc: crate::config::RpcConfig, + docker: &Option, ) -> Result { use anyhow::Context; @@ -75,6 +76,7 @@ impl ClementineIntegration { AggregatorConfig, ClementineClusterConfig, ClementineConfig, OperatorConfig, VerifierConfig, }, + node::NodeKind, utils::get_available_port, }; @@ -96,11 +98,20 @@ impl ClementineIntegration { bitcoin_config.clone(), full_node_rpc.clone(), light_client_rpc.clone(), + docker, clementine_dir.to_path_buf(), port, T::clementine_verifier_config(i), )); - verifier_endpoints.push(format!("https://127.0.0.1:{}", port)); + // When running in Docker, use host.docker.internal so containers can reach host-published ports + let host = docker + .as_ref() + .and_then(|d| { + d.clementine() + .then(|| d.get_hostname(&NodeKind::ClementineVerifier(i))) + }) + .unwrap_or("127.0.0.1".to_string()); + verifier_endpoints.push(format!("https://{}:{}", host, port)); } let mut operators = vec![]; @@ -113,11 +124,19 @@ impl ClementineIntegration { bitcoin_config.clone(), full_node_rpc.clone(), light_client_rpc.clone(), + docker, clementine_dir.to_path_buf(), port, T::clementine_operator_config(i), )); - operator_endpoints.push(format!("https://127.0.0.1:{}", port)); + let host = docker + .as_ref() + .and_then(|d| { + d.clementine() + .then(|| d.get_hostname(&NodeKind::ClementineOperator(i))) + }) + .unwrap_or("127.0.0.1".to_string()); + operator_endpoints.push(format!("https://{}:{}", host, port)); } let port = get_available_port()?; @@ -128,6 +147,7 @@ impl ClementineIntegration { bitcoin_config.clone(), full_node_rpc.clone(), light_client_rpc.clone(), + docker, clementine_dir.to_path_buf(), port, T::clementine_aggregator_config(), diff --git a/src/clementine/nodes.rs b/src/clementine/nodes.rs index dc26c24..deb3e0c 100644 --- a/src/clementine/nodes.rs +++ b/src/clementine/nodes.rs @@ -1,10 +1,10 @@ use std::{ collections::HashMap, - fmt::Debug, fs::File, + path::PathBuf, process::Stdio, - sync::{Arc, LazyLock}, - time::Duration, + sync::Arc, + time::{Duration, Instant}, }; use anyhow::{anyhow, Context}; @@ -20,7 +20,8 @@ use tracing::{debug, error, info, warn}; use super::client; use crate::{ config::{ - AggregatorConfig, ClementineClusterConfig, ClementineConfig, OperatorConfig, VerifierConfig, + AggregatorConfig, ClementineClusterConfig, ClementineConfig, ClementineEntityConfig, + DockerConfig, OperatorConfig, VerifierConfig, VolumeConfig, }, docker::DockerEnv, log_provider::LogPathProvider, @@ -30,6 +31,9 @@ use crate::{ }; pub const CLEMENTINE_NODE_STARTUP_TIMEOUT: Duration = Duration::from_secs(360); +const DEFAULT_CLEMENTINE_DOCKER_IMAGE: &str = "chainwayxyz/clementine-automation:latest"; +const DEFAULT_DOCKER_CLIENT_URL: &str = + "https://download.docker.com/linux/static/stable/x86_64/docker-27.0.3.tgz"; pub struct ClementineAggregator { pub config: ClementineConfig, @@ -61,11 +65,27 @@ impl ClementineAggregator { "localhost".to_string(), ); - // Create gRPC client - let endpoint = format!("https://{}:{}", config.host, config.port); - let client = ClementineAggregatorTestClient::new(endpoint, tls_config) - .await - .context("Failed to create Clementine aggregator client")?; + let endpoint = format!("https://127.0.0.1:{}", config.port); + + let timeout = CLEMENTINE_NODE_STARTUP_TIMEOUT; + let start = Instant::now(); + let mut result = Err(anyhow!("initial response value")); + + while result.is_err() && (start.elapsed() < timeout) { + debug!( + "Aggregator connect attempt after {} seconds", + start.elapsed().as_secs() + ); + result = + ClementineAggregatorTestClient::new(endpoint.clone(), tls_config.clone()).await; + + tokio::time::sleep(Duration::from_millis(500)).await; + } + + let client = result.context(format!( + "Failed to connect to Clementine aggregator in {} seconds", + start.elapsed().as_secs() + ))?; let instance = Self { config: config.clone(), @@ -83,25 +103,8 @@ impl NodeT for ClementineAggregator { type Config = ClementineConfig; type Client = ClementineAggregatorTestClient; - async fn spawn(config: &Self::Config, _docker: &Arc>) -> Result { - let mut env_vars = get_common_env_vars(config); - - // Aggregator specific configuration - let verifier_endpoints = config.entity_config.verifier_endpoints.join(","); - let operator_endpoints = config.entity_config.operator_endpoints.join(","); - env_vars.insert("VERIFIER_ENDPOINTS".to_string(), verifier_endpoints); - env_vars.insert("OPERATOR_ENDPOINTS".to_string(), operator_endpoints); - env_vars.insert( - "SECRET_KEY".to_string(), - "3333333333333333333333333333333333333333333333333333333333333333".to_string(), - ); - - // Aggregator uses port 8082 for telemetry - if config.telemetry.is_some() { - env_vars.insert("TELEMETRY_PORT".to_string(), "8082".to_string()); - } - - spawn_clementine_node(config, "aggregator", env_vars).await + async fn spawn(config: &Self::Config, docker: &Arc>) -> Result { + spawn_clementine_node(config, "aggregator", docker).await } fn spawn_output(&mut self) -> &mut SpawnOutput { @@ -137,16 +140,10 @@ pub struct ClementineVerifier { impl ClementineVerifier { pub async fn new( config: &ClementineConfig, - _docker: Arc>, + docker: Arc>, index: u8, ) -> Result { - let mut env_vars = get_common_env_vars(config); - env_vars.insert( - "SECRET_KEY".to_string(), - hex::encode(config.entity_config.secret_key.secret_bytes()), - ); - - let spawn_output = spawn_clementine_node(config, "verifier", env_vars).await?; + let spawn_output = ::spawn(config, &docker).await?; // Wait for the gRPC server to be ready wait_for_tcp_bound( @@ -160,19 +157,35 @@ impl ClementineVerifier { index ))?; - // Create TLS configuration + // Create TLS configuration using Aggregator client cert for access + let aggregator_key_path = config.aggregator_cert_path.with_file_name("aggregator.key"); let tls_config = TlsConfig::new( - &config.client_cert_path, - &config.client_key_path, + &config.aggregator_cert_path, + &aggregator_key_path, &config.ca_cert_path, "localhost".to_string(), ); + let endpoint = format!("https://127.0.0.1:{}", config.port); - // Create gRPC client - let endpoint = format!("https://{}:{}", config.host, config.port); - let client = ClementineVerifierTestClient::new(endpoint, tls_config) - .await - .with_context(|| format!("Failed to create Clementine verifier {} client", index))?; + let timeout = CLEMENTINE_NODE_STARTUP_TIMEOUT; + let start = Instant::now(); + let mut result = Err(anyhow!("initial response value")); + + while result.is_err() && (start.elapsed() < timeout) { + debug!( + "Verifier connect attempt after {} seconds", + start.elapsed().as_secs() + ); + result = ClementineVerifierTestClient::new(endpoint.clone(), tls_config.clone()).await; + + tokio::time::sleep(Duration::from_millis(500)).await; + } + + let client = result.context(format!( + "Failed to connect to Clementine verifier {} in {} seconds", + index, + start.elapsed().as_secs() + ))?; let instance = Self { config: config.clone(), @@ -191,14 +204,8 @@ impl NodeT for ClementineVerifier { type Config = ClementineConfig; type Client = ClementineVerifierTestClient; - async fn spawn(config: &Self::Config, _docker: &Arc>) -> Result { - let mut env_vars = get_common_env_vars(config); - env_vars.insert( - "SECRET_KEY".to_string(), - hex::encode(config.entity_config.secret_key.secret_bytes()), - ); - - spawn_clementine_node(config, "verifier", env_vars).await + async fn spawn(config: &Self::Config, docker: &Arc>) -> Result { + spawn_clementine_node(config, "verifier", docker).await } fn spawn_output(&mut self) -> &mut SpawnOutput { @@ -237,44 +244,10 @@ pub struct ClementineOperator { impl ClementineOperator { pub async fn new( config: &ClementineConfig, - _docker: Arc>, + docker: Arc>, index: u8, ) -> Result { - let mut env_vars = get_common_env_vars(config); - - // Operator specific configuration - env_vars.insert( - "SECRET_KEY".to_string(), - hex::encode(config.entity_config.secret_key.secret_bytes()), - ); - env_vars.insert( - "WINTERNITZ_SECRET_KEY".to_string(), - hex::encode(config.entity_config.winternitz_secret_key.secret_bytes()), - ); - env_vars.insert( - "OPERATOR_WITHDRAWAL_FEE_SATS".to_string(), - config - .entity_config - .operator_withdrawal_fee_sats - .to_sat() - .to_string(), - ); - - if let Some(addr) = &config.entity_config.operator_reimbursement_address { - env_vars.insert( - "OPERATOR_REIMBURSEMENT_ADDRESS".to_string(), - addr.clone().assume_checked().to_string(), - ); - } - - if let Some(outpoint) = &config.entity_config.operator_collateral_funding_outpoint { - env_vars.insert( - "OPERATOR_COLLATERAL_FUNDING_OUTPOINT".to_string(), - format!("{}:{}", outpoint.txid, outpoint.vout), - ); - } - - let spawn_output = spawn_clementine_node(config, "operator", env_vars).await?; + let spawn_output = ::spawn(config, &docker).await?; // Wait for the gRPC server to be ready wait_for_tcp_bound( @@ -293,11 +266,27 @@ impl ClementineOperator { "localhost".to_string(), ); - // Create gRPC client - let endpoint = format!("https://{}:{}", config.host, config.port); - let client = ClementineOperatorTestClient::new(endpoint, tls_config) - .await - .with_context(|| format!("Failed to create Clementine operator {} client", index))?; + let endpoint = format!("https://127.0.0.1:{}", config.port); + + let timeout = CLEMENTINE_NODE_STARTUP_TIMEOUT; + let start = Instant::now(); + let mut result = Err(anyhow!("initial response value")); + + while result.is_err() && (start.elapsed() < timeout) { + debug!( + "Operator connect attempt after {} seconds", + start.elapsed().as_secs() + ); + result = ClementineOperatorTestClient::new(endpoint.clone(), tls_config.clone()).await; + + tokio::time::sleep(Duration::from_millis(500)).await; + } + + let client = result.context(format!( + "Failed to connect to Clementine verifier {} in {} seconds", + index, + start.elapsed().as_secs() + ))?; let instance = Self { config: config.clone(), @@ -316,42 +305,8 @@ impl NodeT for ClementineOperator { type Config = ClementineConfig; type Client = ClementineOperatorTestClient; - async fn spawn(config: &Self::Config, _docker: &Arc>) -> Result { - let mut env_vars = get_common_env_vars(config); - - // Operator specific configuration - env_vars.insert( - "SECRET_KEY".to_string(), - hex::encode(config.entity_config.secret_key.secret_bytes()), - ); - env_vars.insert( - "WINTERNITZ_SECRET_KEY".to_string(), - hex::encode(config.entity_config.winternitz_secret_key.secret_bytes()), - ); - env_vars.insert( - "OPERATOR_WITHDRAWAL_FEE_SATS".to_string(), - config - .entity_config - .operator_withdrawal_fee_sats - .to_sat() - .to_string(), - ); - - if let Some(addr) = &config.entity_config.operator_reimbursement_address { - env_vars.insert( - "OPERATOR_REIMBURSEMENT_ADDRESS".to_string(), - addr.clone().assume_checked().to_string(), - ); - } - - if let Some(outpoint) = &config.entity_config.operator_collateral_funding_outpoint { - env_vars.insert( - "OPERATOR_COLLATERAL_FUNDING_OUTPOINT".to_string(), - format!("{}:{}", outpoint.txid, outpoint.vout), - ); - } - - spawn_clementine_node(config, "operator", env_vars).await + async fn spawn(config: &Self::Config, docker: &Arc>) -> Result { + spawn_clementine_node(config, "operator", docker).await } fn spawn_output(&mut self) -> &mut SpawnOutput { @@ -455,45 +410,194 @@ pub fn copy_resources( /// Ensures that TLS certificates exist for tests. /// This will run the certificate generation script if certificates don't exist. pub async fn generate_certs_if_needed() -> std::result::Result<(), std::io::Error> { - // avoids double generation of certs when multiple tests run in parallel - static GENERATE_LOCK: LazyLock> = - LazyLock::new(|| tokio::sync::Mutex::new(())); - - if !get_workspace_root() - .join("resources/clementine/certs/ca/ca.pem") - .exists() - { - let _lock = GENERATE_LOCK.lock().await; + // Prepare lock file next to the certs directory + let lock_file_path = get_workspace_root().join("resources/clementine/certs/.certs.lock"); + if let Some(dir) = lock_file_path.parent() { + let _ = std::fs::create_dir_all(dir); + } + let lock_file = std::fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_file_path)?; + + // Acquire an exclusive cross-process lock + use fs2::FileExt; + lock_file.lock_exclusive()?; + + // Ensure we always release the lock + let res = async { + let ca_pem = get_workspace_root().join("resources/clementine/certs/ca/ca.pem"); + if !ca_pem.exists() { + debug!("Generating TLS certificates for tests..."); + let script_path = get_workspace_root().join("resources/clementine/generate-certs.sh"); + let output = Command::new("/bin/bash").arg(script_path).output().await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("Failed to generate certificates: {}", stderr); + return Err(std::io::Error::other(format!( + "Certificate generation failed: {}", + stderr + ))); + } + } + Ok(()) + } + .await; - debug!("Generating TLS certificates for tests..."); + let _ = fs2::FileExt::unlock(&lock_file); - let script_path = get_workspace_root().join("resources/clementine/generate-certs.sh"); + res +} - let output = Command::new("sh").arg(script_path).output().await?; +/// Ensures that a docker client binary exists for Clementine containers. +/// If no env is provided, uses a default static Docker client URL. +/// This downloads to `resources/docker/docker-linux-amd64` under +/// an exclusive file lock so concurrent test runs do not race. +/// Returns the path to the binary if it exists after this call, otherwise None. +pub async fn ensure_docker_client_if_needed() -> std::result::Result, std::io::Error> +{ + let url = std::env::var("CLEMENTINE_DOCKER_BINARY_URL") + .ok() + .filter(|u| !u.is_empty()) + .unwrap_or_else(|| DEFAULT_DOCKER_CLIENT_URL.to_string()); + + // Prepare lock file next to the docker binary directory + let lock_file_path = get_workspace_root().join("resources/docker/.docker.lock"); + if let Some(dir) = lock_file_path.parent() { + let _ = std::fs::create_dir_all(dir); + } + let lock_file = std::fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_file_path)?; + + // Acquire an exclusive cross-process lock + use fs2::FileExt; + lock_file.lock_exclusive()?; + + // Ensure we always release the lock + let res = async { + let target_path = get_workspace_root().join("resources/docker/docker-linux-amd64"); + + if !target_path.exists() { + debug!("Downloading docker client for Clementine from {}...", url); + if let Some(dir) = target_path.parent() { + let _ = tokio::fs::create_dir_all(dir).await; + } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - error!("Failed to generate certificates: {}", stderr); - return Err(std::io::Error::other(format!( - "Certificate generation failed: {}", - stderr - ))); + if url.ends_with(".tgz") || url.ends_with(".tar.gz") { + // Download tarball + let tmp_tgz = target_path.with_extension("tgz.partial"); + let output = Command::new("/usr/bin/env") + .arg("bash") + .arg("-c") + .arg(format!("curl -fsSL '{}' -o '{}'", url, tmp_tgz.display())) + .output() + .await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("Failed to download docker client tarball: {}", stderr); + return Err(std::io::Error::other(format!( + "Docker client download failed: {}", + stderr + ))); + } + + // Extract and move the docker binary into place + let extract_dir = target_path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")) + .join(".docker-extract"); + let _ = tokio::fs::remove_dir_all(&extract_dir).await; + tokio::fs::create_dir_all(&extract_dir).await?; + + let output = Command::new("/usr/bin/env") + .arg("bash") + .arg("-c") + .arg(format!( + "tar -xzf '{}' -C '{}'", + tmp_tgz.display(), + extract_dir.display() + )) + .output() + .await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("Failed to extract docker client tarball: {}", stderr); + return Err(std::io::Error::other(format!( + "Docker client extract failed: {}", + stderr + ))); + } + + let extracted_binary = extract_dir.join("docker").join("docker"); + if !extracted_binary.exists() { + return Err(std::io::Error::other("Extracted docker binary not found")); + } + + let mut perms = tokio::fs::metadata(&extracted_binary).await?.permissions(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + perms.set_mode(0o755); + } + tokio::fs::set_permissions(&extracted_binary, perms).await?; + tokio::fs::rename(&extracted_binary, &target_path).await?; + let _ = tokio::fs::remove_dir_all(&extract_dir).await; + let _ = tokio::fs::remove_file(&tmp_tgz).await; + } else { + // Download single binary + let tmp_path = target_path.with_extension("partial"); + let output = Command::new("/usr/bin/env") + .arg("bash") + .arg("-c") + .arg(format!("curl -fsSL '{}' -o '{}'", url, tmp_path.display())) + .output() + .await?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + error!("Failed to download docker client: {}", stderr); + return Err(std::io::Error::other(format!( + "Docker client download failed: {}", + stderr + ))); + } + + let mut perms = tokio::fs::metadata(&tmp_path).await?.permissions(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + perms.set_mode(0o755); + } + tokio::fs::set_permissions(&tmp_path, perms).await?; + tokio::fs::rename(&tmp_path, &target_path).await?; + } } + + Ok(Some(target_path)) } + .await; - Ok(()) + let _ = fs2::FileExt::unlock(&lock_file); + + res } /// Shared function to spawn any Clementine node type -async fn spawn_clementine_node( +async fn spawn_clementine_node( config: &ClementineConfig, role: &str, - mut env_vars: HashMap, + docker: &Arc>, ) -> Result where ClementineConfig: LogPathProvider, { - let binary_path = get_clementine_path()?; + let idx = config.entity_config.idx(); if std::env::var("RISC0_DEV_MODE") != Ok("1".to_string()) && cfg!(target_arch = "aarch64") { warn!("Spawning Clementine {role} without dev mode in arm64, likely to crash"); @@ -501,140 +605,140 @@ where debug!("Spawning Clementine {} with config {:?}", role, config); - // Create log directory if it doesn't exist + // Create directories if they don't exist tokio::fs::create_dir_all(&config.log_dir) .await .context("Failed to create log directory")?; + tokio::fs::create_dir_all(&config.base_dir.join("configs")) + .await + .context("Failed to create base directory")?; let log_path = config.log_path(); let stderr_path = config.stderr_path(); - let stdout_file = File::create(&log_path).context("Failed to create stdout file")?; info!("{} stdout logs available at: {}", role, log_path.display()); let stderr_file = File::create(&stderr_path).context("Failed to create stderr file")?; - // Add Rust runtime environment variables if they exist - for var in &["RUSTFLAGS", "CARGO_LLVM_COV", "LLVM_PROFILE_FILE"] { - if let Ok(val) = std::env::var(var) { - env_vars.insert(var.to_string(), val); - } - } - - // Build command arguments - let mut args = vec![role.to_string()]; - // Add protocol paramset if specified - if let Some(paramset_path) = &config.protocol_paramset { - args = vec![ - "--protocol-params".to_string(), - paramset_path.display().to_string(), - role.to_string(), - ]; - } - - let child = Command::new(&binary_path) - .args(args) - .envs(env_vars) - .stdout(Stdio::from(stdout_file)) - .stderr(Stdio::from(stderr_file)) - .spawn() - .with_context(|| format!("Failed to spawn Clementine {} process", role))?; - - Ok(SpawnOutput::Child(child)) -} - -/// Common environment variables shared by all Clementine node types -fn get_common_env_vars(config: &ClementineConfig) -> HashMap { - let mut env = HashMap::new(); - - // Basic environment setup - env.insert("READ_CONFIG_FROM_ENV".to_string(), "1".to_string()); - env.insert("HOST".to_string(), config.host.clone()); - env.insert("PORT".to_string(), config.port.to_string()); - env.insert("DB_HOST".to_string(), config.db_host.clone()); - env.insert("DB_PORT".to_string(), config.db_port.to_string()); - env.insert("DB_USER".to_string(), config.db_user.clone()); - env.insert("DB_PASSWORD".to_string(), config.db_password.clone()); - env.insert("DB_NAME".to_string(), config.db_name.clone()); - - // Bitcoin configuration - env.insert( - "BITCOIN_RPC_URL".to_string(), - config.bitcoin_rpc_url.clone(), - ); - env.insert( - "BITCOIN_RPC_USER".to_string(), - config.bitcoin_rpc_user.clone(), - ); - env.insert( - "BITCOIN_RPC_PASSWORD".to_string(), - config.bitcoin_rpc_password.clone(), - ); + let paramset_path = config + .protocol_paramset + .as_ref() + .expect("Expected paramset to be defined here by ClementineConfig"); + + let config_path = config + .base_dir + .join("configs") + .join(format!("{role}-{idx}.toml")); + + tokio::fs::write(&config_path, toml::to_string(&config).unwrap()).await?; + + let args = vec![ + "--protocol-params".to_string(), + paramset_path.display().to_string(), + "--config".to_string(), + config_path.display().to_string(), + role.to_string(), + ]; + + let bitvm_cache_path = { + let Ok(cache) = std::env::var("BITVM_CACHE_PATH") else { + anyhow::bail!("BITVM_CACHE_PATH is not set for Clementine {role}"); + }; - // Citrea configuration - env.insert("CITREA_RPC_URL".to_string(), config.citrea_rpc_url.clone()); - env.insert( - "CITREA_LIGHT_CLIENT_PROVER_URL".to_string(), - config.citrea_light_client_prover_url.clone(), - ); - env.insert( - "CITREA_CHAIN_ID".to_string(), - config.citrea_chain_id.to_string(), - ); - env.insert( - "BRIDGE_CONTRACT_ADDRESS".to_string(), - config.bridge_contract_address.clone(), - ); + let cache_path = PathBuf::from(cache); - // TLS configuration - env.insert( - "CA_CERT_PATH".to_string(), - config.ca_cert_path.display().to_string(), - ); - env.insert( - "SERVER_CERT_PATH".to_string(), - config.server_cert_path.display().to_string(), - ); - env.insert( - "SERVER_KEY_PATH".to_string(), - config.server_key_path.display().to_string(), - ); - env.insert( - "CLIENT_CERT_PATH".to_string(), - config.client_cert_path.display().to_string(), - ); - env.insert( - "CLIENT_KEY_PATH".to_string(), - config.client_key_path.display().to_string(), - ); - env.insert( - "AGGREGATOR_CERT_PATH".to_string(), - config.aggregator_cert_path.display().to_string(), - ); - env.insert( - "CLIENT_VERIFICATION".to_string(), - if config.client_verification { - "1".to_string() - } else { - "0".to_string() - }, - ); - - // Security council - env.insert( - "SECURITY_COUNCIL".to_string(), - config.security_council.to_string(), - ); + if !matches!(tokio::fs::try_exists(&cache_path).await, Ok(true)) { + anyhow::bail!("BITVM_CACHE_PATH does not exist: {}", cache_path.display()); + } - // Telemetry (default configuration, may be overridden by specific node types) - if let Some(telemetry) = &config.telemetry { - env.insert("TELEMETRY_HOST".to_string(), telemetry.host.clone()); - env.insert("TELEMETRY_PORT".to_string(), telemetry.port.to_string()); + cache_path + }; + + // Set RISC0_WORK_DIR for HCP + let work_dir = config + .base_dir + .join("workdir") + .join(format!("clementine-{}-{}", role, idx)); + + // Ensure host work dir exists before bind-mounting + if let Err(e) = tokio::fs::create_dir_all(&work_dir).await { + error!( + "Failed to create RISC0_WORK_DIR at {}: {}", + work_dir.display(), + e + ); } - env + let env = { + let mut env = HashMap::new(); + // Inherit environment variables from the host process + for var in &[ + "RUSTFLAGS", + "CARGO_LLVM_COV", + "LLVM_PROFILE_FILE", + "BITVM_CACHE_PATH", + "RISC0_DEV_MODE", + "RUST_MIN_STACK", + "RUST_LOG", + ] { + if let Ok(val) = std::env::var(var) { + env.insert(var.to_string(), val); + } + } + + env.insert("RISC0_WORK_DIR".to_string(), work_dir.display().to_string()); + env + }; + + let spawn_output = match docker.as_ref() { + Some(docker) if docker.clementine() => { + // Ensure docker client binary if configured; mounting is handled centrally in DockerEnv + let _ = ensure_docker_client_if_needed().await; + docker + .spawn(DockerConfig { + ports: vec![config.port], + image: config + .image + .as_deref() + .unwrap_or(DEFAULT_CLEMENTINE_DOCKER_IMAGE) + .to_string(), + cmd: args, + host_dir: Some({ + let mounts = vec![ + // Mount the base_dir for config and paramset to be accessible + config.base_dir.display().to_string(), + paramset_path.display().to_string(), + // Mount the bitvm cache + bitvm_cache_path.display().to_string(), + work_dir.display().to_string(), + ]; + mounts + }), + log_path: config.log_path(), + volume: VolumeConfig { + name: role.to_string(), + target: "/not-used".to_string(), + }, + kind: config.kind(), + throttle: None, + env, + }) + .await? + } + _ => SpawnOutput::Child( + Command::new(&get_clementine_path()?) + .args(args) + .envs(env) + .stdout(Stdio::from(stdout_file)) + .stderr(Stdio::from(stderr_file)) + .spawn() + .with_context(|| format!("Failed to spawn Clementine {} process", role))?, + ), + }; + + Ok(spawn_output) } async fn setup_clementine_databases( @@ -657,13 +761,12 @@ async fn setup_clementine_databases( // Get Postgres connection info from the first verifier config if let Some(first_verifier) = verifiers.first() { let db_user = &first_verifier.db_user; - let db_host = &first_verifier.db_host; let db_port = first_verifier.db_port; // Set environment variables for Postgres tools std::env::set_var("PGUSER", db_user); std::env::set_var("PGPASSWORD", &first_verifier.db_password); - std::env::set_var("PGHOST", db_host); + std::env::set_var("PGHOST", "127.0.0.1"); std::env::set_var("PGPORT", db_port.to_string()); // Drop and recreate databases diff --git a/src/config/clementine.rs b/src/config/clementine.rs index a19499d..7a6ed0b 100644 --- a/src/config/clementine.rs +++ b/src/config/clementine.rs @@ -12,6 +12,7 @@ use tempfile::TempDir; use crate::{ config::{BitcoinConfig, PostgresConfig, RpcConfig}, + docker::DockerEnv, log_provider::LogPathProvider, node::NodeKind, }; @@ -122,10 +123,22 @@ impl Default for TelemetryConfig { } } -#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct AggregatorConfig { pub verifier_endpoints: Vec, pub operator_endpoints: Vec, + pub secret_key: SecretKey, +} + +impl Default for AggregatorConfig { + fn default() -> Self { + Self { + verifier_endpoints: Vec::new(), + operator_endpoints: Vec::new(), + secret_key: SecretKey::from_slice(&seeded_key("aggregator", 0)) + .expect("known valid input"), + } + } } impl ClementineConfig { @@ -137,16 +150,18 @@ impl ClementineConfig { bitcoin_config: BitcoinConfig, citrea_rpc: RpcConfig, citrea_light_client_prover_rpc: RpcConfig, + docker: &Option, clementine_dir: PathBuf, port: u16, overrides: ClementineConfig, ) -> Self { + let mut entity_config = overrides.entity_config.clone(); + entity_config.verifier_endpoints = verifier_endpoints; + entity_config.operator_endpoints = operator_endpoints; + Self { port, - entity_config: AggregatorConfig { - verifier_endpoints, - operator_endpoints, - }, + entity_config, // Aggregator uses the first verifier's database db_name: format!("clementine-{}", 0), ..ClementineConfig::::from_configs( @@ -154,6 +169,7 @@ impl ClementineConfig { bitcoin_config, citrea_rpc, citrea_light_client_prover_rpc, + docker, clementine_dir, overrides, ) @@ -213,6 +229,7 @@ impl ClementineConfig { bitcoin_config: BitcoinConfig, citrea_rpc: RpcConfig, citrea_light_client_prover: RpcConfig, + docker: &Option, clementine_dir: PathBuf, port: u16, overrides: ClementineConfig, @@ -226,6 +243,7 @@ impl ClementineConfig { bitcoin_config, citrea_rpc, citrea_light_client_prover, + docker, clementine_dir, overrides, ) @@ -269,6 +287,7 @@ impl ClementineConfig { bitcoin_config: BitcoinConfig, citrea_rpc: RpcConfig, citrea_light_client_prover_rpc: RpcConfig, + docker: &Option, clementine_dir: PathBuf, port: u16, overrides: ClementineConfig, @@ -282,6 +301,7 @@ impl ClementineConfig { bitcoin_config, citrea_rpc, citrea_light_client_prover_rpc, + docker, clementine_dir, overrides, ) @@ -291,7 +311,7 @@ impl ClementineConfig { /// Configuration options for any Clementine target (tests, binaries etc.). /// Named `BridgeConfig` in the original Clementine codebase. #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ClementineConfig { +pub struct ClementineConfig { // -- Required by all entities -- /// Protocol paramset /// @@ -334,6 +354,15 @@ pub struct ClementineConfig { /// Security council. pub security_council: SecurityCouncil, + /// The ECDSA address of the citrea/aggregator that will sign the withdrawal params + /// after manual verification of the optimistic payout and operator's withdrawal. + /// Used for both an extra verification of aggregator's identity and to force citrea + /// to check withdrawal params manually during some time after launch. + pub aggregator_verification_address: Option, + + /// The X25519 public key that will be used to encrypt the emergency stop message. + pub emergency_stop_encryption_public_key: Option<[u8; 32]>, + // TLS certificates /// Path to the server certificate file. /// @@ -380,13 +409,19 @@ pub struct ClementineConfig { /// Logging directory (used in Citrea-E2E code, NOT passed to Clementine) pub log_dir: PathBuf, + + /// Base directory for Clementine + pub base_dir: PathBuf, + + /// Docker image to use for the node + pub image: Option, } -impl Default for ClementineConfig { +impl Default for ClementineConfig { fn default() -> Self { Self { protocol_paramset: None, - host: "127.0.0.1".to_string(), + host: "0.0.0.0".to_string(), port: 17000, bitcoin_rpc_url: "http://127.0.0.1:18443/wallet/admin".to_string(), @@ -411,6 +446,13 @@ impl Default for ClementineConfig { threshold: 1, }, + emergency_stop_encryption_public_key: Some( + hex::decode("025d32d10ec7b899df4eeb4d80918b7f0a1f2a28f6af24f71aa2a59c69c0d531") + .expect("valid hex") + .try_into() + .expect("valid key"), + ), + server_cert_path: PathBuf::from("certs/server/server.pem"), server_key_path: PathBuf::from("certs/server/server.key"), client_cert_path: PathBuf::from("certs/client/client.pem"), @@ -421,16 +463,30 @@ impl Default for ClementineConfig { telemetry: Some(TelemetryConfig::default()), + // This defaults to a fixed key in Clementine + aggregator_verification_address: None, + entity_config: E::default(), log_dir: TempDir::new() .expect("Failed to create temporary directory") .keep(), + + base_dir: TempDir::new() + .expect("Failed to create temporary directory") + .keep(), + + image: None, } } } -impl ClementineConfig { +impl ClementineConfig { + pub fn with_aggregator_verification(mut self, address: alloy_primitives::Address) -> Self { + self.aggregator_verification_address = Some(address); + self + } + /// Uses other configs to generate a ClementineConfig for the given entity type. /// /// Matches the AggregatorConfig type to determine if the entity is an @@ -442,6 +498,7 @@ impl ClementineConfig { bitcoin_config: BitcoinConfig, citrea_rpc: RpcConfig, citrea_light_client_prover_rpc: RpcConfig, + docker: &Option, base_dir: PathBuf, overrides: ClementineConfig, ) -> Self { @@ -449,26 +506,43 @@ impl ClementineConfig { std::any::TypeId::of::() == std::any::TypeId::of::(); let certificate_base_dir = base_dir.join("certs"); + let full_node_host = docker + .as_ref() + .and_then(|d| d.citrea().then(|| d.get_hostname(&NodeKind::FullNode))) + .unwrap_or("127.0.0.1".to_string()); + let lcp_host = docker + .as_ref() + .and_then(|d| { + d.citrea() + .then(|| d.get_hostname(&NodeKind::LightClientProver)) + }) + .unwrap_or("127.0.0.1".to_string()); + Self { // TODO: need to change the host to 127.0.0.1 until docker support is added bitcoin_rpc_url: format!( - "http://127.0.0.1:{}/wallet/{}", + "http://{}:{}/wallet/{}", + bitcoin_config + .docker_host + .unwrap_or("127.0.0.1".to_string()), bitcoin_config.rpc_port, NodeKind::Bitcoin ), bitcoin_rpc_user: bitcoin_config.rpc_user, bitcoin_rpc_password: bitcoin_config.rpc_password, - db_host: "127.0.0.1".to_string(), + db_host: postgres_config + .docker_host + .unwrap_or("127.0.0.1".to_string()), db_port: postgres_config.port as usize, db_user: postgres_config.user, db_password: postgres_config.password, db_name: "clementine".to_string(), // overriden by caller - citrea_rpc_url: format!("http://127.0.0.1:{}", citrea_rpc.bind_port), + citrea_rpc_url: format!("http://{}:{}", full_node_host, citrea_rpc.bind_port), citrea_light_client_prover_url: format!( - "http://127.0.0.1:{}", - citrea_light_client_prover_rpc.bind_port + "http://{}:{}", + lcp_host, citrea_light_client_prover_rpc.bind_port ), server_cert_path: certificate_base_dir.join("server").join("server.pem"), @@ -500,6 +574,8 @@ impl ClementineConfig { Some(base_dir.join("regtest_paramset.toml")) }), + base_dir: base_dir.clone(), + // These values can be overridden by the caller using overrides. // header_chain_proof_path: None, // security_council: SecurityCouncil { @@ -536,7 +612,7 @@ impl LogPathProvider for ClementineConfig { } fn kind(&self) -> NodeKind { - NodeKind::ClementineOperator + NodeKind::ClementineOperator(self.entity_config.idx()) } fn stderr_path(&self) -> PathBuf { @@ -552,7 +628,7 @@ impl LogPathProvider for ClementineConfig { } fn kind(&self) -> NodeKind { - NodeKind::ClementineVerifier + NodeKind::ClementineVerifier(self.entity_config.idx()) } fn stderr_path(&self) -> PathBuf { @@ -568,6 +644,28 @@ pub struct ClementineClusterConfig { pub verifiers: Vec>, } +pub trait ClementineEntityConfig: Serialize + Debug + Clone + Default { + fn idx(&self) -> u8; +} + +impl ClementineEntityConfig for AggregatorConfig { + fn idx(&self) -> u8 { + 0 + } +} + +impl ClementineEntityConfig for OperatorConfig { + fn idx(&self) -> u8 { + self.idx + } +} + +impl ClementineEntityConfig for VerifierConfig { + fn idx(&self) -> u8 { + self.idx + } +} + #[cfg(test)] mod test { use super::*; diff --git a/src/config/docker.rs b/src/config/docker.rs index 81bc2b1..aaf5874 100644 --- a/src/config/docker.rs +++ b/src/config/docker.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, path::PathBuf}; +use std::{collections::HashMap, fmt::Debug, path::PathBuf}; use serde::Serialize; use tracing::debug; @@ -33,6 +33,7 @@ pub struct DockerConfig { pub host_dir: Option>, pub kind: NodeKind, pub throttle: Option, + pub env: HashMap, } impl From<&BitcoinConfig> for DockerConfig { @@ -61,6 +62,7 @@ impl From<&BitcoinConfig> for DockerConfig { host_dir: None, kind: NodeKind::Bitcoin, throttle: None, // Not supported for bitcoin yet. Easy to toggle if it ever makes sense to throttle bitcoind nodes + env: HashMap::new(), } } } @@ -95,6 +97,7 @@ where ]), kind, throttle: config.throttle.clone(), + env: HashMap::new(), } } } @@ -131,6 +134,7 @@ impl From<&PostgresConfig> for DockerConfig { host_dir: None, kind: NodeKind::Postgres, throttle: None, + env: HashMap::new(), } } } diff --git a/src/config/mod.rs b/src/config/mod.rs index 6e75497..3a2c6b2 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -17,7 +17,8 @@ use std::{ pub use bitcoin::BitcoinConfig; #[cfg(feature = "clementine")] pub use clementine::{ - AggregatorConfig, ClementineClusterConfig, ClementineConfig, OperatorConfig, VerifierConfig, + AggregatorConfig, ClementineClusterConfig, ClementineConfig, ClementineEntityConfig, + OperatorConfig, VerifierConfig, }; pub use docker::{DockerConfig, VolumeConfig}; #[cfg(feature = "clementine")] diff --git a/src/config/postgres.rs b/src/config/postgres.rs index c621c77..7b7de44 100644 --- a/src/config/postgres.rs +++ b/src/config/postgres.rs @@ -9,6 +9,7 @@ use crate::{log_provider::LogPathProvider, node::NodeKind}; pub struct PostgresConfig { pub port: u16, pub user: String, + pub docker_host: Option, pub password: String, pub log_dir: PathBuf, pub extra_args: Vec, @@ -21,6 +22,7 @@ impl Default for PostgresConfig { Self { port: 5432, user: "clementine".to_string(), + docker_host: None, password: "clementine".to_string(), log_dir: TempDir::new() .expect("Failed to create temporary directory") diff --git a/src/docker.rs b/src/docker.rs index 3c17dae..06be92e 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -1,7 +1,7 @@ #![allow(deprecated)] // Allowing deprecation for now as bollard v0.19.1 has bogus warning messages that cannot be fixed as of now. TODO remove when possible use std::{ collections::{HashMap, HashSet}, - path::PathBuf, + path::{Path, PathBuf}, }; use anyhow::{anyhow, Context, Result}; @@ -9,7 +9,9 @@ use bollard::{ container::{Config, LogOutput, NetworkingConfig}, models::{EndpointSettings, Mount, PortBinding}, network::CreateNetworkOptions, - query_parameters::{CreateContainerOptions, CreateImageOptions, LogsOptions}, + query_parameters::{ + CreateContainerOptions, CreateImageOptions, ListContainersOptionsBuilder, LogsOptions, + }, secret::MountTypeEnum, service::HostConfig, volume::CreateVolumeOptions, @@ -20,7 +22,7 @@ use tokio::{fs::File, io::AsyncWriteExt, sync::Mutex, task::JoinHandle}; use tracing::{debug, error, info}; use super::{config::DockerConfig, traits::SpawnOutput, utils::generate_test_id}; -use crate::{config::TestCaseDockerConfig, node::NodeKind}; +use crate::{config::TestCaseDockerConfig, node::NodeKind, utils::get_workspace_root}; #[derive(Debug)] pub struct ContainerSpawnOutput { @@ -96,7 +98,8 @@ impl DockerEnv { } pub fn get_hostname(&self, kind: &NodeKind) -> String { - format!("{kind}-{}", self.id) + // Use a three-label domain so wildcard SANs like *.e2e.internal are valid for rustls + format!("{kind}-{}.e2e.internal", self.id) } pub async fn spawn(&self, config: DockerConfig) -> Result { @@ -128,7 +131,8 @@ impl DockerEnv { network_config.insert( self.network_info.id.clone(), EndpointSettings { - ip_address: Some(self.get_hostname(&config.kind)), + // ip_address: Some(self.get_hostname(&config.kind)), + aliases: Some(vec![self.get_hostname(&config.kind)]), ..Default::default() }, ); @@ -152,6 +156,38 @@ impl DockerEnv { } } + // Always forward host Docker socket into containers + let docker_sock = "/var/run/docker.sock"; + if Path::new(docker_sock).exists() { + mounts.push(Mount { + target: Some(docker_sock.to_string()), + source: Some(docker_sock.to_string()), + typ: Some(MountTypeEnum::BIND), + ..Default::default() + }); + } else { + debug!("Host docker socket not found at {docker_sock}, skipping mount"); + } + + // Optionally mount a docker CLI into the container if provided in resources + // This is useful for images that do not include the docker client. + let resources_cli = get_workspace_root() + .join("resources") + .join("docker") + .join("docker-linux-amd64"); + if resources_cli.exists() { + mounts.push(Mount { + target: Some("/usr/local/bin/docker".to_string()), + source: Some(resources_cli.display().to_string()), + typ: Some(MountTypeEnum::BIND), + ..Default::default() + }); + debug!( + "Mounted docker CLI from resources: {}", + resources_cli.display() + ); + } + let mut host_config = HostConfig { port_bindings: Some(port_bindings), mounts: Some(mounts), @@ -171,12 +207,23 @@ impl DockerEnv { } } + let mut envs = config.env.clone(); + + // Backwards compatibility for old fixed env + envs.entry("PARALLEL_PROOF_LIMIT".to_string()) + .or_insert("1".to_string()); + + let envs = envs + .into_iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>(); + let container_config = Config { hostname: Some(format!("{}-{}", config.kind, self.id)), image: Some(config.image), cmd: Some(config.cmd), exposed_ports: Some(exposed_ports), - env: Some(vec!["PARALLEL_PROOF_LIMIT=1".to_string()]), // Todo proper env handling + env: Some(envs), host_config: Some(host_config), networking_config: Some(NetworkingConfig { endpoints_config: network_config, @@ -286,7 +333,7 @@ impl DockerEnv { let containers = self .docker - .list_containers(None::) + .list_containers(Some(ListContainersOptionsBuilder::new().all(true).build())) .await?; for container in containers { if let (Some(id), Some(networks)) = ( @@ -300,6 +347,7 @@ impl DockerEnv { None::, ) .await?; + self.docker .remove_container( &id, diff --git a/src/framework.rs b/src/framework.rs index 5e9ab80..21ed7fe 100644 --- a/src/framework.rs +++ b/src/framework.rs @@ -545,21 +545,32 @@ fn generate_test_config( }; #[cfg(feature = "clementine")] - let postgres = PostgresConfig { - port: get_available_port()?, - log_dir: _postgres_dir, - ..Default::default() - }; + let (clementine, postgres) = { + let postgres = PostgresConfig { + port: get_available_port()?, + log_dir: _postgres_dir, + docker_host: docker + .as_ref() + .and_then(|d| d.clementine().then(|| d.get_hostname(&NodeKind::Postgres))), + ..Default::default() + }; - #[cfg(feature = "clementine")] - let clementine = ClementineIntegration::generate_cluster_config::( - &test_case, - &clementine_dir, - postgres.clone(), - bitcoin_confs[0].clone(), - full_node_rollup.rpc.clone(), - light_client_prover_rollup.rpc.clone(), - )?; + let mut clementine_btc_conf = bitcoin_confs[0].clone(); + clementine_btc_conf.docker_host = docker + .as_ref() + .and_then(|d| d.clementine().then(|| d.get_hostname(&NodeKind::Bitcoin))); + + let clementine = ClementineIntegration::generate_cluster_config::( + &test_case, + &clementine_dir, + postgres.clone(), + clementine_btc_conf, + full_node_rollup.rpc.clone(), + light_client_prover_rollup.rpc.clone(), + docker, + )?; + (clementine, postgres) + }; let citrea_docker_image = std::env::var("CITREA_DOCKER_IMAGE").ok(); Ok(TestConfig { diff --git a/src/node.rs b/src/node.rs index 1048b31..12ccabb 100644 --- a/src/node.rs +++ b/src/node.rs @@ -44,9 +44,9 @@ pub enum NodeKind { #[cfg(feature = "clementine")] ClementineAggregator, #[cfg(feature = "clementine")] - ClementineVerifier, + ClementineVerifier(u8), #[cfg(feature = "clementine")] - ClementineOperator, + ClementineOperator(u8), Postgres, } @@ -61,9 +61,9 @@ impl NodeKind { #[cfg(feature = "clementine")] NodeKind::ClementineAggregator => 6, #[cfg(feature = "clementine")] - NodeKind::ClementineVerifier => 7, + NodeKind::ClementineVerifier(_) => 7, #[cfg(feature = "clementine")] - NodeKind::ClementineOperator => 8, + NodeKind::ClementineOperator(_) => 8, NodeKind::Postgres => 9, } } @@ -80,9 +80,9 @@ impl fmt::Display for NodeKind { #[cfg(feature = "clementine")] NodeKind::ClementineAggregator => write!(f, "clementine-aggregator"), #[cfg(feature = "clementine")] - NodeKind::ClementineVerifier => write!(f, "clementine-verifier"), + NodeKind::ClementineVerifier(idx) => write!(f, "clementine-verifier-{idx}"), #[cfg(feature = "clementine")] - NodeKind::ClementineOperator => write!(f, "clementine-operator"), + NodeKind::ClementineOperator(idx) => write!(f, "clementine-operator-{idx}"), NodeKind::Postgres => write!(f, "postgres"), } } diff --git a/tests/clementine.rs b/tests/clementine.rs index 285e0b0..732441e 100644 --- a/tests/clementine.rs +++ b/tests/clementine.rs @@ -7,6 +7,7 @@ use citrea_e2e::{ test_case::{TestCase, TestCaseRunner}, Result, }; +use tracing::info; /// Integration test for Clementine gRPC clients. /// @@ -25,20 +26,21 @@ use citrea_e2e::{ /// ### Verifier Service: /// - `get_params()` - Returns verifier parameters including public key /// - `get_current_status()` - Provides status of the verifier node -struct ClementineIntegrationTest; +struct ClementineIntegrationTest; #[async_trait] -impl TestCase for ClementineIntegrationTest { +impl TestCase for ClementineIntegrationTest { fn test_config() -> TestCaseConfig { TestCaseConfig { with_clementine: true, n_verifiers: 2, n_operators: 2, with_full_node: true, + with_light_client_prover: true, docker: TestCaseDockerConfig { bitcoin: true, citrea: true, - clementine: false, + clementine: WITH_DOCKER, }, ..Default::default() } @@ -87,7 +89,9 @@ impl TestCase for ClementineIntegrationTest { println!( "Operator {}: automation={}, balance={}", - i, status.automation, status.wallet_balance + i, + status.automation, + status.wallet_balance.expect("Balance should be present") ); } @@ -107,15 +111,86 @@ impl TestCase for ClementineIntegrationTest { println!( "Verifier {}: automation={}, balance={}", - i, status.automation, status.wallet_balance + i, + status.automation, + status.wallet_balance.expect("Balance should be present") ); } + // If running Clementine in Docker, mine blocks and ensure HCP catches up + if WITH_DOCKER { + use bitcoincore_rpc::RpcApi; + use citrea_e2e::bitcoin::DEFAULT_FINALITY_DEPTH; + // Mine a bunch of blocks on DA + let da = f.bitcoin_nodes.get(0).unwrap(); + + let target_height = da.get_block_count().await?; + if target_height < 100 { + da.generate(100 - target_height).await?; + } + + // round down to nearest 100, HCP proves in 100 block batches + let target_height = target_height / 100 * 100; + + // ensure target_height finalized + da.generate(DEFAULT_FINALITY_DEPTH + 1).await?; + + // Ask aggregator for entity statuses until HCP height catches up + let mut attempts = 0; + let max_attempts = 120; // allow up to ~2 minutes + loop { + let statuses = clementine + .aggregator + .client + .get_entity_statuses(false) + .await + .expect("Failed to get entity statuses from aggregator"); + + let mut all_ok = true; + for es in statuses.entity_statuses { + if let Some(sr) = es.status_result { + let status = match sr { + citrea_e2e::clementine::client::clementine::entity_status_with_id::StatusResult::Status(s) => s, + _ => { all_ok = false; break; } + }; + let h = status.hcp_last_proven_height.unwrap_or(0) as u64; + if h < target_height { + info!( + "entity {:?} behind, {h} (height) < {target_height} (target)", + es.entity_id.unwrap() + ); + all_ok = false; + break; + } + } else { + all_ok = false; + break; + } + } + if all_ok { + break; + } + attempts += 1; + anyhow::ensure!(attempts < max_attempts, "HCP did not catch up in time"); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } + } + Ok(()) } } #[tokio::test] -async fn test_clementine_integration() -> Result<()> { - TestCaseRunner::new(ClementineIntegrationTest).run().await +async fn test_clementine_integration_w_docker() -> Result<()> { + TestCaseRunner::new(ClementineIntegrationTest::) + .run() + .await +} + +#[tokio::test] +#[ignore = "won't pass before Clementine releases again with fixes"] +async fn test_clementine_integration_wo_docker() -> Result<()> { + TestCaseRunner::new(ClementineIntegrationTest::) + .run() + .await }