diff --git a/Cargo.lock b/Cargo.lock index 48a39cf304d..23e0a7e7354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -953,6 +953,7 @@ dependencies = [ "eth2", "futures", "itertools 0.10.5", + "sensitive_url", "serde", "slot_clock", "strum", diff --git a/book/src/api_vc_endpoints.md b/book/src/api_vc_endpoints.md index 87c9a517a5d..14f4933e171 100644 --- a/book/src/api_vc_endpoints.md +++ b/book/src/api_vc_endpoints.md @@ -19,6 +19,7 @@ | [`POST /lighthouse/validators/web3signer`](#post-lighthousevalidatorsweb3signer) | Add web3signer validators. | | [`GET /lighthouse/logs`](#get-lighthouselogs) | Get logs | | [`GET /lighthouse/beacon/health`](#get-lighthousebeaconhealth) | Get health information for each connected beacon node. | +| [`POST /lighthouse/beacon/update`](#post-lighthousebeaconupdate) | Update the `--beacon-nodes` list. | The query to Lighthouse API endpoints requires authorization, see [Authorization Header](./api_vc_auth_header.md). @@ -926,3 +927,57 @@ curl -X GET http://localhost:5062/lighthouse/beacon/health \ } } ``` + +## `POST /lighthouse/beacon/update` + +Updates the list of beacon nodes originally specified by the `--beacon-nodes` CLI flag. +Use this endpoint when you don't want to restart the VC to add, remove or reorder beacon nodes. + +### HTTP Specification + +| Property | Specification | +|-------------------|--------------------------------------------| +| Path | `/lighthouse/beacon/update` | +| Method | POST | +| Required Headers | [`Authorization`](./api_vc_auth_header.md) | +| Typical Responses | 200, 400 | + +### Example Request Body + +```json +{ + "beacon_nodes": [ + "http://beacon-node1:5052", + "http://beacon-node2:5052", + "http://beacon-node3:5052" + ] +} +``` + +Command: + +```bash +DATADIR=/var/lib/lighthouse +curl -X POST http://localhost:5062/lighthouse/beacon/update \ + -H "Authorization: Bearer $(cat ${DATADIR}/validators/api-token.txt)" \ + -H "Content-Type: application/json" \ + -d "{\"beacon_nodes\":[\"http://beacon-node1:5052\",\"http://beacon-node2:5052\",\"http://beacon-node3:5052\"]}" +``` + +### Example Response Body + +```json +{ + "data": { + "new_beacon_nodes_list": [ + "http://beacon-node1:5052", + "http://beacon-node2:5052", + "http://beacon-node3:5052" + ] + } +} +``` + +If successful, the response will be a copy of the new list included in the request. +If unsuccessful, an error will be shown and the beacon nodes list will not be updated. +You can verify the results of the endpoint by using the `/lighthouse/beacon/health` endpoint. diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 52cc91ba298..47a0c3ea2f0 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -51,6 +51,22 @@ pub const CONTENT_TYPE_HEADER: &str = "Content-Type"; pub const SSZ_CONTENT_TYPE_HEADER: &str = "application/octet-stream"; pub const JSON_CONTENT_TYPE_HEADER: &str = "application/json"; +/// Specific optimized timeout constants for HTTP requests involved in different validator duties. +/// This can help ensure that proper endpoint fallback occurs. +const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; +const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; +const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; +const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; +const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; +const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4; + #[derive(Debug)] pub enum Error { /// The `reqwest` client raised an error. @@ -165,6 +181,26 @@ impl Timeouts { default: timeout, } } + + pub fn use_optimized_timeouts(base_timeout: Duration) -> Self { + Timeouts { + attestation: base_timeout / HTTP_ATTESTATION_TIMEOUT_QUOTIENT, + attester_duties: base_timeout / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, + attestation_subscriptions: base_timeout + / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, + liveness: base_timeout / HTTP_LIVENESS_TIMEOUT_QUOTIENT, + proposal: base_timeout / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, + proposer_duties: base_timeout / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, + sync_committee_contribution: base_timeout + / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT, + sync_duties: base_timeout / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT, + get_beacon_blocks_ssz: base_timeout / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, + get_debug_beacon_states: base_timeout / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, + get_deposit_snapshot: base_timeout / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, + get_validator_block: base_timeout / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, + default: base_timeout / HTTP_DEFAULT_TIMEOUT_QUOTIENT, + } + } } /// A wrapper around `reqwest::Client` which provides convenience methods for interfacing with a diff --git a/common/eth2/src/lighthouse_vc/types.rs b/common/eth2/src/lighthouse_vc/types.rs index d7d5a00df51..4407e30e436 100644 --- a/common/eth2/src/lighthouse_vc/types.rs +++ b/common/eth2/src/lighthouse_vc/types.rs @@ -197,3 +197,13 @@ pub struct SingleExportKeystoresResponse { pub struct SetGraffitiRequest { pub graffiti: GraffitiString, } + +#[derive(Serialize, Deserialize, Debug)] +pub struct UpdateCandidatesRequest { + pub beacon_nodes: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct UpdateCandidatesResponse { + pub new_beacon_nodes_list: Vec, +} diff --git a/validator_client/beacon_node_fallback/Cargo.toml b/validator_client/beacon_node_fallback/Cargo.toml index 3bcb0d7034c..5fe2af4cb0b 100644 --- a/validator_client/beacon_node_fallback/Cargo.toml +++ b/validator_client/beacon_node_fallback/Cargo.toml @@ -13,6 +13,7 @@ clap = { workspace = true } eth2 = { workspace = true } futures = { workspace = true } itertools = { workspace = true } +sensitive_url = { workspace = true } serde = { workspace = true } slot_clock = { workspace = true } strum = { workspace = true } diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index e11cc97e791..b3158cd380c 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -8,8 +8,9 @@ use beacon_node_health::{ IsOptimistic, SyncDistanceTier, }; use clap::ValueEnum; -use eth2::BeaconNodeHttpClient; +use eth2::{BeaconNodeHttpClient, Timeouts}; use futures::future; +use sensitive_url::SensitiveUrl; use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer}; use slot_clock::SlotClock; use std::cmp::Ordering; @@ -455,6 +456,39 @@ impl BeaconNodeFallback { (candidate_info, num_available, num_synced) } + /// Update the list of candidates with a new list. + /// Returns `Ok(new_list)` if the update was successful. + /// Returns `Err(some_err)` if the list is empty. + pub async fn update_candidates_list( + &self, + new_list: Vec, + use_long_timeouts: bool, + ) -> Result, String> { + if new_list.is_empty() { + return Err("list cannot be empty".to_string()); + } + + let timeouts: Timeouts = if new_list.len() == 1 || use_long_timeouts { + Timeouts::set_all(Duration::from_secs(self.spec.seconds_per_slot)) + } else { + Timeouts::use_optimized_timeouts(Duration::from_secs(self.spec.seconds_per_slot)) + }; + + let new_candidates: Vec = new_list + .clone() + .into_iter() + .enumerate() + .map(|(index, url)| { + CandidateBeaconNode::new(BeaconNodeHttpClient::new(url, timeouts.clone()), index) + }) + .collect(); + + let mut candidates = self.candidates.write().await; + *candidates = new_candidates; + + Ok(new_list) + } + /// Loop through ALL candidates in `self.candidates` and update their sync status. /// /// It is possible for a node to return an unsynced status while continuing to serve diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index aebe179567e..d5de24229c4 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -22,6 +22,7 @@ use account_utils::{ }; pub use api_secret::ApiSecret; use beacon_node_fallback::CandidateInfo; +use core::convert::Infallible; use create_validator::{ create_validators_mnemonic, create_validators_web3signer, get_voting_password_storage, }; @@ -30,7 +31,7 @@ use eth2::lighthouse_vc::{ std_types::{AuthResponse, GetFeeRecipientResponse, GetGasLimitResponse}, types::{ self as api_types, GenericResponse, GetGraffitiResponse, Graffiti, PublicKey, - PublicKeyBytes, SetGraffitiRequest, + PublicKeyBytes, SetGraffitiRequest, UpdateCandidatesRequest, UpdateCandidatesResponse, }, }; use health_metrics::observe::Observe; @@ -38,6 +39,7 @@ use lighthouse_version::version_with_platform; use logging::crit; use logging::SSELoggingComponents; use parking_lot::RwLock; +use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use slot_clock::SlotClock; use std::collections::HashMap; @@ -53,7 +55,8 @@ use tracing::{info, warn}; use types::{ChainSpec, ConfigAndPreset, EthSpec}; use validator_dir::Builder as ValidatorDirBuilder; use validator_services::block_service::BlockService; -use warp::{sse::Event, Filter}; +use warp::{reply::Response, sse::Event, Filter}; +use warp_utils::reject::convert_rejection; use warp_utils::task::blocking_json_task; #[derive(Debug)] @@ -102,6 +105,7 @@ pub struct Config { pub allow_keystore_export: bool, pub store_passwords_in_secrets_dir: bool, pub http_token_path: PathBuf, + pub bn_long_timeouts: bool, } impl Default for Config { @@ -121,6 +125,7 @@ impl Default for Config { allow_keystore_export: false, store_passwords_in_secrets_dir: false, http_token_path, + bn_long_timeouts: false, } } } @@ -147,6 +152,7 @@ pub fn serve( let config = &ctx.config; let allow_keystore_export = config.allow_keystore_export; let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir; + let use_long_timeouts = config.bn_long_timeouts; // Configure CORS. let cors_builder = { @@ -839,6 +845,59 @@ pub fn serve( }) }); + // POST /lighthouse/beacon/update + let post_lighthouse_beacon_update = warp::path("lighthouse") + .and(warp::path("beacon")) + .and(warp::path("update")) + .and(warp::path::end()) + .and(warp::body::json()) + .and(block_service_filter.clone()) + .then( + move |request: UpdateCandidatesRequest, + block_service: BlockService, T>| async move { + async fn parse_urls(urls: &[String]) -> Result, Response> { + match urls + .iter() + .map(|url| SensitiveUrl::parse(url).map_err(|e| e.to_string())) + .collect() + { + Ok(sensitive_urls) => Ok(sensitive_urls), + Err(_) => Err(convert_rejection::(Err( + warp_utils::reject::custom_bad_request( + "one or more urls could not be parsed".to_string(), + ), + )) + .await), + } + } + + let beacons: Vec = match parse_urls(&request.beacon_nodes).await { + Ok(new_beacons) => { + match block_service + .beacon_nodes + .update_candidates_list(new_beacons, use_long_timeouts) + .await + { + Ok(beacons) => beacons, + Err(e) => { + return convert_rejection::(Err( + warp_utils::reject::custom_bad_request(e.to_string()), + )) + .await + } + } + } + Err(e) => return e, + }; + + let response: UpdateCandidatesResponse = UpdateCandidatesResponse { + new_beacon_nodes_list: beacons.iter().map(|surl| surl.to_string()).collect(), + }; + + blocking_json_task(move || Ok(api_types::GenericResponse::from(response))).await + }, + ); + // Standard key-manager endpoints. let eth_v1 = warp::path("eth").and(warp::path("v1")); let std_keystores = eth_v1.and(warp::path("keystores")).and(warp::path::end()); @@ -1316,6 +1375,7 @@ pub fn serve( .or(post_std_keystores) .or(post_std_remotekeys) .or(post_graffiti) + .or(post_lighthouse_beacon_update) .recover(warp_utils::reject::handle_rejection), )) .or(warp::patch() diff --git a/validator_client/http_api/src/test_utils.rs b/validator_client/http_api/src/test_utils.rs index 08447a82ce7..8c23f79fd3b 100644 --- a/validator_client/http_api/src/test_utils.rs +++ b/validator_client/http_api/src/test_utils.rs @@ -173,6 +173,7 @@ impl ApiTester { allow_keystore_export: true, store_passwords_in_secrets_dir: false, http_token_path: tempdir().unwrap().path().join(PK_FILENAME), + bn_long_timeouts: false, } } diff --git a/validator_client/http_api/src/tests.rs b/validator_client/http_api/src/tests.rs index 4b1a3c0059c..7d421cd7d58 100644 --- a/validator_client/http_api/src/tests.rs +++ b/validator_client/http_api/src/tests.rs @@ -126,6 +126,7 @@ impl ApiTester { allow_keystore_export: true, store_passwords_in_secrets_dir: false, http_token_path: token_path, + bn_long_timeouts: false, }, sse_logging_components: None, slot_clock: slot_clock.clone(), diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 6692fe3a7b6..f6d1cc8af5d 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -53,22 +53,6 @@ const RETRY_DELAY: Duration = Duration::from_secs(2); /// The time between polls when waiting for genesis. const WAITING_FOR_GENESIS_POLL_TIME: Duration = Duration::from_secs(12); -/// Specific timeout constants for HTTP requests involved in different validator duties. -/// This can help ensure that proper endpoint fallback occurs. -const HTTP_ATTESTATION_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT: u32 = 24; -const HTTP_LIVENESS_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_PROPOSAL_TIMEOUT_QUOTIENT: u32 = 2; -const HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT: u32 = 4; -const HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT: u32 = 4; -const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; -const HTTP_DEFAULT_TIMEOUT_QUOTIENT: u32 = 4; - const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; type ValidatorStore = LighthouseValidatorStore; @@ -291,24 +275,7 @@ impl ProductionValidatorClient { // Use quicker timeouts if a fallback beacon node exists. let timeouts = if i < last_beacon_node_index && !config.use_long_timeouts { info!("Fallback endpoints are available, using optimized timeouts."); - Timeouts { - attestation: slot_duration / HTTP_ATTESTATION_TIMEOUT_QUOTIENT, - attester_duties: slot_duration / HTTP_ATTESTER_DUTIES_TIMEOUT_QUOTIENT, - attestation_subscriptions: slot_duration - / HTTP_ATTESTATION_SUBSCRIPTIONS_TIMEOUT_QUOTIENT, - liveness: slot_duration / HTTP_LIVENESS_TIMEOUT_QUOTIENT, - proposal: slot_duration / HTTP_PROPOSAL_TIMEOUT_QUOTIENT, - proposer_duties: slot_duration / HTTP_PROPOSER_DUTIES_TIMEOUT_QUOTIENT, - sync_committee_contribution: slot_duration - / HTTP_SYNC_COMMITTEE_CONTRIBUTION_TIMEOUT_QUOTIENT, - sync_duties: slot_duration / HTTP_SYNC_DUTIES_TIMEOUT_QUOTIENT, - get_beacon_blocks_ssz: slot_duration - / HTTP_GET_BEACON_BLOCK_SSZ_TIMEOUT_QUOTIENT, - get_debug_beacon_states: slot_duration / HTTP_GET_DEBUG_BEACON_STATE_QUOTIENT, - get_deposit_snapshot: slot_duration / HTTP_GET_DEPOSIT_SNAPSHOT_QUOTIENT, - get_validator_block: slot_duration / HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT, - default: slot_duration / HTTP_DEFAULT_TIMEOUT_QUOTIENT, - } + Timeouts::use_optimized_timeouts(slot_duration) } else { Timeouts::set_all(slot_duration.saturating_mul(config.long_timeouts_multiplier)) };