From 7ad3fb6beb6fe9b7b4977cb6fca541ca36924d39 Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 3 Dec 2024 16:52:02 +1100 Subject: [PATCH 1/8] Add endpoint docs --- book/src/api-vc-endpoints.md | 46 ++++++++++++++++++++++++++++ book/src/redundancy.md | 18 +++++++++-- validator_client/http_api/src/lib.rs | 10 +++--- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/book/src/api-vc-endpoints.md b/book/src/api-vc-endpoints.md index 98605a3dcd0..fd6db813012 100644 --- a/book/src/api-vc-endpoints.md +++ b/book/src/api-vc-endpoints.md @@ -18,6 +18,7 @@ | [`POST /lighthouse/validators/mnemonic`](#post-lighthousevalidatorsmnemonic) | Create a new validator from an existing mnemonic. | | [`POST /lighthouse/validators/web3signer`](#post-lighthousevalidatorsweb3signer) | Add web3signer validators. | | [`GET /lighthouse/logs`](#get-lighthouselogs) | Get logs | +| [`GET /lighthouse/beacon/health`](#get-lighthousebeaconhealth) | Get health information for each connected beacon node. | The query to Lighthouse API endpoints requires authorization, see [Authorization Header](./api-vc-auth-header.md). @@ -816,3 +817,48 @@ logs emitted are INFO level or higher. } } ``` + +## `GET /lighthouse/beacon/health` + +Provides information about the sync status and execution layer health of each connected beacon node. +For more inormation about how to interpret the beacon node health, see [Fallback Health](./redundancy.md#fallback-health). + +### HTTP Specification + +| Property | Specification | +|-------------------|--------------------------------------------| +| Path | `/lighthouse/beacon/health` | +| Method | GET | +| Required Headers | [`Authorization`](./api-vc-auth-header.md) | +| Typical Responses | 200, 400 | + +### Example Response Body + +```json +{ + "data": { + "beacon_nodes": [ + { + "index": 0, + "endpoint": "http://localhost:5052", + "health": { + "user_index": 0, + "head": 10500000, + "optimistic_status": "No", + "execution_status": "Healthy", + "health_tier": { + "tier": 1, + "sync_distance": 0, + "distance_tier": "Synced" + } + } + }, + { + "index": 1, + "endpoint": "http://fallbacks-r.us", + "health": "Offline" + } + ] + } +} +``` diff --git a/book/src/redundancy.md b/book/src/redundancy.md index daf0eb4a5b4..2868ad688f6 100644 --- a/book/src/redundancy.md +++ b/book/src/redundancy.md @@ -39,9 +39,6 @@ There are a few interesting properties about the list of `--beacon-nodes`: earlier in the list. - *Synced is preferred*: the validator client prefers a synced beacon node over one that is still syncing. -- *Failure is sticky*: if a beacon node fails, it will be flagged as offline - and won't be retried again for the rest of the slot (12 seconds). This helps prevent the impact - of time-outs and other lengthy errors. > Note: When supplying multiple beacon nodes the `http://localhost:5052` address must be explicitly > provided (if it is desired). It will only be used as default if no `--beacon-nodes` flag is @@ -76,6 +73,21 @@ Prior to v3.2.0 fallback beacon nodes also required the `--subscribe-all-subnets now broadcast subscriptions to all connected beacon nodes by default. This broadcast behaviour can be disabled using the `--broadcast none` flag for `lighthouse vc`. +### Fallback Health +Since v6.0.0, the validator client will be more aggressive in switching to a fallback node. To do this, +it uses the concept of "Health". Every slot, the validator client checks each connected beacon node +to determine which node is the "Healthiest". In general, the validator client will prefer nodes +which are synced, have synced execution layers and which are not currently optimisitically +syncing. + +Sync distance is separated out into 4 tiers: "Synced", "Small", "Medium", "Large". Nodes are then +sorted into tiers based onto sync distance and execution layer status. You can use the +`--beacon-nodes-sync-tolerances` to change how many slots wide each tier is. In the case where +multiple nodes fall into the same tier, user order is used to tie-break. + +To see health information for each connected node, you can use the +[`/lighthouse/beacon/health` API endpoint](./api-vc-endpoints.md#get-lighthousebeaconhealth). + ### Broadcast modes Since v4.6.0, the Lighthouse VC can be configured to broadcast messages to all configured beacon diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index 73ebe717af3..ffc0d7d429e 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -425,10 +425,10 @@ pub fn serve( }, ); - // GET lighthouse/ui/fallback_health - let get_lighthouse_ui_fallback_health = warp::path("lighthouse") - .and(warp::path("ui")) - .and(warp::path("fallback_health")) + // GET lighthouse/beacon/health + let get_lighthouse_beacon_health = warp::path("lighthouse") + .and(warp::path("beacon")) + .and(warp::path("health")) .and(warp::path::end()) .and(block_service_filter.clone()) .then(|block_filter: BlockService| async move { @@ -1314,7 +1314,7 @@ pub fn serve( .or(get_lighthouse_validators_pubkey) .or(get_lighthouse_ui_health) .or(get_lighthouse_ui_graffiti) - .or(get_lighthouse_ui_fallback_health) + .or(get_lighthouse_beacon_health) .or(get_fee_recipient) .or(get_gas_limit) .or(get_graffiti) From dcbccf97d7e081ec18b7f22f54d691d3c8e2eef3 Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 3 Dec 2024 19:14:18 +1100 Subject: [PATCH 2/8] Linting --- book/src/redundancy.md | 1 + 1 file changed, 1 insertion(+) diff --git a/book/src/redundancy.md b/book/src/redundancy.md index 2868ad688f6..7b886344fc5 100644 --- a/book/src/redundancy.md +++ b/book/src/redundancy.md @@ -74,6 +74,7 @@ now broadcast subscriptions to all connected beacon nodes by default. This broad can be disabled using the `--broadcast none` flag for `lighthouse vc`. ### Fallback Health + Since v6.0.0, the validator client will be more aggressive in switching to a fallback node. To do this, it uses the concept of "Health". Every slot, the validator client checks each connected beacon node to determine which node is the "Healthiest". In general, the validator client will prefer nodes From 9e2de0f67efe16ed707a559bf76680c79c953bfa Mon Sep 17 00:00:00 2001 From: Mac L Date: Wed, 4 Dec 2024 20:56:37 +1100 Subject: [PATCH 3/8] Improve docs --- book/src/api-vc-endpoints.md | 7 +++++++ book/src/redundancy.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/book/src/api-vc-endpoints.md b/book/src/api-vc-endpoints.md index fd6db813012..255ddd9ccab 100644 --- a/book/src/api-vc-endpoints.md +++ b/book/src/api-vc-endpoints.md @@ -832,6 +832,13 @@ For more inormation about how to interpret the beacon node health, see [Fallback | Required Headers | [`Authorization`](./api-vc-auth-header.md) | | Typical Responses | 200, 400 | +Command: +```bash +DATADIR=/var/lib/lighthouse +curl -X GET http://localhost:5062/lighthouse/beacon/health \ + -H "Authorization: Bearer $(cat ${DATADIR}/validators/api-token.txt)" | jq + ``` + ### Example Response Body ```json diff --git a/book/src/redundancy.md b/book/src/redundancy.md index 7b886344fc5..3b488137231 100644 --- a/book/src/redundancy.md +++ b/book/src/redundancy.md @@ -77,7 +77,7 @@ can be disabled using the `--broadcast none` flag for `lighthouse vc`. Since v6.0.0, the validator client will be more aggressive in switching to a fallback node. To do this, it uses the concept of "Health". Every slot, the validator client checks each connected beacon node -to determine which node is the "Healthiest". In general, the validator client will prefer nodes +to determine which node is the "Healthiest". In general, the validator client will prefer nodes which are synced, have synced execution layers and which are not currently optimisitically syncing. From d72fa41b9953ab93b40b345eef4743178449210e Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 27 Mar 2025 19:48:29 +1100 Subject: [PATCH 4/8] Fix markdown --- book/src/api_vc_endpoints.md | 1 + 1 file changed, 1 insertion(+) diff --git a/book/src/api_vc_endpoints.md b/book/src/api_vc_endpoints.md index dbe88c1db53..569f7f93183 100644 --- a/book/src/api_vc_endpoints.md +++ b/book/src/api_vc_endpoints.md @@ -833,6 +833,7 @@ For more inormation about how to interpret the beacon node health, see [Fallback | Typical Responses | 200, 400 | Command: + ```bash DATADIR=/var/lib/lighthouse curl -X GET http://localhost:5062/lighthouse/beacon/health \ From bdc16e633da10a6ece1951acafa63461dc65c92d Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 27 Mar 2025 20:05:45 +1100 Subject: [PATCH 5/8] Fix broken links --- book/src/advanced_redundancy.md | 2 +- book/src/api_vc_endpoints.md | 4 ++-- book/src/archived_key_management.md | 1 + book/src/archived_merge_migration.md | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 book/src/archived_key_management.md create mode 100644 book/src/archived_merge_migration.md diff --git a/book/src/advanced_redundancy.md b/book/src/advanced_redundancy.md index 11373baaa8d..e6449843411 100644 --- a/book/src/advanced_redundancy.md +++ b/book/src/advanced_redundancy.md @@ -87,7 +87,7 @@ sorted into tiers based onto sync distance and execution layer status. You can u multiple nodes fall into the same tier, user order is used to tie-break. To see health information for each connected node, you can use the -[`/lighthouse/beacon/health` API endpoint](./api-vc-endpoints.md#get-lighthousebeaconhealth). +[`/lighthouse/beacon/health` API endpoint](./api_vc_endpoints.md#get-lighthousebeaconhealth). ### Broadcast modes diff --git a/book/src/api_vc_endpoints.md b/book/src/api_vc_endpoints.md index 569f7f93183..e510271f60a 100644 --- a/book/src/api_vc_endpoints.md +++ b/book/src/api_vc_endpoints.md @@ -821,7 +821,7 @@ logs emitted are INFO level or higher. ## `GET /lighthouse/beacon/health` Provides information about the sync status and execution layer health of each connected beacon node. -For more inormation about how to interpret the beacon node health, see [Fallback Health](./redundancy.md#fallback-health). +For more inormation about how to interpret the beacon node health, see [Fallback Health](./advanced_redundancy.md#fallback-health). ### HTTP Specification @@ -829,7 +829,7 @@ For more inormation about how to interpret the beacon node health, see [Fallback |-------------------|--------------------------------------------| | Path | `/lighthouse/beacon/health` | | Method | GET | -| Required Headers | [`Authorization`](./api-vc-auth-header.md) | +| Required Headers | [`Authorization`](./api_vc_auth_header.md) | | Typical Responses | 200, 400 | Command: diff --git a/book/src/archived_key_management.md b/book/src/archived_key_management.md new file mode 100644 index 00000000000..7ec26c0bab4 --- /dev/null +++ b/book/src/archived_key_management.md @@ -0,0 +1 @@ +# Key Management diff --git a/book/src/archived_merge_migration.md b/book/src/archived_merge_migration.md new file mode 100644 index 00000000000..afb48aeec57 --- /dev/null +++ b/book/src/archived_merge_migration.md @@ -0,0 +1 @@ +# Merge Migration From cb6881dc1ee98d8439298444e3a331cb632865cd Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 27 Mar 2025 23:05:46 +1100 Subject: [PATCH 6/8] Spellcheck --- book/src/api_vc_endpoints.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/src/api_vc_endpoints.md b/book/src/api_vc_endpoints.md index e510271f60a..e51f5d29ae1 100644 --- a/book/src/api_vc_endpoints.md +++ b/book/src/api_vc_endpoints.md @@ -821,7 +821,7 @@ logs emitted are INFO level or higher. ## `GET /lighthouse/beacon/health` Provides information about the sync status and execution layer health of each connected beacon node. -For more inormation about how to interpret the beacon node health, see [Fallback Health](./advanced_redundancy.md#fallback-health). +For more information about how to interpret the beacon node health, see [Fallback Health](./advanced_redundancy.md#fallback-health). ### HTTP Specification From 7ba3a4a9fd7a03c3092be5e95dfe754d70e1ce41 Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 27 Mar 2025 23:12:50 +1100 Subject: [PATCH 7/8] Remove old files --- book/src/archived_key_management.md | 1 - book/src/archived_merge_migration.md | 1 - 2 files changed, 2 deletions(-) delete mode 100644 book/src/archived_key_management.md delete mode 100644 book/src/archived_merge_migration.md diff --git a/book/src/archived_key_management.md b/book/src/archived_key_management.md deleted file mode 100644 index 7ec26c0bab4..00000000000 --- a/book/src/archived_key_management.md +++ /dev/null @@ -1 +0,0 @@ -# Key Management diff --git a/book/src/archived_merge_migration.md b/book/src/archived_merge_migration.md deleted file mode 100644 index afb48aeec57..00000000000 --- a/book/src/archived_merge_migration.md +++ /dev/null @@ -1 +0,0 @@ -# Merge Migration From 291091930f306da2c5ec49251bd682b2bee53a6d Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 27 Mar 2025 23:33:09 +1100 Subject: [PATCH 8/8] Spellcheck --- book/src/advanced_redundancy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/src/advanced_redundancy.md b/book/src/advanced_redundancy.md index e6449843411..4c231ed6ab3 100644 --- a/book/src/advanced_redundancy.md +++ b/book/src/advanced_redundancy.md @@ -78,7 +78,7 @@ can be disabled using the `--broadcast none` flag for `lighthouse vc`. Since v6.0.0, the validator client will be more aggressive in switching to a fallback node. To do this, it uses the concept of "Health". Every slot, the validator client checks each connected beacon node to determine which node is the "Healthiest". In general, the validator client will prefer nodes -which are synced, have synced execution layers and which are not currently optimisitically +which are synced, have synced execution layers and which are not currently optimistically syncing. Sync distance is separated out into 4 tiers: "Synced", "Small", "Medium", "Large". Nodes are then