feat: state cache for ePBS

packages/beacon-node/src/api/impl/beacon/state/utils.ts

@@ -1,6 +1,6 @@
 import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {routes} from "@lodestar/api";
-import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
+import {CheckpointWithPayload, IForkChoice} from "@lodestar/fork-choice";
 import {GENESIS_SLOT} from "@lodestar/params";
 import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {BLSPubkey, Epoch, RootHex, Slot, ValidatorIndex, getValidatorStatus, phase0} from "@lodestar/types";
@@ -11,7 +11,7 @@ import {ApiError, ValidationError} from "../../errors.js";
 export function resolveStateId(
   forkChoice: IForkChoice,
   stateId: routes.beacon.StateId
-): RootHex | Slot | CheckpointWithHex {
+): RootHex | Slot | CheckpointWithPayload {
   if (stateId === "head") {
     return forkChoice.getHead().stateRoot;
   }

packages/beacon-node/src/api/impl/validator/index.ts

@@ -1,7 +1,7 @@
 import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {routes} from "@lodestar/api";
 import {ApplicationMethods} from "@lodestar/api/server";
-import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {ExecutionStatus, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {
   ForkName,
   ForkPostBellatrix,
@@ -71,7 +71,7 @@ import {ChainEvent, CommonBlockBody} from "../../../chain/index.js";
 import {PREPARE_NEXT_SLOT_BPS} from "../../../chain/prepareNextSlot.js";
 import {BlockType, ProduceFullDeneb} from "../../../chain/produceBlock/index.js";
 import {RegenCaller} from "../../../chain/regen/index.js";
-import {CheckpointHex} from "../../../chain/stateCache/types.js";
+import {CheckpointHexPayload} from "../../../chain/stateCache/types.js";
 import {validateApiAggregateAndProof} from "../../../chain/validation/index.js";
 import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import {ZERO_HASH} from "../../../constants/index.js";
@@ -300,7 +300,7 @@ export function getValidatorApi(
    *                    |
    *              prepareNextSlot (4s before next slot)
    */
-  async function waitForCheckpointState(cpHex: CheckpointHex): Promise<CachedBeaconStateAllForks | null> {
+  async function waitForCheckpointState(cpHex: CheckpointHexPayload): Promise<CachedBeaconStateAllForks | null> {
     const cpState = chain.regen.getCheckpointStateSync(cpHex);
     if (cpState) {
       return cpState;
@@ -1024,7 +1024,11 @@ export function getValidatorApi(
       // this is to avoid missed block proposal due to 0 epoch look ahead
       if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) {
         // wait for maximum 1 slot for cp state which is the timeout of validator api
-        const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch});
+        const cpState = await waitForCheckpointState({
+          rootHex: head.blockRoot,
+          epoch,
+          payloadPresent: head.payloadStatus === PayloadStatus.FULL,
+        });
         if (cpState) {
           state = cpState;
           metrics?.duties.requestNextEpochProposalDutiesHit.inc();

packages/beacon-node/src/chain/archiveStore/archiveStore.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithPayload} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {Checkpoint} from "@lodestar/types/phase0";
 import {callFnWhenAwait} from "@lodestar/utils";
@@ -41,7 +41,7 @@ export enum ArchiveStoreTask {
  */
 export class ArchiveStore {
   private archiveMode: ArchiveMode;
-  private jobQueue: JobItemQueue<[CheckpointWithHex], void>;
+  private jobQueue: JobItemQueue<[CheckpointWithPayload], void>;
 
   private archiveDataEpochs?: number;
   private readonly statesArchiverStrategy: StateArchiveStrategy;
@@ -64,7 +64,7 @@ export class ArchiveStore {
     this.archiveMode = opts.archiveMode;
     this.archiveDataEpochs = opts.archiveDataEpochs;
 
-    this.jobQueue = new JobItemQueue<[CheckpointWithHex], void>(this.processFinalizedCheckpoint, {
+    this.jobQueue = new JobItemQueue<[CheckpointWithPayload], void>(this.processFinalizedCheckpoint, {
       maxLength: PROCESS_FINALIZED_CHECKPOINT_QUEUE_LENGTH,
       signal,
     });
@@ -165,7 +165,7 @@ export class ArchiveStore {
   //-------------------------------------------------------------------------
   // Event handlers
   //-------------------------------------------------------------------------
-  private onFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise<void> => {
+  private onFinalizedCheckpoint = async (finalized: CheckpointWithPayload): Promise<void> => {
     return this.jobQueue.push(finalized);
   };
 
@@ -182,7 +182,7 @@ export class ArchiveStore {
     });
   };
 
-  private processFinalizedCheckpoint = async (finalized: CheckpointWithHex): Promise<void> => {
+  private processFinalizedCheckpoint = async (finalized: CheckpointWithPayload): Promise<void> => {
     try {
       const finalizedEpoch = finalized.epoch;
       this.logger.verbose("Start processing finalized checkpoint", {epoch: finalizedEpoch, rootHex: finalized.rootHex});

packages/beacon-node/src/chain/archiveStore/interface.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithPayload} from "@lodestar/fork-choice";
 import {RootHex} from "@lodestar/types";
 import {Metrics} from "../../metrics/metrics.js";
 
@@ -44,9 +44,9 @@ export type FinalizedStats = {
 
 export interface StateArchiveStrategy {
   onCheckpoint(stateRoot: RootHex, metrics?: Metrics | null): Promise<void>;
-  onFinalizedCheckpoint(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-  maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
-  archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void>;
+  onFinalizedCheckpoint(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise<void>;
+  maybeArchiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise<void>;
+  archiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise<void>;
 }
 
 export interface IArchiveStore {

packages/beacon-node/src/chain/archiveStore/strategies/frequencyStateArchiveStrategy.ts

@@ -1,4 +1,4 @@
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithPayload} from "@lodestar/fork-choice";
 import {SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, RootHex, Slot} from "@lodestar/types";
@@ -9,6 +9,7 @@ import {AllocSource, BufferPool} from "../../../util/bufferPool.js";
 import {getStateSlotFromBytes} from "../../../util/multifork.js";
 import {IStateRegenerator} from "../../regen/interface.js";
 import {serializeState} from "../../serializeState.js";
+import {fcCheckpointToHexPayload} from "../../stateCache/persistentCheckpointsCache.js";
 import {StateArchiveStrategy, StatesArchiveOpts} from "../interface.js";
 
 /**
@@ -40,7 +41,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
     private readonly bufferPool?: BufferPool | null
   ) {}
 
-  async onFinalizedCheckpoint(_finalized: CheckpointWithHex, _metrics?: Metrics | null): Promise<void> {}
+  async onFinalizedCheckpoint(_finalized: CheckpointWithPayload, _metrics?: Metrics | null): Promise<void> {}
   async onCheckpoint(_stateRoot: RootHex, _metrics?: Metrics | null): Promise<void> {}
 
   /**
@@ -55,7 +56,7 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
    * epoch - 1024*2    epoch - 1024    epoch - 32    epoch
    * ```
    */
-  async maybeArchiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
+  async maybeArchiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise<void> {
     let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
     const lastStoredSlot = await this.db.stateArchive.lastKey();
     timer?.({step: FrequencyStateArchiveStep.LoadLastStoredSlot});
@@ -104,10 +105,12 @@ export class FrequencyStateArchiveStrategy implements StateArchiveStrategy {
    * Archives finalized states from active bucket to archive bucket.
    * Only the new finalized state is stored to disk
    */
-  async archiveState(finalized: CheckpointWithHex, metrics?: Metrics | null): Promise<void> {
+  async archiveState(finalized: CheckpointWithPayload, metrics?: Metrics | null): Promise<void> {
     // starting from Mar 2024, the finalized state could be from disk or in memory
     let timer = metrics?.processFinalizedCheckpoint.frequencyStateArchive.startTimer();
-    const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized);
+    // Convert fork-choice checkpoint to beacon-node checkpoint with payloadPresent
+    const finalizedHexPayload = fcCheckpointToHexPayload(finalized);
+    const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalizedHexPayload);
     timer?.({step: FrequencyStateArchiveStep.GetFinalizedState});
 
     const {rootHex} = finalized;

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -30,7 +30,7 @@ import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
-import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
+import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
@@ -111,6 +111,11 @@ export async function importBlock(
 
   // This adds the state necessary to process the next block
   // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
+  // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true (execution payload embedded in block)
+  // Post-Gloas: blockSummary.payloadStatus is always PENDING (EMPTY variant also created), payloadPresent = false (block state only, no payload processing yet)
+  const isGloasBlock = blockSummary.blockHashFromBid !== null;
+  const payloadPresent = !isGloasBlock;
+  // processState manages both block state and payload state variants together for memory/disk management
   this.regen.processState(blockRootHex, postState);
 
   this.metrics?.importBlock.bySource.inc({source: source.source});
@@ -425,12 +430,14 @@ export async function importBlock(
     // Cache state to preserve epoch transition work
     const checkpointState = postState;
     const cp = getCheckpointFromState(checkpointState);
-    this.regen.addCheckpointState(cp, checkpointState);
+    // Pre-Gloas: payloadPresent = true (FULL variant, execution payload embedded in block)
+    // Post-Gloas: payloadPresent = false (PENDING variant with EMPTY also created, block state only)
+    this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
     // consumers should not mutate state ever
     this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
 
     // Note: in-lined code from previos handler of ChainEvent.checkpoint
-    this.logger.verbose("Checkpoint processed", toCheckpointHex(cp));
+    this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent));
 
     const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length;
     this.metrics?.currentActiveValidators.set(activeValidatorsCount);
@@ -448,7 +455,7 @@ export async function importBlock(
       const justifiedEpoch = justifiedCheckpoint.epoch;
       const preJustifiedEpoch = parentBlockSummary.justifiedEpoch;
       if (justifiedEpoch > preJustifiedEpoch) {
-        this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint));
+        this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent));
         this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
         this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
       }
@@ -462,7 +469,7 @@ export async function importBlock(
           state: toRootHex(checkpointState.hashTreeRoot()),
           executionOptimistic: false,
         });
-        this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
+        this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent));
         this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
       }
     }

packages/beacon-node/src/chain/chain.ts

@@ -3,7 +3,7 @@ import {PrivateKey} from "@libp2p/interface";
 import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
-import {CheckpointWithHex, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
+import {CheckpointWithPayload, IForkChoice, ProtoBlock, UpdateHeadOpt} from "@lodestar/fork-choice";
 import {LoggerNode} from "@lodestar/logger/node";
 import {EFFECTIVE_BALANCE_INCREMENT, GENESIS_SLOT, SLOTS_PER_EPOCH, isForkPostElectra} from "@lodestar/params";
 import {
@@ -111,7 +111,7 @@ import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datasto
 import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
 import {CPStateDatastore} from "./stateCache/datastore/types.js";
 import {FIFOBlockStateCache} from "./stateCache/fifoBlockStateCache.js";
-import {PersistentCheckpointStateCache} from "./stateCache/persistentCheckpointsCache.js";
+import {PersistentCheckpointStateCache, fcCheckpointToHexPayload} from "./stateCache/persistentCheckpointsCache.js";
 import {CheckpointStateCache} from "./stateCache/types.js";
 import {ValidatorMonitor} from "./validatorMonitor.js";
 
@@ -357,7 +357,11 @@ export class BeaconChain implements IBeaconChain {
     const {checkpoint} = computeAnchorCheckpoint(config, anchorState);
     blockStateCache.add(anchorState);
     blockStateCache.setHeadState(anchorState);
-    checkpointStateCache.add(checkpoint, anchorState);
+    // TODO: For Gloas, determine if anchor state is block state or payload state
+    // Pre-Gloas: payloadPresent is always true (execution payload embedded in block)
+    // Post-Gloas: Could be either - depends on whether anchor was loaded with payload processing
+    // For now, assume true
+    checkpointStateCache.add(checkpoint, anchorState, true);
 
     const forkChoice = initializeForkChoice(
       config,
@@ -637,10 +641,11 @@ export class BeaconChain implements IBeaconChain {
   }
 
   getStateByCheckpoint(
-    checkpoint: CheckpointWithHex
+    checkpoint: CheckpointWithPayload
   ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null {
     // finalized or justified checkpoint states maynot be available with PersistentCheckpointStateCache, use getCheckpointStateOrBytes() api to get Uint8Array
-    const cachedStateCtx = this.regen.getCheckpointStateSync(checkpoint);
+    const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
+    const cachedStateCtx = this.regen.getCheckpointStateSync(checkpointHexPayload);
     if (cachedStateCtx) {
       const block = this.forkChoice.getBlockDefaultStatus(cachedStateCtx.latestBlockHeader.hashTreeRoot());
       const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
@@ -655,9 +660,10 @@ export class BeaconChain implements IBeaconChain {
   }
 
   async getStateOrBytesByCheckpoint(
-    checkpoint: CheckpointWithHex
+    checkpoint: CheckpointWithPayload
   ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> {
-    const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpoint);
+    const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
+    const cachedStateCtx = await this.regen.getCheckpointStateOrBytes(checkpointHexPayload);
     if (cachedStateCtx) {
       const block = this.forkChoice.getBlockDefaultStatus(checkpoint.root);
       const finalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
@@ -1158,7 +1164,7 @@ export class BeaconChain implements IBeaconChain {
    * @param blockState state that declares justified checkpoint `checkpoint`
    */
   private justifiedBalancesGetter(
-    checkpoint: CheckpointWithHex,
+    checkpoint: CheckpointWithPayload,
     blockState: CachedBeaconStateAllForks
   ): EffectiveBalanceIncrements {
     this.metrics?.balancesCache.requests.inc();
@@ -1197,10 +1203,11 @@ export class BeaconChain implements IBeaconChain {
    * @param blockState state that declares justified checkpoint `checkpoint`
    */
   private closestJustifiedBalancesStateToCheckpoint(
-    checkpoint: CheckpointWithHex,
+    checkpoint: CheckpointWithPayload,
     blockState: CachedBeaconStateAllForks
   ): {state: CachedBeaconStateAllForks; stateId: string; shouldWarn: boolean} {
-    const state = this.regen.getCheckpointStateSync(checkpoint);
+    const checkpointHexPayload = fcCheckpointToHexPayload(checkpoint);
+    const state = this.regen.getCheckpointStateSync(checkpointHexPayload);
     if (state) {
       return {state, stateId: "checkpoint_state", shouldWarn: false};
     }
@@ -1334,7 +1341,7 @@ export class BeaconChain implements IBeaconChain {
     this.seenContributionAndProof.prune(head.slot);
   }
 
-  private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithHex): void {
+  private onForkChoiceJustified(this: BeaconChain, cp: CheckpointWithPayload): void {
     this.logger.verbose("Fork choice justified", {epoch: cp.epoch, root: cp.rootHex});
   }
 
@@ -1345,7 +1352,7 @@ export class BeaconChain implements IBeaconChain {
     });
   }
 
-  private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithHex): Promise<void> {
+  private async onForkChoiceFinalized(this: BeaconChain, cp: CheckpointWithPayload): Promise<void> {
     this.logger.verbose("Fork choice finalized", {epoch: cp.epoch, root: cp.rootHex});
     const finalizedSlot = computeStartSlotAtEpoch(cp.epoch);
     this.seenBlockProposers.prune(finalizedSlot);
@@ -1387,7 +1394,7 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
-  private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise<void> {
+  private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithPayload): Promise<void> {
     if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) {
       // Custody requirements can only be increased, we can disable dynamic custody updates
       // if the node already maintains custody of all custody groups in case it is configured

packages/beacon-node/src/chain/interface.ts

@@ -1,7 +1,7 @@
 import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz";
 import {BeaconConfig} from "@lodestar/config";
-import {CheckpointWithHex, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {CheckpointWithHex, CheckpointWithPayload, IForkChoice, ProtoBlock} from "@lodestar/fork-choice";
 import {
   BeaconStateAllForks,
   CachedBeaconStateAllForks,
@@ -199,7 +199,7 @@ export interface IBeaconChain {
   ): {state: BeaconStateAllForks; executionOptimistic: boolean; finalized: boolean} | null;
   /** Return state bytes by checkpoint */
   getStateOrBytesByCheckpoint(
-    checkpoint: CheckpointWithHex
+    checkpoint: CheckpointWithPayload
   ): Promise<{state: CachedBeaconStateAllForks | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null>;
 
   /**