[CI] Add chaotic devnet workflow

.ci/devnet_ci.sh → .ci/test_devnet.sh

@@ -96,7 +96,8 @@ for client_index in $(seq 0 $((total_clients-1))); do
 done
 
 # Ensure all nodes are up and running.
-wait_for_nodes "$total_validators" "$total_clients" "$network_name"
+# Wait up to two minutes, as this can take long in CI.
+wait_for_nodes "$total_validators" "$total_clients" "$network_name" 180
 
 # Wait for validators to be fully connected.
 log "ℹ️ Waiting for validators to be fully connected..." 
@@ -351,25 +352,11 @@ fi
 log "ℹ️Testing network progress"
 
 # Check heights periodically with a timeout
-SECONDS=0
-while (( SECONDS < 600 )); do  # 10 minutes max
-  if check_heights 0 $((total_validators+total_clients)) "$min_height" "$network_name" "$SECONDS"; then
-    log "🎉 Test passed! All nodes reached minimum height."
-
-    if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings"; then
-      exit 0
-    else
-      exit 1
-    fi
-  fi
-
-  # Continue waiting
-  sleep 30
-  log "Waited $SECONDS seconds so far..."
-done
-
-log "❌ Test failed! Not all nodes reached minimum height within 10 minutes."
-log_validator_logs "$log_dir" "$total_validators" "$total_clients"
-log_client_logs "$log_dir" "$total_validators" "$total_clients"
-
-exit 1
+if wait_for_heights 0 $((total_validators+total_clients)) "$min_height" "$network_name" 600; then
+  log "🎉 Test passed! All nodes reached minimum height."
+else
+  log "❌ Test failed! Not all nodes reached minimum height within 10 minutes."
+  log_validator_logs "$log_dir" "$total_validators" "$total_clients"
+  log_client_logs "$log_dir" "$total_validators" "$total_clients"
+  exit 1
+fi

.ci/test_reset_majority.sh

@@ -0,0 +1,109 @@
+#! /usr/bin/env bash
+
+set -eo pipefail # error on any command failure
+
+# Ensure that we run a recent version of bash
+if [ "${BASH_VERSINFO[0]}" -lt 5 ]; then
+  echo "Error: This script requires bash version 5.0 or higher."
+  exit 1
+fi
+
+#shellcheck source=SCRIPTDIR/utils.sh
+. ./.ci/utils.sh
+
+# Set up the logging directory
+init_log_dir
+
+# Network parameters
+total_validators=$1
+network_id=$2
+reset_interval=$3
+final_height=$4
+num_resets=$5
+
+# Default values if not provided
+: "${total_validators:=7}"
+: "${network_id:=0}"
+: "${reset_interval:=10}"
+: "${final_height:=20}"
+: "${num_resets:=3}"
+
+max_faulty=$(( (total_validators - 1) / 3 ))
+# AleoBFT needs at least N-f for a quorum, not 2*f+1.
+majority=$((total_validators - max_faulty))
+network_name=$(get_network_name "$network_id")
+
+# Keep verbosity low as we are running many nodes.
+verbosity=0
+
+# The time that is used to determine the total timeout for the test.
+# This is higher than the one for the minority test to account for the higher number of nodes that are reset.
+max_wait_per_block=20
+
+# Define a trap handler that cleans up all processes on exit.
+trap stop_nodes EXIT
+
+# Define a trap handler that prints a message when an error occurs 
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+
+# Define flags used by all nodes.
+common_flags=(
+  --nodisplay --nobanner --noupdater "--network=$network_id" "--verbosity=$verbosity"
+  "--dev-num-validators=$total_validators"
+)
+
+# Start all validator nodes in the background
+for validator_index in $(seq 0 $((total_validators-1))); do
+  snarkos clean "--dev=$validator_index" "--network=$network_id"
+
+  snarkos start "${common_flags[@]}" "--dev=$validator_index" --validator --logfile="$log_dir/validator-$validator_index.log" &
+  PIDS[validator_index]=$!
+  log "Started validator $validator_index with PID ${PIDS[$validator_index]}"
+  # Add 1-second delay between starting nodes to avoid hitting rate limits
+  sleep 1
+done
+
+wait_for_nodes "$total_validators" 0 "$network_name" 180
+
+# Wait longer if there are more blocks to reach.
+max_wait=$((final_height * max_wait_per_block))
+
+for iter in $(seq 1 "$num_resets"); do
+  reset_height=$(( iter * reset_interval ));
+
+  # Block until the reset height is reached.
+  if ! wait_for_heights 0 "$total_validators" "$reset_height" "$network_name" $((reset_interval * max_wait_per_block)); then
+    log "❌ Test failed! Not all nodes reached reset height of $reset_height within $((reset_interval * max_wait_per_block)) seconds."
+    exit 1
+  fi
+  log "All nodes reached the next reset height."
+
+  # Gracefully shut down a majority of the validators
+  mapfile -t target_indices < <(generate_random_indices "$majority" $(( ${#PIDS[@]} - 1 )))
+  stop_some_nodes "${target_indices[@]}"
+
+  for target_index in "${target_indices[@]}"; do
+    # Remove the original ledger
+    snarkos clean "--network=$network_id" "--dev=$target_index"
+  done
+
+  # wait for a non-trivial amount of time
+  sleep 30
+
+  for target_index in "${target_indices[@]}"; do
+    # Restart
+    snarkos start "${common_flags[@]}" "--dev=$target_index" --validator --logfile="$log_dir/validator-$target_index.log" &
+    PIDS[target_index]=$!
+    log "Restarted a fresh validator $target_index with PID ${PIDS[$target_index]}"
+    # Add 1-second delay between starting nodes to avoid hitting rate limits
+    sleep 1
+  done
+done
+
+if wait_for_heights 0 "$total_validators" "$final_height" "$network_name" "$max_wait"; then
+  log "SUCCESS!"
+  exit 0
+else
+  log "❌ Test failed! Not all nodes reached final height of $final_height within $max_wait seconds."
+  exit 1
+fi

.ci/test_reset_minority.sh

@@ -0,0 +1,102 @@
+#! /usr/bin/env bash
+
+set -eo pipefail # error on any command failure
+
+# Ensure that we run a recent version of bash
+if [ "${BASH_VERSINFO[0]}" -lt 5 ]; then
+  echo "Error: This script requires bash version 5.0 or higher."
+  exit 1
+fi
+
+#shellcheck source=SCRIPTDIR/utils.sh
+. ./.ci/utils.sh
+
+# Set up the logging directory
+init_log_dir
+
+# Network parameters
+total_validators=$1
+network_id=$2
+reset_interval=$3
+final_height=$4
+num_resets=$5
+
+# Default values if not provided
+: "${total_validators:=7}"
+: "${network_id:=0}"
+: "${reset_interval:=20}"
+: "${final_height:=100}"
+: "${num_resets:=3}"
+
+minority=$(( (total_validators - 1) / 3 ))
+network_name=$(get_network_name "$network_id")
+verbosity=0
+
+# The time that is used to determine the total timeout for the test
+max_wait_per_block=10
+
+# Define a trap handler that cleans up all processes on exit.
+trap stop_nodes EXIT
+
+# Define a trap handler that prints a message when an error occurs 
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+
+# Define flags used by all nodes.
+common_flags=(
+  --nodisplay --nobanner --noupdater "--network=$network_id" "--verbosity=$verbosity"
+  "--dev-num-validators=$total_validators"
+)
+
+# Start all validator nodes in the background
+for validator_index in $(seq 0 $((total_validators-1))); do
+  snarkos clean "--dev=$validator_index" "--network=$network_id"
+
+  snarkos start "${common_flags[@]}" "--dev=$validator_index" --validator --logfile="$log_dir/validator-$validator_index.log" & 
+  PIDS[validator_index]=$!
+
+  log "Started validator $validator_index with PID ${PIDS[$validator_index]}"
+  # Add 1-second delay between starting nodes to avoid hitting rate limits
+  sleep 1
+done
+
+wait_for_nodes "$total_validators" 0 "$network_name" 180
+
+# Wait longer if there are more blocks to reach.
+max_wait=$((final_height * max_wait_per_block));
+
+SECONDS=0
+for iter in $(seq 1 "$num_resets"); do
+  reset_height=$(( iter * reset_interval ));
+
+  # Wait until all nodes reach the reset height.
+  if ! wait_for_heights 0 "$total_validators" "$reset_height" "$network_name" $((reset_interval * max_wait_per_block)); then
+    log "❌ Test failed! Not all nodes reached reset height of $reset_height within $((reset_interval * max_wait_per_block)) seconds."
+    exit 1
+  fi
+  log "All nodes reached reset height."
+
+  # Gracefully shut down a minority of the validators
+  mapfile -t target_indices < <(generate_random_indices "$minority" $(( ${#PIDS[@]} - 1 )))
+  stop_some_nodes "${target_indices[@]}"
+
+  for target_index in "${target_indices[@]}"; do
+    # Remove the original ledger
+    snarkos clean "--network=$network_id" "--dev=$target_index"
+    # Wait until the cleanup concludes
+    sleep 1
+    # Restart
+    snarkos start "${common_flags[@]}" "--dev=$target_index" --validator --logfile="$log_dir/validator-$target_index.log" &
+    PIDS[target_index]=$!
+    log "Restarted a fresh validator $target_index with PID ${PIDS[$target_index]}"
+    # Add 1-second delay between starting nodes to avoid hitting rate limits
+    sleep 1
+  done
+done
+
+if wait_for_heights 0 "$total_validators" "$final_height" "$network_name" "$max_wait"; then
+  log "SUCCESS!"
+  exit 0
+else
+  log "❌ Test failed! Not all nodes reached final height of $final_height within $max_wait seconds."
+  exit 1
+fi

.ci/test_restart_all.sh

@@ -0,0 +1,100 @@
+#! /usr/bin/env bash
+
+set -eo pipefail # error on any command failure
+
+# Ensure that we run a recent version of bash
+if [ "${BASH_VERSINFO[0]}" -lt 5 ]; then
+  echo "Error: This script requires bash version 5.0 or higher."
+  exit 1
+fi
+
+#shellcheck source=SCRIPTDIR/utils.sh
+. ./.ci/utils.sh
+
+# Set up the logging directory
+init_log_dir
+
+# Network parameters
+total_validators=$1
+network_id=$2
+restart_interval=$3
+final_height=$4
+num_restarts=$5
+
+# Default values if not provided.
+: "${total_validators:=7}"
+: "${network_id:=0}"
+: "${restart_interval:=10}"
+: "${final_height:=100}"
+: "${num_restarts:=3}"
+
+# The time that is used to determine the total timeout for the test.
+max_wait_per_block=10
+
+network_name=$(get_network_name "$network_id")
+
+# Keep verbosity low as we are running many nodes.
+verbosity=0
+
+# Define a trap handler that cleans up all processes on exit.
+trap stop_nodes EXIT
+
+# Define a trap handler that prints a message when an error occurs 
+trap 'log "⛔️ Error in $BASH_SOURCE at line $LINENO: \"$BASH_COMMAND\" failed (exit $?)"' ERR
+
+# Define flags used by all nodes.
+common_flags=(
+  --nodisplay --nobanner --noupdater "--network=$network_id" "--verbosity=$verbosity"
+  "--dev-num-validators=$total_validators"
+)
+
+# Start all validator nodes in the background
+for validator_index in $(seq 0 $((total_validators-1))); do
+  snarkos clean "--dev=$validator_index" "--network=$network_id"
+
+  snarkos start "${common_flags[@]}" "--dev=$validator_index" --validator --logfile="$log_dir/validator-$validator_index.log" &
+  PIDS[validator_index]=$!
+  log "Started validator $validator_index with PID ${PIDS[$validator_index]}"
+  # Add 1-second delay between starting nodes to avoid hitting rate limits
+  sleep 1
+done
+
+wait_for_nodes "$total_validators" 0 "$network_name" 180
+
+# Wait longer if there are more blocks to reach.
+max_wait=$((final_height * max_wait_per_block))
+
+for iter in $(seq 1 "$num_restarts"); do
+  restart_height=$(( iter * restart_interval ))
+
+  # Wait until all nodes reach the restart height.
+  if ! wait_for_heights 0 "$total_validators" "$restart_height" "$network_name" $((restart_interval * max_wait_per_block)); then
+    log "❌ Test failed! Not all nodes reached restart height of $restart_height within $((restart_interval * max_wait_per_block)) seconds."
+    exit 1
+  fi
+  log "All nodes reached restart height $restart_height. Restarting all validators (iteration $iter/$num_restarts)..."
+
+  # Gracefully shut down all validators
+  stop_nodes
+
+  # Wait briefly before restarting
+  sleep 5
+
+  # Restart all validators without cleaning their ledger
+  for validator_index in $(seq 0 $((total_validators-1))); do
+    snarkos start "${common_flags[@]}" "--dev=$validator_index" --validator --logfile="$log_dir/validator-$validator_index.log" &
+    PIDS[validator_index]=$!
+    log "Restarted validator $validator_index with PID ${PIDS[$validator_index]}"
+    # Add 1-second delay between starting nodes to avoid hitting rate limits
+    sleep 1
+  done
+done
+
+# Wait for final height
+if wait_for_heights 0 "$total_validators" "$final_height" "$network_name"; then
+ log "SUCCESS!"
+  exit 0
+else
+  log "❌ Test failed! Not all nodes reached final height of $final_height within $max_wait seconds."
+  exit 1
+fi