Have nightly tests against nightly generated artifacts

.github/workflows/nightly-tests.yml

@@ -0,0 +1,38 @@
+name: Nightly mainnet tests
+
+defaults:
+  run:
+    shell: zsh {0}
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: The branch, tag or SHA to checkout and build from
+        default: unstable
+        type: string
+        required: true
+  schedule:
+    # Every day at 00:00 UTC
+    - cron: '0 0 * * *'
+
+jobs:
+  ef-nightly-test:
+    name: ef-tests-ubuntu
+    needs: [check-labels]
+    if: needs.check-labels.outputs.skip_ci != 'true'
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest'  }}
+    env:
+      LIGHTHOUSE_GITHUB_TOKEN: ${{ secrets.LIGHTHOUSE_GITHUB_TOKEN }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+    - name: Run consensus-spec-tests with blst and fake_crypto
+      run: make run-ef-nightly-tests

Makefile

@@ -144,6 +144,14 @@ run-ef-tests:
 	cargo test --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
 	./$(EF_TESTS)/check_all_files_accessed.py $(EF_TESTS)/.accessed_file_log.txt $(EF_TESTS)/consensus-spec-tests
 
+# Downloads the nighlty mainnet artifacts
+run-ef-nightly-tests:
+	rm -rf $(EF_TESTS)/.accessed_file_log.txt
+	make -C $(EF_TESTS) nightly
+	cargo test --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES)"
+	cargo test --release -p ef_tests --features "ef_tests,$(EF_TEST_FEATURES),fake_crypto"
+	./$(EF_TESTS_NIGHTLY)/check_all_files_accessed.py $(EF_TESTS_NIGHTLY)/.accessed_file_log.txt $(EF_TESTS_NIGHTLY)/consensus-spec-tests
+
 # Runs EF test vectors with nextest
 nextest-run-ef-tests:
 	rm -rf $(EF_TESTS)/.accessed_file_log.txt

testing/ef_tests/Makefile

@@ -5,13 +5,15 @@ TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS))
 REPO_NAME := consensus-spec-tests
 OUTPUT_DIR := ./$(REPO_NAME)
 BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(TESTS_TAG)
-
 BLS_TEST_REPO_NAME := bls12-381-tests
 BLS_TEST_TAG := v0.1.1
 BLS_TEST = bls_tests_yaml
 BLS_TARBALL = $(patsubst %,%-$(BLS_TEST_TAG).tar.gz,$(BLS_TEST))
 BLS_OUTPUT_DIR := $(OUTPUT_DIR)/$(BLS_TEST_REPO_NAME)
+
 BLS_BASE_URL := https://github.com/ethereum/$(BLS_TEST_REPO_NAME)/releases/download/$(BLS_TEST_TAG)
+WORKFLOW_FILE := generate_vectors.yml
+
 
 CURL := $(if $(LIGHTHOUSE_GITHUB_TOKEN),curl -L --header "Authorization: $(LIGHTHOUSE_GITHUB_TOKEN)",curl -L)
 
@@ -41,4 +43,32 @@ clean-archives:
 
 clean: clean-test-files clean-archives
 
-.PHONY: clean clean-archives clean-test-files
+nightly:
+	rm -rf $(OUTPUT_DIR)
+	mkdir -p $(OUTPUT_DIR)
+	make $(BLS_OUTPUT_DIR)
+	RUN_ID="$$( \
+		$(CURL) \
+		"https://api.github.com/repos/ethereum/consensus-specs/actions/workflows/$(WORKFLOW_FILE)/runs?per_page=1" \
+		| jq -r '.workflow_runs[0].id' \
+	)"; \
+	$(CURL) \
+		"https://api.github.com/repos/ethereum/consensus-specs/actions/runs/$$RUN_ID/artifacts" \
+		| jq -r '.artifacts[] | select(.name!="consensustestgen.log") | "\(.id) \(.name)"' \
+		| while read -r ART_ID ART_NAME; do \
+			curl -sSL -L \
+				$(if $(LIGHTHOUSE_GITHUB_TOKEN),-H "Authorization: Bearer $(LIGHTHOUSE_GITHUB_TOKEN)") \
+				-o "$(OUTPUT_DIR)/$$ART_NAME.zip" \
+				"https://api.github.com/repos/ethereum/consensus-specs/actions/artifacts/$$ART_ID/zip"; \
+			unzip -q -o "$(OUTPUT_DIR)/$$ART_NAME.zip" -d "$(OUTPUT_DIR)/"; \
+			rm -rf "$(OUTPUT_DIR)/$$ART_NAME.zip"; \
+		done; \
+	find $(OUTPUT_DIR) -type f -name '*.tar.gz' -print0 | \
+	while IFS= read -r -d '' archive; do \
+		dest=$$(dirname "$$archive"); \
+		tar -xzf "$$archive" -C "$$dest"; \
+		rm -f "$$archive"; \
+	done;
+
+
+.PHONY: clean clean-archives clean-test-files nightly