add plumbing for graph smoke tests (#505)

* fix graph command not working without experiment enabled

* oopsy, no space in the json

* add plumbing for graph smoke tests

Author: jakecoffman

cmd/dependabot/internal/cmd/graph.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/MakeNowJust/heredoc"
 	"github.com/dependabot/cli/internal/infra"
+	"github.com/dependabot/cli/internal/model"
 	"github.com/spf13/cobra"
 )
 
@@ -66,7 +67,7 @@ func NewGraphCommand() *cobra.Command {
 			}
 
 			if err := infra.Run(infra.RunParams{
-				Command:             infra.UpdateGraphCommand,
+				Command:             model.UpdateGraphCommand,
 				CacheDir:            flags.cache,
 				CollectorConfigPath: flags.collectorConfigPath,
 				CollectorImage:      collectorImage,

cmd/dependabot/internal/cmd/test.go

@@ -35,6 +35,7 @@ func NewTestCommand() *cobra.Command {
 			processInput(&smokeTest.Input, nil)
 
 			if err := executeTestJob(infra.RunParams{
+				Command:             smokeTest.Command,
 				CacheDir:            flags.cache,
 				CollectorConfigPath: flags.collectorConfigPath,
 				CollectorImage:      collectorImage,

internal/infra/run.go

@@ -30,23 +30,16 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-type RunCommand int
-
-const (
-	UpdateFilesCommand RunCommand = iota
-	UpdateGraphCommand
-)
-
-var runCmds = map[RunCommand]string{
-	UpdateFilesCommand: "bin/run fetch_files && bin/run update_files",
-	UpdateGraphCommand: "bin/run fetch_files && bin/run update_graph",
+var runCmds = map[model.RunCommand]string{
+	model.UpdateFilesCommand: "bin/run fetch_files && bin/run update_files",
+	model.UpdateGraphCommand: "bin/run fetch_files && bin/run update_graph",
 }
 
 type RunParams struct {
 	// Input file
 	Input string
 	// Which command to use, this will default to UpdateFilesCommand
-	Command RunCommand
+	Command model.RunCommand
 	// job definition passed to the updater
 	Job *model.Job
 	// expectations asserted at the end of a test
@@ -102,6 +95,10 @@ func (p *RunParams) Validate() error {
 	if p.Job.Source.Commit != "" && !gitShaRegex.MatchString(p.Job.Source.Commit) {
 		return fmt.Errorf("commit must be a SHA, or not provided")
 	}
+	// Allows for older smoke tests without the command field to keep working.
+	if p.Command == "" {
+		p.Command = model.UpdateFilesCommand
+	}
 	return nil
 }
 
@@ -177,6 +174,7 @@ func generateOutput(params RunParams, api *server.API, outFile *os.File) ([]byte
 		// store the SHA we worked with for reproducible tests
 		params.Job.Source.Commit = api.Actual.Input.Job.Source.Commit
 	}
+	api.Actual.Command = params.Command
 	api.Actual.Input.Job = *params.Job
 
 	// ignore conditions help make tests reproducible, so they are generated if there aren't any yet

internal/model/smoke.go

@@ -1,7 +1,16 @@
 package model
 
+type RunCommand string
+
+const (
+	UpdateFilesCommand RunCommand = "update"
+	UpdateGraphCommand RunCommand = "graph"
+)
+
 // SmokeTest is a way to test a job by asserting the outputs.
 type SmokeTest struct {
+	// Command is the command to run for testing a smoke test (update, graph)
+	Command RunCommand `yaml:"command"`
 	// Input is the input parameters
 	Input Input `yaml:"input"`
 	// Output is the list of expected outputs

internal/model/update.go

@@ -35,7 +35,6 @@ type DependencySubmissionRequest struct {
 	Ref       string         `json:"ref" yaml:"ref"`
 	Job       map[string]any `json:"job" yaml:"job"`
 	Detector  map[string]any `json:"detector" yaml:"detector"`
-	Scanned   string         `json:"scanned" yaml:"scanned"`
 	Manifests map[string]any `json:"manifests" yaml:"manifests"`
 }
 

internal/server/api.go

@@ -282,13 +282,13 @@ func decode[T any](data []byte) (T, error) {
 	return wrapper.Data, nil
 }
 
-// TODO(brrygrdn): Add model.DependencySubmissionRequest to support smoke tests
-//
 // The test command only expects to run with `update` operations right now so
 // we will need to incorporate which run command is expected as well, but we
 // don't need regression coverage yet.
 func compare(expect, actual *model.UpdateWrapper) error {
 	switch v := expect.Data.(type) {
+	case model.DependencySubmissionRequest:
+		return compareDependencySubmissionRequest(v, actual.Data.(model.DependencySubmissionRequest))
 	case model.UpdateDependencyList:
 		return compareUpdateDependencyList(v, actual.Data.(model.UpdateDependencyList))
 	case model.CreatePullRequest:
@@ -314,6 +314,13 @@ func unexpectedBody(kind string) error {
 	return fmt.Errorf("unexpected body for %s", kind)
 }
 
+func compareDependencySubmissionRequest(expect, actual model.DependencySubmissionRequest) error {
+	if reflect.DeepEqual(expect, actual) {
+		return nil
+	}
+	return unexpectedBody("dependency_submission_request")
+}
+
 func compareUpdateDependencyList(expect, actual model.UpdateDependencyList) error {
 	if reflect.DeepEqual(expect, actual) {
 		return nil

testdata/scripts/graph.txt

@@ -1,14 +1,20 @@
 # Build the dummy Dockerfile
 exec docker build -qt graph-updater .
 
-# Run the dependabot command
-dependabot graph go_modules dependabot/cli --updater-image graph-updater
+# Run the graph command and create a smoke test
+dependabot graph go_modules dependabot/cli -o out.yml --updater-image graph-updater
 
 # assert the dummy is working
 stderr 'bin/run arguments: fetch_files'
 stderr 'bin/run arguments: update_graph'
 stderr '"enable_dependency_submission_poc":true'
 
+# Verify the smoke test can run the graph command
+dependabot test -f out.yml --updater-image graph-updater
+
+stderr 'bin/run arguments: fetch_files'
+stderr 'bin/run arguments: update_graph'
+
 exec docker rmi -f graph-updater
 
 -- Dockerfile --