diff --git a/cmd/dependabot/internal/cmd/root.go b/cmd/dependabot/internal/cmd/root.go index 2b29c26e..3243c58c 100644 --- a/cmd/dependabot/internal/cmd/root.go +++ b/cmd/dependabot/internal/cmd/root.go @@ -31,6 +31,7 @@ var ( updaterImage string proxyImage string collectorImage string + storageImage string ) // rootCmd represents the base command when called without any subcommands @@ -59,4 +60,5 @@ func init() { rootCmd.PersistentFlags().StringVar(&updaterImage, "updater-image", "", "container image to use for the updater") rootCmd.PersistentFlags().StringVar(&proxyImage, "proxy-image", infra.ProxyImageName, "container image to use for the proxy") rootCmd.PersistentFlags().StringVar(&collectorImage, "collector-image", infra.CollectorImageName, "container image to use for the OpenTelemetry collector") + rootCmd.PersistentFlags().StringVar(&storageImage, "storage-image", infra.StorageImageName, "container image to use for the storage service") } diff --git a/cmd/dependabot/internal/cmd/test.go b/cmd/dependabot/internal/cmd/test.go index b1a2d236..387488d7 100644 --- a/cmd/dependabot/internal/cmd/test.go +++ b/cmd/dependabot/internal/cmd/test.go @@ -50,6 +50,7 @@ func NewTestCommand() *cobra.Command { ProxyCertPath: flags.proxyCertPath, ProxyImage: proxyImage, PullImages: flags.pullImages, + StorageImage: storageImage, Timeout: flags.timeout, UpdaterImage: updaterImage, Volumes: flags.volumes, diff --git a/cmd/dependabot/internal/cmd/update.go b/cmd/dependabot/internal/cmd/update.go index fd2151ea..d3aa11b8 100644 --- a/cmd/dependabot/internal/cmd/update.go +++ b/cmd/dependabot/internal/cmd/update.go @@ -96,6 +96,7 @@ func NewUpdateCommand() *cobra.Command { ProxyCertPath: flags.proxyCertPath, ProxyImage: proxyImage, PullImages: flags.pullImages, + StorageImage: storageImage, Timeout: flags.timeout, UpdaterImage: updaterImage, Volumes: flags.volumes, diff --git a/internal/infra/run.go b/internal/infra/run.go index 910248fc..1157fb5c 100644 --- a/internal/infra/run.go +++ b/internal/infra/run.go @@ -68,6 +68,8 @@ type RunParams struct { CollectorImage string // CollectorConfigPath is the path to the OpenTelemetry collector configuration file CollectorConfigPath string + // StorageImage is the image to use for the storage service + StorageImage string // Writer is where API calls will be written to Writer io.Writer InputName string @@ -369,6 +371,13 @@ func runContainers(ctx context.Context, params RunParams) (err error) { if err != nil { return err } + + if params.Job.UseCaseInsensitiveFileSystem() { + err = pullImage(ctx, cli, params.StorageImage) + if err != nil { + return err + } + } } networks, err := NewNetworks(ctx, cli) @@ -416,13 +425,18 @@ func runContainers(ctx context.Context, params RunParams) (err error) { // put the clone dir in the updater container to be used by during the update if params.LocalDir != "" { - if err = putCloneDir(ctx, cli, updater, params.LocalDir); err != nil { + containerDir := guestRepoDir + if params.Job.UseCaseInsensitiveFileSystem() { + // since the updater is using the storage container, we need to populate the repo on that device because that's the directory that will be used for the update + containerDir = caseSensitiveRepoContentsPath + } + if err = putCloneDir(ctx, cli, updater, params.LocalDir, containerDir); err != nil { return err } } if params.Debug { - if err := updater.RunShell(ctx, prox.url, params.ApiUrl); err != nil { + if err := updater.RunShell(ctx, prox.url, params.ApiUrl, params.Job); err != nil { return err } } else { @@ -432,7 +446,7 @@ func runContainers(ctx context.Context, params RunParams) (err error) { } // Then run the dependabot commands as the dependabot user - env := userEnv(prox.url, params.ApiUrl) + env := userEnv(prox.url, params.ApiUrl, params.Job) if params.Flamegraph { env = append(env, "FLAMEGRAPH=1") } @@ -473,33 +487,33 @@ func getFromContainer(ctx context.Context, cli *client.Client, containerID, srcP } } -func putCloneDir(ctx context.Context, cli *client.Client, updater *Updater, dir string) error { +func putCloneDir(ctx context.Context, cli *client.Client, updater *Updater, localDir, containerDir string) error { // Docker won't create the directory, so we have to do it first. - const cmd = "mkdir -p " + guestRepoDir + cmd := fmt.Sprintf("mkdir -p %s", containerDir) err := updater.RunCmd(ctx, cmd, dependabot) if err != nil { return fmt.Errorf("failed to create clone dir: %w", err) } - r, err := archive.TarWithOptions(dir, &archive.TarOptions{}) + r, err := archive.TarWithOptions(localDir, &archive.TarOptions{}) if err != nil { return fmt.Errorf("failed to tar clone dir: %w", err) } opt := container.CopyToContainerOptions{} - err = cli.CopyToContainer(ctx, updater.containerID, guestRepoDir, r, opt) + err = cli.CopyToContainer(ctx, updater.containerID, containerDir, r, opt) if err != nil { return fmt.Errorf("failed to copy clone dir to container: %w", err) } - err = updater.RunCmd(ctx, "chown -R dependabot "+guestRepoDir, root) + err = updater.RunCmd(ctx, "chown -R dependabot "+containerDir, root) if err != nil { return fmt.Errorf("failed to initialize clone dir: %w", err) } // The directory needs to be a git repo, so we need to initialize it. commands := []string{ - "cd " + guestRepoDir, + "cd " + containerDir, "git config --global init.defaultBranch main", "git init", "git config user.email 'dependabot@github.com'", diff --git a/internal/infra/updater.go b/internal/infra/updater.go index 7c9a5b52..8259b702 100644 --- a/internal/infra/updater.go +++ b/internal/infra/updater.go @@ -7,16 +7,20 @@ import ( "encoding/json" "fmt" "io" + "log" "os" "path" "path/filepath" "strings" + "time" "github.com/dependabot/cli/internal/model" "github.com/docker/cli/cli/streams" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" "github.com/goware/prefixer" "github.com/moby/moby/pkg/stdcopy" @@ -32,11 +36,23 @@ const ( guestInputDir = "/home/dependabot/dependabot-updater/job.json" guestOutput = "/home/dependabot/dependabot-updater/output.json" guestRepoDir = "/home/dependabot/dependabot-updater/repo" + + caseSensitiveContainerRoot = "/dpdbot" + caseSensitiveRepoContentsPath = "/dpdbot/repo" + + caseInsensitiveContainerRoot = "/nocase" + caseInsensitiveRepoContentsPath = "/nocase/repo" + + StorageImageName = "ghcr.io/dependabot/dependabot-storage" + storageUser = "dpduser" + storagePass = "dpdpass" ) type Updater struct { - cli *client.Client - containerID string + cli *client.Client + containerID string + storageContainerID string + storageVolumes []string // ExitCode is set once an Updater command has completed. ExitCode *int @@ -82,6 +98,16 @@ func NewUpdater(ctx context.Context, cli *client.Client, net *Networks, params * ReadOnly: readOnly, }) } + + storageContainerID := "" + storageVolumes := []string{} + if params.Job.UseCaseInsensitiveFileSystem() { + storageContainerID, storageVolumes, err = createStorageVolumes(hostCfg, ctx, cli, net, params.StorageImage) + if err != nil { + return nil, fmt.Errorf("failed to create storage volumes: %w", err) + } + } + netCfg := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ net.noInternetName: { @@ -96,8 +122,10 @@ func NewUpdater(ctx context.Context, cli *client.Client, net *Networks, params * } updater := &Updater{ - cli: cli, - containerID: updaterContainer.ID, + cli: cli, + containerID: updaterContainer.ID, + storageContainerID: storageContainerID, + storageVolumes: storageVolumes, } if err = putUpdaterInputs(ctx, cli, prox.ca.Cert, updaterContainer.ID, params.Job); err != nil { @@ -113,6 +141,128 @@ func NewUpdater(ctx context.Context, cli *client.Client, net *Networks, params * return updater, nil } +func createStorageVolumes(hostCfg *container.HostConfig, ctx context.Context, cli *client.Client, net *Networks, storageImageName string) (storageContainerID string, volumeNames []string, err error) { + log.Printf("Preparing case insensitive filesystem") + + // create container hosting the storage + storageContainerCfg := &container.Config{ + User: root, + Image: storageImageName, + Tty: true, // prevent container from stopping + } + storageHostCfg := &container.HostConfig{} + storageNetCfg := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + net.noInternetName: { + NetworkID: net.NoInternet.ID, // no external access for this container + }, + }, + } + storageContainer, err := cli.ContainerCreate(ctx, storageContainerCfg, storageHostCfg, storageNetCfg, nil, "") + if err != nil { + err = fmt.Errorf("failed to create storage container: %w", err) + return + } + storageContainerID = storageContainer.ID + caseSensitiveVolumeName := "dpdbot-storage-" + storageContainer.ID[:12] + caseInsensitiveVolumeName := "dpdbot-nocase-" + storageContainer.ID[:12] + volumeNames = []string{caseSensitiveVolumeName, caseInsensitiveVolumeName} + + defer func() { + if err != nil { + removeStorageVolume(cli, ctx, caseSensitiveVolumeName) + removeStorageVolume(cli, ctx, caseInsensitiveVolumeName) + } + }() + + // start storage container + if err = cli.ContainerStart(ctx, storageContainer.ID, container.StartOptions{}); err != nil { + err = fmt.Errorf("failed to start storage container: %w", err) + return + } + + // wait for port 445 to be listening on the storage container + log.Printf(" waiting for storage container port 445 to be ready") + err = waitForPort(ctx, cli, storageContainer.ID, 445) + if err != nil { + err = fmt.Errorf("failed to wait for storage container port 445: %w", err) + return + } + + // add volume mounts from the storage container; container IP is needed because the host is making a direct connection and it has not been given internet access + inspect, err := cli.ContainerInspect(ctx, storageContainerID) + if err != nil { + err = fmt.Errorf("failed to inspect storage container: %w", err) + return + } + storageContainerAddress := inspect.NetworkSettings.Networks[net.noInternetName].IPAddress + addStorageMounts(hostCfg, storageContainerAddress, caseSensitiveVolumeName, caseSensitiveContainerRoot, caseInsensitiveVolumeName, caseInsensitiveContainerRoot) + return +} + +func removeStorageVolume(cli *client.Client, ctx context.Context, name string) error { + listOptions := volume.ListOptions{ + Filters: filters.NewArgs( + filters.KeyValuePair{Key: "name", Value: name}, + ), + } + ls, err := cli.VolumeList(ctx, listOptions) + if err != nil { + return err + } + + for _, v := range ls.Volumes { + if v.Name == name { + err = cli.VolumeRemove(ctx, v.Name, true) + if err != nil { + return err + } + } + } + + return nil +} + +func addStorageMounts(hostCfg *container.HostConfig, storageContainerAddress string, caseSensitiveVolumeName, caseSensitiveContainerRoot, caseInsensitiveVolumeName, caseInsensitiveContainerRoot string) { + const cifsVolumeType = "cifs" + localShareName := fmt.Sprintf("//%s/dpdbot", storageContainerAddress) + connectionOptions := fmt.Sprintf("username=%s,password=%s,uid=1000,gid=1000", storageUser, storagePass) + + // create case-sensitive layer + hostCfg.Mounts = append(hostCfg.Mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: caseSensitiveVolumeName, + Target: caseSensitiveContainerRoot, + VolumeOptions: &mount.VolumeOptions{ + DriverConfig: &mount.Driver{ + Name: "local", + Options: map[string]string{ + "type": cifsVolumeType, + "device": localShareName, + "o": connectionOptions, + }, + }, + }, + }) + + // create case-insensitive layer + hostCfg.Mounts = append(hostCfg.Mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: caseInsensitiveVolumeName, + Target: caseInsensitiveContainerRoot, + VolumeOptions: &mount.VolumeOptions{ + DriverConfig: &mount.Driver{ + Name: "local", + Options: map[string]string{ + "type": cifsVolumeType, + "device": localShareName, + "o": fmt.Sprintf("nocase,%s", connectionOptions), + }, + }, + }, + }) +} + func putUpdaterInputs(ctx context.Context, cli *client.Client, cert, id string, job *model.Job) error { opt := container.CopyToContainerOptions{} if t, err := tarball(dbotCert, cert); err != nil { @@ -155,8 +305,8 @@ func mountOptions(v string) (local, remote string, readOnly bool, err error) { return local, remote, readOnly, nil } -func userEnv(proxyURL string, apiUrl string) []string { - return []string{ +func userEnv(proxyURL string, apiUrl string, job *model.Job) []string { + envVars := []string{ "GITHUB_ACTIONS=true", // sets exit code when fetch fails fmt.Sprintf("http_proxy=%s", proxyURL), fmt.Sprintf("HTTP_PROXY=%s", proxyURL), @@ -166,23 +316,31 @@ func userEnv(proxyURL string, apiUrl string) []string { fmt.Sprintf("DEPENDABOT_JOB_TOKEN=%v", ""), fmt.Sprintf("DEPENDABOT_JOB_PATH=%v", guestInputDir), fmt.Sprintf("DEPENDABOT_OUTPUT_PATH=%v", guestOutput), - fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%v", guestRepoDir), fmt.Sprintf("DEPENDABOT_API_URL=%s", apiUrl), fmt.Sprintf("SSL_CERT_FILE=%v/ca-certificates.crt", certsPath), "UPDATER_ONE_CONTAINER=true", "UPDATER_DETERMINISTIC=true", } + + if job.UseCaseInsensitiveFileSystem() { + envVars = append(envVars, fmt.Sprintf("DEPENDABOT_CASE_INSENSITIVE_REPO_CONTENTS_PATH=%s", caseInsensitiveRepoContentsPath)) + envVars = append(envVars, fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%s", caseSensitiveRepoContentsPath)) + } else { + envVars = append(envVars, fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%s", guestRepoDir)) + } + + return envVars } // RunShell executes an interactive shell, blocks until complete. -func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiUrl string) error { +func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiUrl string, job *model.Job) error { execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, container.ExecOptions{ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, User: dependabot, - Env: append(userEnv(proxyURL, apiUrl), "DEBUG=1"), + Env: append(userEnv(proxyURL, apiUrl, job), "DEBUG=1"), Cmd: []string{"/bin/bash", "-c", "update-ca-certificates && /bin/bash"}, }) if err != nil { @@ -298,6 +456,20 @@ func (u *Updater) Close() (err error) { if removeErr != nil { err = fmt.Errorf("failed to remove proxy container: %w", removeErr) } + + for _, v := range u.storageVolumes { + removeErr = u.cli.VolumeRemove(context.Background(), v, true) + if removeErr != nil { + err = fmt.Errorf("failed to remove storage volume %s: %w", v, removeErr) + } + } + + if u.storageContainerID != "" { + removeErr = u.cli.ContainerRemove(context.Background(), u.storageContainerID, container.RemoveOptions{Force: true}) + if removeErr != nil { + err = fmt.Errorf("failed to remove storage container: %w", removeErr) + } + } }() // Handle non-zero exit codes. @@ -360,3 +532,51 @@ func firstNonEmpty(values ...string) string { return "" } + +func waitForPort(ctx context.Context, cli *client.Client, containerID string, port int) error { + const maxAttempts = 5 + const sleepDuration = time.Second + + // check /proc/net/tcp for the requested port; n.b., it is hex encoded and 4 characters wide + testCmd := fmt.Sprintf("test -f /proc/net/tcp && grep ' *\\d+: [A-F0-9]{8}:%04X ' /proc/net/tcp >/dev/null 2>&1", port) + + for i := range maxAttempts { + execCreate, err := cli.ContainerExecCreate(ctx, containerID, container.ExecOptions{ + AttachStdout: false, + AttachStderr: false, + User: root, + Cmd: []string{"/bin/sh", "-c", testCmd}, + }) + if err != nil { + return fmt.Errorf("failed to create exec for port check: %w", err) + } + + execResp, err := cli.ContainerExecAttach(ctx, execCreate.ID, container.ExecAttachOptions{}) + if err != nil { + return fmt.Errorf("failed to attach to exec for port check: %w", err) + } + + // wait for completion and check the exit code + execResp.Close() + execInspect, err := cli.ContainerExecInspect(ctx, execCreate.ID) + if err != nil { + return fmt.Errorf("failed to inspect exec: %w", err) + } + + if execInspect.ExitCode == 0 { + // port is listening + log.Printf(" port %d is listening after %d attempts", port, i+1) + + // in a few instances, the port is open but the service isn't yet ready for connections + // no more reliable method has been found, other than a short delay + time.Sleep(sleepDuration) + return nil + } + + if i < maxAttempts-1 { + time.Sleep(sleepDuration) + } + } + + return fmt.Errorf("port %d is not listening after %d attempts", port, maxAttempts) +} diff --git a/internal/model/job.go b/internal/model/job.go index 103b01f7..bfe85f95 100644 --- a/internal/model/job.go +++ b/internal/model/job.go @@ -54,6 +54,14 @@ type Job struct { UpdateCooldown *UpdateCooldown `json:"cooldown,omitempty" yaml:"cooldown,omitempty"` } +func (j *Job) UseCaseInsensitiveFileSystem() bool { + if experimentValue, isBoolean := j.Experiments["use_case_insensitive_filesystem"].(bool); isBoolean && experimentValue { + return true + } + + return false +} + // Source is a reference to some source code type Source struct { Provider string `json:"provider" yaml:"provider,omitempty"` diff --git a/testdata/scripts/smb-mount.txt b/testdata/scripts/smb-mount.txt new file mode 100644 index 00000000..afdf1f7a --- /dev/null +++ b/testdata/scripts/smb-mount.txt @@ -0,0 +1,49 @@ +# Build the dummy Dockerfile +exec docker build -qt dummy-nocase-updater . + +# Run the dependabot command +dependabot update -f job.yml --updater-image dummy-nocase-updater + +# assert the dummy is working +stderr 'case-insensitive storage is working' + +exec docker rmi -f dummy-nocase-updater + +-- Dockerfile -- +FROM ubuntu:22.04 + +RUN useradd dependabot + +COPY --chown=dependabot --chmod=755 update-ca-certificates /usr/bin/update-ca-certificates +COPY --chown=dependabot --chmod=755 run bin/run + +-- update-ca-certificates -- +#!/usr/bin/env bash + +echo "Updated those certificates for ya" + +-- run -- +#!/usr/bin/env bash + +if [ "$1" = "fetch_files" ]; then + # git clone would have created this directory + mkdir -p "$DEPENDABOT_REPO_CONTENTS_PATH" + exit 0 +fi + +echo "test file" > "$DEPENDABOT_REPO_CONTENTS_PATH/test.txt" +if [ -e "$DEPENDABOT_CASE_INSENSITIVE_REPO_CONTENTS_PATH/TEST.TXT" ]; then + echo "case-insensitive storage is working" +else + echo "case-insensitive storage is not working" +fi + +-- job.yml -- +job: + experiments: + use_case_insensitive_filesystem: true + source: + provider: github + repo: test/repo + directory: / + package-manager: nuget