diff --git a/client/allocrunner/taskrunner/secrets/plugin_provider.go b/client/allocrunner/taskrunner/secrets/plugin_provider.go index c18c1fd7bbd..43ca56838e2 100644 --- a/client/allocrunner/taskrunner/secrets/plugin_provider.go +++ b/client/allocrunner/taskrunner/secrets/plugin_provider.go @@ -23,6 +23,9 @@ type ExternalPluginProvider struct { // path is the secret location used in Fetch path string + + // env is the set of environment variables passed into plugin + env map[string]string } type Response struct { @@ -30,17 +33,24 @@ type Response struct { Error *string `json:"error,omitempty"` } -func NewExternalPluginProvider(plugin commonplugins.SecretsPlugin, pluginName, secretName, path string) *ExternalPluginProvider { +func NewExternalPluginProvider(plugin commonplugins.SecretsPlugin, pluginName, secretName, path string, env map[string]string) *ExternalPluginProvider { return &ExternalPluginProvider{ plugin: plugin, pluginName: pluginName, secretName: secretName, path: path, + env: env, + } +} + +func (p *ExternalPluginProvider) InterpolateEnv(interpolate func(string) string) { + for key, value := range p.env { + p.env[key] = interpolate(value) } } func (p *ExternalPluginProvider) Fetch(ctx context.Context) (map[string]string, error) { - resp, err := p.plugin.Fetch(ctx, p.path) + resp, err := p.plugin.Fetch(ctx, p.path, p.env) if err != nil { return nil, fmt.Errorf("failed executing plugin %q for secret %q: %w", p.pluginName, p.secretName, err) } diff --git a/client/allocrunner/taskrunner/secrets/plugin_provider_test.go b/client/allocrunner/taskrunner/secrets/plugin_provider_test.go index 1dbd41a7c45..4ab67eeab82 100644 --- a/client/allocrunner/taskrunner/secrets/plugin_provider_test.go +++ b/client/allocrunner/taskrunner/secrets/plugin_provider_test.go @@ -21,8 +21,8 @@ func (m *MockSecretPlugin) Fingerprint(ctx context.Context) (*commonplugins.Plug return nil, nil } -func (m *MockSecretPlugin) Fetch(ctx context.Context, path string) (*commonplugins.SecretResponse, error) { - args := m.Called() +func (m *MockSecretPlugin) Fetch(ctx context.Context, path string, env map[string]string) (*commonplugins.SecretResponse, error) { + args := m.Called(ctx, path, env) if args.Get(0) == nil { return nil, args.Error(1) @@ -40,9 +40,9 @@ func (m *MockSecretPlugin) Parse() (map[string]string, error) { func TestExternalPluginProvider_Fetch(t *testing.T) { t.Run("errors if fetch errors", func(t *testing.T) { mockSecretPlugin := new(MockSecretPlugin) - mockSecretPlugin.On("Fetch", mock.Anything).Return(nil, errors.New("something bad")) + mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("something bad")) - testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test") + testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil) vars, err := testProvider.Fetch(t.Context()) must.ErrorContains(t, err, "something bad") @@ -52,12 +52,12 @@ func TestExternalPluginProvider_Fetch(t *testing.T) { t.Run("errors if fetch response contains error", func(t *testing.T) { mockSecretPlugin := new(MockSecretPlugin) testError := "something bad" - mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{ + mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(&commonplugins.SecretResponse{ Result: nil, Error: &testError, }, nil) - testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test") + testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil) vars, err := testProvider.Fetch(t.Context()) must.ErrorContains(t, err, "provider \"test-provider\" for secret \"test\" response contained error") @@ -66,14 +66,14 @@ func TestExternalPluginProvider_Fetch(t *testing.T) { t.Run("formats response correctly", func(t *testing.T) { mockSecretPlugin := new(MockSecretPlugin) - mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{ + mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(&commonplugins.SecretResponse{ Result: map[string]string{ "testkey": "testvalue", }, Error: nil, }, nil) - testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test") + testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil) result, err := testProvider.Fetch(t.Context()) must.NoError(t, err) diff --git a/client/allocrunner/taskrunner/secrets_hook.go b/client/allocrunner/taskrunner/secrets_hook.go index daf771cf4fd..3239d543e29 100644 --- a/client/allocrunner/taskrunner/secrets_hook.go +++ b/client/allocrunner/taskrunner/secrets_hook.go @@ -167,8 +167,12 @@ func (h *secretsHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart } h.envBuilder.SetSecrets(m) - // Set secrets from plugin providers + taskEnv := h.envBuilder.Build() + for _, p := range pluginProvider { + if ep, ok := p.(*secrets.ExternalPluginProvider); ok { + ep.InterpolateEnv(taskEnv.ReplaceEnv) + } vars, err := p.Fetch(ctx) if err != nil { return err @@ -205,15 +209,14 @@ func (h *secretsHook) buildSecretProviders(secretDir string) ([]TemplateProvider tmplProvider = append(tmplProvider, p) } default: - // Add/overwrite the nomad namespace and jobID envVars - s.Env = h.setupPluginEnv(s.Env) - - plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider, s.Env) + plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider) if err != nil { multierror.Append(mErr, err) continue } - pluginProvider = append(pluginProvider, secrets.NewExternalPluginProvider(plug, s.Provider, s.Name, s.Path)) + // Add/overwrite the nomad namespace and jobID envVars + s.Env = h.setupPluginEnv(s.Env) + pluginProvider = append(pluginProvider, secrets.NewExternalPluginProvider(plug, s.Provider, s.Name, s.Path, s.Env)) } } diff --git a/client/allocrunner/taskrunner/secrets_hook_test.go b/client/allocrunner/taskrunner/secrets_hook_test.go index 6032f984f7f..27dd2be02b1 100644 --- a/client/allocrunner/taskrunner/secrets_hook_test.go +++ b/client/allocrunner/taskrunner/secrets_hook_test.go @@ -407,4 +407,103 @@ fi` must.Eq(t, exp, taskEnv.Build().TaskSecrets) }) + + t.Run("interpolates secret references in plugin env", func(t *testing.T) { + // Setup Nomad variable server that returns a token + secretsResp := ` + { + "CreateIndex": 812, + "CreateTime": 1750782609539170600, + "Items": { + "token": "my-secret-token" + }, + "ModifyIndex": 812, + "ModifyTime": 1750782609539170600, + "Namespace": "default", + "Path": "nomad/jobs/creds" + } + ` + count := 0 + nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Nomad-Index", strconv.Itoa(count)) + fmt.Fprintln(w, secretsResp) + count += 1 + })) + t.Cleanup(nomadServer.Close) + + l, d := bufconndialer.New() + nomadServer.Listener = l + nomadServer.Start() + + clientConfig := config.DefaultConfig() + clientConfig.TemplateDialer = d + clientConfig.TemplateConfig.DisableSandbox = true + clientConfig.CommonPluginDir = t.TempDir() + + // Create plugin that echoes back the SERVICE_TOKEN env var + pluginDir := filepath.Join(clientConfig.CommonPluginDir, "secrets") + err := os.MkdirAll(pluginDir, 0755) + must.NoError(t, err) + + pluginPath := filepath.Join(pluginDir, "test-plugin") + testPlugin := fmt.Sprintf(basePlugin, `"received_token": "${SERVICE_TOKEN}"`) + err = os.WriteFile(pluginPath, []byte(testPlugin), 0755) + must.NoError(t, err) + + taskDir := t.TempDir() + alloc := mock.MinAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + + taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region) + conf := &secretsHookConfig{ + logger: testlog.HCLogger(t), + lifecycle: trtesting.NewMockTaskHooks(), + events: &trtesting.MockEmitter{}, + clientConfig: clientConfig, + envBuilder: taskEnv, + nomadNamespace: "default", + jobId: "test-job", + } + + // First secret: nomad variable that resolves token + // Second secret: plugin that references the resolved token via ${secret.creds.token} + secretHook := newSecretsHook(conf, []*structs.Secret{ + { + Name: "creds", + Provider: "nomad", + Path: "nomad/jobs/creds", + Config: map[string]any{ + "namespace": "default", + }, + }, + { + Name: "my_plugin", + Provider: "test-plugin", + Path: "/some/path", + Env: map[string]string{ + "SERVICE_TOKEN": "${secret.creds.token}", + }, + }, + }) + + req := &interfaces.TaskPrestartRequest{ + Alloc: alloc, + Task: task, + TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir}, + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + + err = secretHook.Prestart(ctx, req, &interfaces.TaskPrestartResponse{}) + must.NoError(t, err) + + secrets := taskEnv.Build().TaskSecrets + + // Verify the nomad variable was resolved + must.Eq(t, "my-secret-token", secrets["secret.creds.token"]) + + // Verify the plugin received the interpolated token value + must.Eq(t, "my-secret-token", secrets["secret.my_plugin.received_token"]) + }) } diff --git a/client/commonplugins/secrets_plugin.go b/client/commonplugins/secrets_plugin.go index 768c649ddf0..38afb778efe 100644 --- a/client/commonplugins/secrets_plugin.go +++ b/client/commonplugins/secrets_plugin.go @@ -29,7 +29,7 @@ const ( type SecretsPlugin interface { CommonPlugin - Fetch(ctx context.Context, path string) (*SecretResponse, error) + Fetch(ctx context.Context, path string, env map[string]string) (*SecretResponse, error) } type SecretResponse struct { @@ -42,15 +42,12 @@ type externalSecretsPlugin struct { // pluginPath is the path on the host to the plugin executable pluginPath string - - // env is optional envVars passed to the plugin process - env map[string]string } // NewExternalSecretsPlugin creates an instance of a secrets plugin by validating the plugin // binary exists and is executable, and parsing any string key/value pairs out of the config // which will be used as environment variables for Fetch. -func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[string]string) (*externalSecretsPlugin, error) { +func NewExternalSecretsPlugin(commonPluginDir string, name string) (*externalSecretsPlugin, error) { // validate plugin if runtime.GOOS == "windows" { name += ".exe" @@ -67,10 +64,7 @@ func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[strin return nil, fmt.Errorf("%w: %q", ErrPluginNotExecutable, name) } - return &externalSecretsPlugin{ - pluginPath: executable, - env: env, - }, nil + return &externalSecretsPlugin{pluginPath: executable}, nil } func (e *externalSecretsPlugin) Fingerprint(ctx context.Context) (*PluginFingerprint, error) { @@ -99,7 +93,7 @@ func (e *externalSecretsPlugin) Fingerprint(ctx context.Context) (*PluginFingerp return res, nil } -func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*SecretResponse, error) { +func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string, env map[string]string) (*SecretResponse, error) { plugCtx, cancel := context.WithTimeout(ctx, SecretsCmdTimeout) defer cancel() @@ -108,8 +102,8 @@ func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*Secret "CPI_OPERATION=fetch", } - for env, val := range e.env { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env, val)) + for envKey, val := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, val)) } stdout, stderr, err := runPlugin(cmd, SecretsKillTimeout) diff --git a/client/commonplugins/secrets_plugin_test.go b/client/commonplugins/secrets_plugin_test.go index 6571406cdba..4fa934b9106 100644 --- a/client/commonplugins/secrets_plugin_test.go +++ b/client/commonplugins/secrets_plugin_test.go @@ -21,7 +21,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) { t.Run("runs successfully", func(t *testing.T) { pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <