Skip to content

Backport of add interpolation support in env stanzas for secret providers into release/1.11.x #27621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mismithhisler merged 1 commit into from
Mar 2, 2026
Merged

Backport of add interpolation support in env stanzas for secret providers into release/1.11.x #27621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions client/allocrunner/taskrunner/secrets/plugin_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,34 @@ type ExternalPluginProvider struct {

// path is the secret location used in Fetch
path string

// env is the set of environment variables passed into plugin
env map[string]string
}

type Response struct {
Result map[string]string `json:"result"`
Error *string `json:"error,omitempty"`
}

func NewExternalPluginProvider(plugin commonplugins.SecretsPlugin, pluginName, secretName, path string) *ExternalPluginProvider {
func NewExternalPluginProvider(plugin commonplugins.SecretsPlugin, pluginName, secretName, path string, env map[string]string) *ExternalPluginProvider {
return &ExternalPluginProvider{
plugin: plugin,
pluginName: pluginName,
secretName: secretName,
path: path,
env: env,
}
}

func (p *ExternalPluginProvider) InterpolateEnv(interpolate func(string) string) {
for key, value := range p.env {
p.env[key] = interpolate(value)
}
}

func (p *ExternalPluginProvider) Fetch(ctx context.Context) (map[string]string, error) {
resp, err := p.plugin.Fetch(ctx, p.path)
resp, err := p.plugin.Fetch(ctx, p.path, p.env)
if err != nil {
return nil, fmt.Errorf("failed executing plugin %q for secret %q: %w", p.pluginName, p.secretName, err)
}
Expand Down
16 changes: 8 additions & 8 deletions client/allocrunner/taskrunner/secrets/plugin_provider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ func (m *MockSecretPlugin) Fingerprint(ctx context.Context) (*commonplugins.Plug
return nil, nil
}

func (m *MockSecretPlugin) Fetch(ctx context.Context, path string) (*commonplugins.SecretResponse, error) {
args := m.Called()
func (m *MockSecretPlugin) Fetch(ctx context.Context, path string, env map[string]string) (*commonplugins.SecretResponse, error) {
args := m.Called(ctx, path, env)

if args.Get(0) == nil {
return nil, args.Error(1)
Expand All @@ -40,9 +40,9 @@ func (m *MockSecretPlugin) Parse() (map[string]string, error) {
func TestExternalPluginProvider_Fetch(t *testing.T) {
t.Run("errors if fetch errors", func(t *testing.T) {
mockSecretPlugin := new(MockSecretPlugin)
mockSecretPlugin.On("Fetch", mock.Anything).Return(nil, errors.New("something bad"))
mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("something bad"))

testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test")
testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil)

vars, err := testProvider.Fetch(t.Context())
must.ErrorContains(t, err, "something bad")
Expand All @@ -52,12 +52,12 @@ func TestExternalPluginProvider_Fetch(t *testing.T) {
t.Run("errors if fetch response contains error", func(t *testing.T) {
mockSecretPlugin := new(MockSecretPlugin)
testError := "something bad"
mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{
mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(&commonplugins.SecretResponse{
Result: nil,
Error: &testError,
}, nil)

testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test")
testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil)

vars, err := testProvider.Fetch(t.Context())
must.ErrorContains(t, err, "provider \"test-provider\" for secret \"test\" response contained error")
Expand All @@ -66,14 +66,14 @@ func TestExternalPluginProvider_Fetch(t *testing.T) {

t.Run("formats response correctly", func(t *testing.T) {
mockSecretPlugin := new(MockSecretPlugin)
mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{
mockSecretPlugin.On("Fetch", mock.Anything, mock.Anything, mock.Anything).Return(&commonplugins.SecretResponse{
Result: map[string]string{
"testkey": "testvalue",
},
Error: nil,
}, nil)

testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test")
testProvider := NewExternalPluginProvider(mockSecretPlugin, "test-provider", "test", "test", nil)

result, err := testProvider.Fetch(t.Context())
must.NoError(t, err)
Expand Down
15 changes: 9 additions & 6 deletions client/allocrunner/taskrunner/secrets_hook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,12 @@ func (h *secretsHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart
}
h.envBuilder.SetSecrets(m)

// Set secrets from plugin providers
taskEnv := h.envBuilder.Build()

for _, p := range pluginProvider {
if ep, ok := p.(*secrets.ExternalPluginProvider); ok {
ep.InterpolateEnv(taskEnv.ReplaceEnv)
}
vars, err := p.Fetch(ctx)
if err != nil {
return err
Expand Down Expand Up @@ -205,15 +209,14 @@ func (h *secretsHook) buildSecretProviders(secretDir string) ([]TemplateProvider
tmplProvider = append(tmplProvider, p)
}
default:
// Add/overwrite the nomad namespace and jobID envVars
s.Env = h.setupPluginEnv(s.Env)

plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider, s.Env)
plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider)
if err != nil {
multierror.Append(mErr, err)
continue
}
pluginProvider = append(pluginProvider, secrets.NewExternalPluginProvider(plug, s.Provider, s.Name, s.Path))
// Add/overwrite the nomad namespace and jobID envVars
s.Env = h.setupPluginEnv(s.Env)
pluginProvider = append(pluginProvider, secrets.NewExternalPluginProvider(plug, s.Provider, s.Name, s.Path, s.Env))
}
}

Expand Down
99 changes: 99 additions & 0 deletions client/allocrunner/taskrunner/secrets_hook_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,4 +407,103 @@ fi`

must.Eq(t, exp, taskEnv.Build().TaskSecrets)
})

t.Run("interpolates secret references in plugin env", func(t *testing.T) {
// Setup Nomad variable server that returns a token
secretsResp := `
{
"CreateIndex": 812,
"CreateTime": 1750782609539170600,
"Items": {
"token": "my-secret-token"
},
"ModifyIndex": 812,
"ModifyTime": 1750782609539170600,
"Namespace": "default",
"Path": "nomad/jobs/creds"
}
`
count := 0
nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("X-Nomad-Index", strconv.Itoa(count))
fmt.Fprintln(w, secretsResp)
count += 1
}))
t.Cleanup(nomadServer.Close)

l, d := bufconndialer.New()
nomadServer.Listener = l
nomadServer.Start()

clientConfig := config.DefaultConfig()
clientConfig.TemplateDialer = d
clientConfig.TemplateConfig.DisableSandbox = true
clientConfig.CommonPluginDir = t.TempDir()

// Create plugin that echoes back the SERVICE_TOKEN env var
pluginDir := filepath.Join(clientConfig.CommonPluginDir, "secrets")
err := os.MkdirAll(pluginDir, 0755)
must.NoError(t, err)

pluginPath := filepath.Join(pluginDir, "test-plugin")
testPlugin := fmt.Sprintf(basePlugin, `"received_token": "${SERVICE_TOKEN}"`)
err = os.WriteFile(pluginPath, []byte(testPlugin), 0755)
must.NoError(t, err)

taskDir := t.TempDir()
alloc := mock.MinAlloc()
task := alloc.Job.TaskGroups[0].Tasks[0]

taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
conf := &secretsHookConfig{
logger: testlog.HCLogger(t),
lifecycle: trtesting.NewMockTaskHooks(),
events: &trtesting.MockEmitter{},
clientConfig: clientConfig,
envBuilder: taskEnv,
nomadNamespace: "default",
jobId: "test-job",
}

// First secret: nomad variable that resolves token
// Second secret: plugin that references the resolved token via ${secret.creds.token}
secretHook := newSecretsHook(conf, []*structs.Secret{
{
Name: "creds",
Provider: "nomad",
Path: "nomad/jobs/creds",
Config: map[string]any{
"namespace": "default",
},
},
{
Name: "my_plugin",
Provider: "test-plugin",
Path: "/some/path",
Env: map[string]string{
"SERVICE_TOKEN": "${secret.creds.token}",
},
},
})

req := &interfaces.TaskPrestartRequest{
Alloc: alloc,
Task: task,
TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
}

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
t.Cleanup(cancel)

err = secretHook.Prestart(ctx, req, &interfaces.TaskPrestartResponse{})
must.NoError(t, err)

secrets := taskEnv.Build().TaskSecrets

// Verify the nomad variable was resolved
must.Eq(t, "my-secret-token", secrets["secret.creds.token"])

// Verify the plugin received the interpolated token value
must.Eq(t, "my-secret-token", secrets["secret.my_plugin.received_token"])
})
}
18 changes: 6 additions & 12 deletions client/commonplugins/secrets_plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const (

type SecretsPlugin interface {
CommonPlugin
Fetch(ctx context.Context, path string) (*SecretResponse, error)
Fetch(ctx context.Context, path string, env map[string]string) (*SecretResponse, error)
}

type SecretResponse struct {
Expand All @@ -42,15 +42,12 @@ type externalSecretsPlugin struct {

// pluginPath is the path on the host to the plugin executable
pluginPath string

// env is optional envVars passed to the plugin process
env map[string]string
}

// NewExternalSecretsPlugin creates an instance of a secrets plugin by validating the plugin
// binary exists and is executable, and parsing any string key/value pairs out of the config
// which will be used as environment variables for Fetch.
func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[string]string) (*externalSecretsPlugin, error) {
func NewExternalSecretsPlugin(commonPluginDir string, name string) (*externalSecretsPlugin, error) {
// validate plugin
if runtime.GOOS == "windows" {
name += ".exe"
Expand All @@ -67,10 +64,7 @@ func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[strin
return nil, fmt.Errorf("%w: %q", ErrPluginNotExecutable, name)
}

return &externalSecretsPlugin{
pluginPath: executable,
env: env,
}, nil
return &externalSecretsPlugin{pluginPath: executable}, nil
}

func (e *externalSecretsPlugin) Fingerprint(ctx context.Context) (*PluginFingerprint, error) {
Expand Down Expand Up @@ -99,7 +93,7 @@ func (e *externalSecretsPlugin) Fingerprint(ctx context.Context) (*PluginFingerp
return res, nil
}

func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*SecretResponse, error) {
func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string, env map[string]string) (*SecretResponse, error) {
plugCtx, cancel := context.WithTimeout(ctx, SecretsCmdTimeout)
defer cancel()

Expand All @@ -108,8 +102,8 @@ func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*Secret
"CPI_OPERATION=fetch",
}

for env, val := range e.env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env, val))
for envKey, val := range env {
cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envKey, val))
}

stdout, stderr, err := runPlugin(cmd, SecretsKillTimeout)
Expand Down
30 changes: 15 additions & 15 deletions client/commonplugins/secrets_plugin_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
t.Run("runs successfully", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"type": "secrets", "version": "1.0.0"}`))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

res, err := plugin.Fingerprint(context.Background())
Expand All @@ -34,7 +34,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
t.Run("errors on non-zero exit code", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

res, err := plugin.Fingerprint(context.Background())
Expand All @@ -45,7 +45,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
t.Run("errors on timeout", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\nleep .5\n"))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
Expand All @@ -58,7 +58,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
t.Run("errors on invalid json", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\ncat <<EOF\ninvalid\nEOF\n"))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

res, err := plugin.Fingerprint(context.Background())
Expand All @@ -73,10 +73,10 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
t.Run("runs successfully", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"key": "value"}}`))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

res, err := plugin.Fetch(context.Background(), "test-path")
res, err := plugin.Fetch(context.Background(), "test-path", nil)
must.NoError(t, err)

exp := map[string]string{"key": "value"}
Expand All @@ -86,44 +86,44 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
t.Run("errors on non-zero exit code", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

_, err = plugin.Fetch(context.Background(), "test-path")
_, err = plugin.Fetch(context.Background(), "test-path", nil)
must.Error(t, err)
})

t.Run("errors on timeout", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nsleep .5\n"))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()

_, err = plugin.Fetch(ctx, "dummy-path")
_, err = plugin.Fetch(ctx, "dummy-path", nil)
must.Error(t, err)
})

t.Run("errors on timeout", func(t *testing.T) {
pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `invalid`))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

_, err = plugin.Fetch(context.Background(), "dummy-path")
_, err = plugin.Fetch(context.Background(), "dummy-path", nil)
must.Error(t, err)
})

t.Run("can be passed environment variables via config", func(t *testing.T) {
t.Run("can be passed environment variables via Fetch", func(t *testing.T) {
// test the passed envVar is parsed and set correctly by printing it as part of the SecretResponse
pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"foo": "$TEST_KEY"}}`))

plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, map[string]string{"TEST_KEY": "TEST_VALUE"})
plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
must.NoError(t, err)

res, err := plugin.Fetch(context.Background(), "dummy-path")
res, err := plugin.Fetch(context.Background(), "dummy-path", map[string]string{"TEST_KEY": "TEST_VALUE"})
must.NoError(t, err)
must.Eq(t, res.Result, map[string]string{"foo": "TEST_VALUE"})
})
Expand Down
2 changes: 1 addition & 1 deletion client/fingerprint/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ func (s *SecretsPluginFingerprint) Fingerprint(request *FingerprintRequest, resp
plugins := map[string]string{}
for name := range files {
name = strings.TrimSuffix(name, ".exe")
plug, err := commonplugins.NewExternalSecretsPlugin(request.Config.CommonPluginDir, name, nil)
plug, err := commonplugins.NewExternalSecretsPlugin(request.Config.CommonPluginDir, name)
if err != nil {
return err
}
Expand Down