Skip to content

Commit cb2eceb

Browse files
authored
Update workflow to embed app assets (#1039)
* Support downloading the static assets from the published OCI image since thats how runmedev/web is publishing them * We want to be able to support serving additional YAML/JSON files from the server e.g. https://acme.dev/configs/app-configs.yaml * This YAML can contain application configuration that the webapp fetches and configures itself with * e.g. it can contain the OIDC and Google Drive configuration. * Then in the web app we can just do ``` app.setConfig("http://localhost:9966/configs/app-configs.yaml") ``` * Allow CORS to be partially set on static assets. During development we might still serve app-configs from the server even though the frontend is running on a different development server. --------- Signed-off-by: Jeremy lewi <jeremy@lewi.us>
1 parent 75e54dd commit cb2eceb

File tree

13 files changed

+374
-47
lines changed

13 files changed

+374
-47
lines changed

.github/workflows/runme-image.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
tags:
88
- 'v*'
99
pull_request:
10+
workflow_dispatch:
1011

1112
jobs:
1213
build:
@@ -36,6 +37,19 @@ jobs:
3637
username: ${{ github.actor }}
3738
password: ${{ secrets.GITHUB_TOKEN }}
3839

40+
# Download static app assets.
41+
# This relies on assets.go embedding the contents of dist/* into the go binary
42+
# The advantage of embedding directly in the go binary is that we can then make the binary serve the static
43+
# assets without needing to manage the assets separately. This could be useful if we want to use the
44+
# web app as an "admin" panel for bootstrapping the UI.
45+
# For a docker container it might make sense to package the assets on the filesystem but then we'd have two
46+
# different mechanisms for the docker image vs. standalone binary.
47+
# Right now the static assets are only around 3.4Mb
48+
- name: Pull app assets (OCI)
49+
run: |
50+
rm -rf ./kodata
51+
go run . agent download-assets --assets-dir ./kodata
52+
3953
- name: Compute image tags
4054
id: tags
4155
shell: bash

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ require (
7070
google.golang.org/genproto/googleapis/rpc v0.0.0-20260203192932-546029d2fa20
7171
google.golang.org/protobuf v1.36.11
7272
mvdan.cc/sh/v3 v3.12.0
73+
oras.land/oras-go/v2 v2.5.0
7374
)
7475

7576
require (

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,3 +604,5 @@ mvdan.cc/gofumpt v0.7.0 h1:bg91ttqXmi9y2xawvkuMXyvAA/1ZGJqYAEGjXuP0JXU=
604604
mvdan.cc/gofumpt v0.7.0/go.mod h1:txVFJy/Sc/mvaycET54pV8SW8gWxTlUuGHVEcncmNUo=
605605
mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI=
606606
mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg=
607+
oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c=
608+
oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg=

pkg/agent/ai/agent.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ type Agent struct {
6666
type AgentOptions struct {
6767
VectorStores []string
6868
Client *openai.Client
69+
6970
// Instructions are the prompt to use when generating responses
7071
Instructions string
7172

@@ -105,7 +106,7 @@ func NewAgent(opts AgentOptions) (*Agent, error) {
105106
}
106107
toolsForContext[agentv1.GenerateRequest_CONTEXT_SLACK] = runTools
107108

108-
log.Info("Creating Agent", "options", opts)
109+
log.Info("Creating Agent", "vectorStores", opts.VectorStores, "instructions", opts.Instructions, "oauthOpenAIOrganization", opts.OAuthOpenAIOrganization, "oauthOpenAIProject", opts.OAuthOpenAIProject)
109110

110111
return &Agent{
111112
Client: opts.Client,

pkg/agent/ai/client.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"github.com/openai/openai-go"
99
"github.com/openai/openai-go/option"
1010

11+
"github.com/runmedev/runme/v3/pkg/agent/logs"
12+
1113
"github.com/runmedev/runme/v3/pkg/agent/config"
1214

1315
"github.com/pkg/errors"
@@ -16,7 +18,9 @@ import (
1618
// NewClient helper function to create a new OpenAI client from a config
1719
func NewClient(cfg config.OpenAIConfig) (*openai.Client, error) {
1820
if cfg.APIKeyFile == "" {
19-
return nil, errors.New("OpenAI API key is empty")
21+
log := logs.NewLogger()
22+
log.Info("OpenAI client configured without APIKeyFile")
23+
return NewClientWithoutKey(), nil
2024
}
2125

2226
b, err := os.ReadFile(cfg.APIKeyFile)

pkg/agent/assets/downloader.go

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package assets
2+
3+
import (
4+
"archive/tar"
5+
"compress/gzip"
6+
"context"
7+
"io"
8+
"net/http"
9+
"os"
10+
"path/filepath"
11+
"strings"
12+
13+
"github.com/pkg/errors"
14+
"oras.land/oras-go/v2"
15+
"oras.land/oras-go/v2/content/file"
16+
"oras.land/oras-go/v2/registry"
17+
"oras.land/oras-go/v2/registry/remote"
18+
"oras.land/oras-go/v2/registry/remote/auth"
19+
)
20+
21+
const (
22+
defaultArchiveName = "app-assets.tgz"
23+
)
24+
25+
// DownloadFromImage pulls assets from an OCI image and unpacks them into outputDir.
26+
func DownloadFromImage(ctx context.Context, imageRef, outputDir string) error {
27+
if imageRef == "" {
28+
return errors.New("image reference is required")
29+
}
30+
if outputDir == "" {
31+
return errors.New("assets output directory is required")
32+
}
33+
34+
if err := os.MkdirAll(outputDir, 0o755); err != nil {
35+
return errors.Wrapf(err, "failed to create assets output directory %s", outputDir)
36+
}
37+
38+
tempDir, err := os.MkdirTemp("", "runme-agent-assets-*")
39+
if err != nil {
40+
return errors.Wrap(err, "failed to create temporary assets directory")
41+
}
42+
defer func() {
43+
_ = os.RemoveAll(tempDir)
44+
}()
45+
46+
if err := pullImage(ctx, imageRef, tempDir); err != nil {
47+
return err
48+
}
49+
50+
archivePath := filepath.Join(tempDir, defaultArchiveName)
51+
if _, err := os.Stat(archivePath); err != nil {
52+
return errors.Wrapf(err, "expected assets archive not found: %s", archivePath)
53+
}
54+
55+
if err := removeIndexFiles(outputDir); err != nil {
56+
return err
57+
}
58+
if err := extractTarGz(archivePath, outputDir); err != nil {
59+
return errors.Wrapf(err, "failed to extract assets archive %s", archivePath)
60+
}
61+
62+
return nil
63+
}
64+
65+
func removeIndexFiles(outputDir string) error {
66+
matches, err := filepath.Glob(filepath.Join(outputDir, "index.*"))
67+
if err != nil {
68+
return errors.Wrapf(err, "failed to glob index files in %s", outputDir)
69+
}
70+
for _, match := range matches {
71+
if err := os.RemoveAll(match); err != nil {
72+
return errors.Wrapf(err, "failed to remove %s", match)
73+
}
74+
}
75+
return nil
76+
}
77+
78+
func pullImage(ctx context.Context, imageRef, outputDir string) error {
79+
ref, err := registry.ParseReference(imageRef)
80+
if err != nil {
81+
return errors.Wrapf(err, "invalid image reference %q", imageRef)
82+
}
83+
84+
repo, err := remote.NewRepository(ref.Registry + "/" + ref.Repository)
85+
if err != nil {
86+
return errors.Wrapf(err, "failed to create repository for %q", imageRef)
87+
}
88+
89+
repo.Client = &auth.Client{
90+
Client: http.DefaultClient,
91+
Cache: auth.NewCache(),
92+
}
93+
94+
if ref.Reference == "" {
95+
ref.Reference = "latest"
96+
}
97+
98+
store, err := file.New(outputDir)
99+
if err != nil {
100+
return errors.Wrapf(err, "failed to create output file store %s", outputDir)
101+
}
102+
defer store.Close()
103+
104+
if _, err := oras.Copy(ctx, repo, ref.Reference, store, "", oras.DefaultCopyOptions); err != nil {
105+
return errors.Wrapf(err, "failed to pull image %s", imageRef)
106+
}
107+
108+
return nil
109+
}
110+
111+
func extractTarGz(archivePath, destDir string) error {
112+
fileHandle, err := os.Open(archivePath)
113+
if err != nil {
114+
return errors.Wrapf(err, "failed to open archive %s", archivePath)
115+
}
116+
defer fileHandle.Close()
117+
118+
gzipReader, err := gzip.NewReader(fileHandle)
119+
if err != nil {
120+
return errors.Wrap(err, "failed to create gzip reader")
121+
}
122+
defer gzipReader.Close()
123+
124+
tarReader := tar.NewReader(gzipReader)
125+
destDirClean := filepath.Clean(destDir)
126+
if !strings.HasSuffix(destDirClean, string(os.PathSeparator)) {
127+
destDirClean += string(os.PathSeparator)
128+
}
129+
130+
for {
131+
header, err := tarReader.Next()
132+
if err == io.EOF {
133+
break
134+
}
135+
if err != nil {
136+
return errors.Wrap(err, "failed to read tar entry")
137+
}
138+
139+
if header == nil {
140+
continue
141+
}
142+
143+
targetPath := filepath.Join(destDir, header.Name)
144+
cleanTarget := filepath.Clean(targetPath)
145+
if !strings.HasPrefix(cleanTarget, destDirClean) {
146+
return errors.Errorf("invalid tar entry path: %s", header.Name)
147+
}
148+
149+
switch header.Typeflag {
150+
case tar.TypeDir:
151+
if err := os.MkdirAll(cleanTarget, os.FileMode(header.Mode)); err != nil {
152+
return errors.Wrapf(err, "failed to create directory %s", cleanTarget)
153+
}
154+
case tar.TypeReg, tar.TypeRegA:
155+
if err := os.MkdirAll(filepath.Dir(cleanTarget), 0o755); err != nil {
156+
return errors.Wrapf(err, "failed to create parent directory for %s", cleanTarget)
157+
}
158+
outFile, err := os.OpenFile(cleanTarget, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(header.Mode))
159+
if err != nil {
160+
return errors.Wrapf(err, "failed to create file %s", cleanTarget)
161+
}
162+
if _, err := io.Copy(outFile, tarReader); err != nil {
163+
_ = outFile.Close()
164+
return errors.Wrapf(err, "failed to write file %s", cleanTarget)
165+
}
166+
if err := outFile.Close(); err != nil {
167+
return errors.Wrapf(err, "failed to close file %s", cleanTarget)
168+
}
169+
default:
170+
return errors.Errorf("unsupported tar entry type %v for %s", header.Typeflag, header.Name)
171+
}
172+
}
173+
174+
return nil
175+
}

pkg/agent/cmd/agent.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ func NewAgentCmd(appName string) *cobra.Command {
3131
agentCmd.AddCommand(NewServeCmd(appName))
3232
agentCmd.AddCommand(NewEnvCmd())
3333
agentCmd.AddCommand(NewEvalCmd(appName))
34+
agentCmd.AddCommand(NewDownloadAssetsCmd(appName))
3435

3536
serveCmd := NewServeCmd(appName)
3637
// Make serveCmd the default command.

pkg/agent/cmd/download_assets.go

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
8+
"github.com/go-logr/zapr"
9+
"github.com/pkg/errors"
10+
"github.com/spf13/cobra"
11+
"go.uber.org/zap"
12+
13+
"github.com/runmedev/runme/v3/pkg/agent/application"
14+
"github.com/runmedev/runme/v3/pkg/agent/assets"
15+
)
16+
17+
// NewDownloadAssetsCmd downloads and unpacks the web app assets from an OCI image.
18+
func NewDownloadAssetsCmd(appName string) *cobra.Command {
19+
var imageRef string
20+
var assetsDirFlag string
21+
22+
cmd := &cobra.Command{
23+
Use: "download-assets",
24+
Short: "Download and unpack web app assets",
25+
Run: func(cmd *cobra.Command, args []string) {
26+
err := func() error {
27+
if imageRef == "" {
28+
return errors.New("image reference is required; set --image")
29+
}
30+
31+
var assetsDir string
32+
if assetsDirFlag != "" {
33+
assetsDir = assetsDirFlag
34+
} else {
35+
app := application.NewApp(appName)
36+
if err := app.LoadConfig(cmd); err != nil {
37+
return err
38+
}
39+
if err := app.SetupLogging(); err != nil {
40+
return err
41+
}
42+
43+
cfg := app.AppConfig.GetConfig()
44+
if cfg.AssistantServer == nil {
45+
return errors.New("assistantServer config must be set to download assets")
46+
}
47+
48+
assetsDir = cfg.AssistantServer.StaticAssets
49+
if assetsDir == "" {
50+
homeDir, err := os.UserHomeDir()
51+
if err != nil {
52+
return errors.Wrap(err, "failed to resolve home directory for assets")
53+
}
54+
assetsDir = filepath.Join(homeDir, "."+appName, "assets")
55+
} else if !filepath.IsAbs(assetsDir) {
56+
homeDir, err := os.UserHomeDir()
57+
if err != nil {
58+
return errors.Wrap(err, "failed to resolve home directory for assets")
59+
}
60+
assetsDir = filepath.Join(homeDir, assetsDir)
61+
}
62+
}
63+
64+
log := zapr.NewLogger(zap.L())
65+
log.Info("Downloading assets image", "image", imageRef, "dir", assetsDir)
66+
67+
if err := assets.DownloadFromImage(cmd.Context(), imageRef, assetsDir); err != nil {
68+
return err
69+
}
70+
71+
log.Info("Assets download complete", "dir", assetsDir)
72+
return nil
73+
}()
74+
if err != nil {
75+
fmt.Printf("Failed to download assets;\n%+v\n", err)
76+
os.Exit(1)
77+
}
78+
},
79+
}
80+
81+
cmd.Flags().StringVar(&imageRef, "image", "ghcr.io/runmedev/app-assets:latest", "OCI image reference to download (e.g. ghcr.io/runmedev/app-assets:latest)")
82+
cmd.Flags().StringVar(&assetsDirFlag, "assets-dir", "", "Directory to download and unpack assets (skips config)")
83+
84+
return cmd
85+
}

pkg/agent/cmd/serve.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ func NewServeCmd(appName string) *cobra.Command {
5555

5656
agentOptions.Client = client
5757

58+
if app.AppConfig.OpenAI != nil {
59+
agentOptions.OAuthOpenAIOrganization = app.AppConfig.OpenAI.Organization
60+
agentOptions.OAuthOpenAIProject = app.AppConfig.OpenAI.Project
61+
}
62+
5863
agent, err := ai.NewAgent(*agentOptions)
5964
if err != nil {
6065
return err

pkg/agent/config/config.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,14 @@ type OpenAIConfig struct {
134134
// APIKeyFile is the file containing the OpenAI API key.
135135
// Optional when the client supplies an OAuth access token per request.
136136
APIKeyFile string `json:"apiKeyFile,omitempty" yaml:"apiKeyFile,omitempty"`
137+
138+
// Organization is the OpenAI organization to use
139+
// Only needs to be set if using OAuth and not using an APIKey.
140+
Organization string `json:"organization,omitempty" yaml:"organization,omitempty"`
141+
142+
// Project is the OpenAI project to use
143+
// Only needs to be set if using OAuth and not using an APIKey.
144+
Project string `json:"project,omitempty" yaml:"project,omitempty"`
137145
}
138146

139147
type Logging struct {
@@ -392,7 +400,9 @@ type AssistantServerConfig struct {
392400
// HttpMaxWriteTimeout is the max write duration.
393401
HttpMaxWriteTimeout time.Duration `json:"httpMaxWriteTimeout" yaml:"httpMaxWriteTimeout"`
394402

395-
// CorsOrigins is a list of allowed origins for CORS requests
403+
// CorsOrigins is a list of allowed origins for CORS requests. For static assets,
404+
// CORS is the only protection, so origins must be explicitly allowlisted; "*" will be removed
405+
// for the static assets.
396406
CorsOrigins []string `json:"corsOrigins" yaml:"corsOrigins"`
397407

398408
// StaticAssets is the path to the static assets to serve

0 commit comments

Comments
 (0)