Skip to content

SDCICD-1735: update krknai prompt and markdown-to-HTML conversion #3128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
minlei98 wants to merge 1 commit into
base: main
Choose a base branch
from
+374 −100
Open

SDCICD-1735: update krknai prompt and markdown-to-HTML conversion #3128

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ require (
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
github.com/golang/glog v1.2.4 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUv
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab h1:VYNivV7P8IRHUam2swVUNkhIdp0LRRFKe4hXNnoZKTc=
github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw=
github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
Expand Down
2 changes: 2 additions & 0 deletions internal/analysisengine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ type ClusterInfo struct {
Region string
CloudProvider string
Version string
Type string // e.g. "rosa", "osd", "aro"
Hypershift bool
}

// Config holds configuration for the analysis engine.
Expand Down
6 changes: 6 additions & 0 deletions internal/prompts/prompts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,12 @@ func (ps *PromptStore) loadTemplates(filesystem fs.FS) error {
})
}

// RegisterTemplates loads additional templates from the given filesystem,
// overwriting any existing templates with the same ID.
func (ps *PromptStore) RegisterTemplates(templatesFS fs.FS) error {
return ps.loadTemplates(templatesFS)
}

func (ps *PromptStore) GetTemplate(id string) (*PromptTemplate, error) {
template, exists := ps.templates[id]
if !exists {
Expand Down
3 changes: 3 additions & 0 deletions internal/prompts/prompts_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ func TestNewPromptStore(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, store)
assert.Greater(t, len(store.templates), 0, "Should have loaded some templates")

_, err = store.GetTemplate("default")
assert.NoError(t, err, "default template should be loaded")
}

func TestGetTemplate(t *testing.T) {
Expand Down
21 changes: 20 additions & 1 deletion pkg/krknai/aggregator/aggregator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ import (
"gopkg.in/yaml.v3"
)

// ClusterInfo holds cluster metadata for krkn-ai analysis.
type ClusterInfo struct {
ID string `json:"id,omitempty" yaml:"id,omitempty"`
Version string `json:"version,omitempty" yaml:"version,omitempty"`
Type string `json:"type,omitempty" yaml:"type,omitempty"` // Combined: "cloud/platform[-hcp]", e.g. "aws/rosa-hcp"
Region string `json:"region,omitempty" yaml:"region,omitempty"`
Environment string `json:"environment,omitempty" yaml:"environment,omitempty"` // e.g. "stage", "production", "integration"
}

const (
// Default file paths relative to results directory
allCSVPath = "reports/all.csv"
Expand All @@ -29,6 +38,7 @@ const (
type KrknAIAggregator struct {
logger logr.Logger
topScenariosCount int
clusterInfo *ClusterInfo
}

// KrknAIData holds aggregated krkn-ai results with minimal context.
Expand All @@ -39,6 +49,7 @@ type KrknAIData struct {
HealthCheckReport []HealthCheckResult `json:"healthCheckReport"`
LogArtifacts []internalAggregator.LogEntry `json:"logArtifacts"`
ConfigSummary string `json:"configSummary,omitempty"`
ClusterInfo *ClusterInfo `json:"clusterInfo,omitempty"`
}

// KrknAISummary provides high-level statistics about the chaos test run.
Expand Down Expand Up @@ -89,6 +100,12 @@ func (a *KrknAIAggregator) WithTopScenariosCount(count int) *KrknAIAggregator {
return a
}

// WithClusterInfo sets cluster metadata to include in collected data.
func (a *KrknAIAggregator) WithClusterInfo(info *ClusterInfo) *KrknAIAggregator {
a.clusterInfo = info
return a
}

// Collect gathers krkn-ai results from the specified directory.
func (a *KrknAIAggregator) Collect(ctx context.Context, resultsDir string) (*KrknAIData, error) {
a.logger.Info("collecting krkn-ai results", "resultsDir", resultsDir)
Expand All @@ -97,7 +114,9 @@ func (a *KrknAIAggregator) Collect(ctx context.Context, resultsDir string) (*Krk
return nil, fmt.Errorf("results directory does not exist: %s", resultsDir)
}

data := &KrknAIData{}
data := &KrknAIData{
ClusterInfo: a.clusterInfo,
}
var collectionErrors []string

// Collect scenario results from all.csv
Expand Down
71 changes: 61 additions & 10 deletions pkg/krknai/analysisengine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
package analysisengine

import (
"bytes"
"context"
"embed"
"fmt"
"html/template"
"io/fs"
"os"
"path/filepath"
"time"

"github.com/gomarkdown/markdown"
mdhtml "github.com/gomarkdown/markdown/html"
"github.com/gomarkdown/markdown/parser"
"github.com/openshift/osde2e/internal/analysisengine"
"github.com/openshift/osde2e/internal/llm"
"github.com/openshift/osde2e/internal/llm/tools"
Expand All @@ -18,21 +23,22 @@ import (
"gopkg.in/yaml.v3"
)

//go:embed prompts/krknai.yaml
var krknaiTemplatesFS embed.FS
//go:embed prompts/*
var krknPrompts embed.FS

const (
analysisDirName = "llm-analysis"
summaryFileName = "summary.yaml"

// krknAIPromptTemplate is the prompt template ID for krkn-ai analysis.
krknAIPromptTemplate = "krknai"
htmlTemplatePath = "prompts/report.html"
)

// Config holds configuration for the krkn-ai analysis engine.
type Config struct {
analysisengine.BaseConfig
TopScenariosCount int // Number of top scenarios to include (default: 10)
TopScenariosCount int // Number of top scenarios to include (default: 10)
ReportFormat string // "json" (default), "markdown", or "html"
}

// Engine analyzes krkn-ai chaos test results using LLM.
Expand Down Expand Up @@ -60,14 +66,17 @@ func New(ctx context.Context, config *Config) (*Engine, error) {
agg.WithTopScenariosCount(config.TopScenariosCount)
}

templatesFS, err := fs.Sub(krknaiTemplatesFS, "prompts")
promptStore, err := prompts.NewPromptStore(prompts.DefaultTemplates())
if err != nil {
return nil, fmt.Errorf("failed to access embedded prompts: %w", err)
return nil, fmt.Errorf("failed to initialize prompt store: %w", err)
}

promptStore, err := prompts.NewPromptStore(templatesFS)
localFS, err := fs.Sub(krknPrompts, "prompts")
if err != nil {
return nil, fmt.Errorf("failed to initialize prompt store: %w", err)
return nil, fmt.Errorf("failed to load krkn-ai prompt templates: %w", err)
}
if err := promptStore.RegisterTemplates(localFS); err != nil {
return nil, fmt.Errorf("failed to register krkn-ai prompt templates: %w", err)
}

client, err := llm.NewGeminiClient(ctx, config.APIKey)
Expand All @@ -88,6 +97,12 @@ func New(ctx context.Context, config *Config) (*Engine, error) {
}, nil
}

// WithClusterInfo sets cluster metadata on the aggregator for inclusion in collected data.
func (e *Engine) WithClusterInfo(info *krknAggregator.ClusterInfo) *Engine {
e.aggregator.WithClusterInfo(info)
return e
}

// Run executes the krkn-ai analysis workflow.
func (e *Engine) Run(ctx context.Context) (*analysisengine.Result, error) {
// Collect krkn-ai results
Expand All @@ -99,7 +114,7 @@ func (e *Engine) Run(ctx context.Context) (*analysisengine.Result, error) {
// Create tool registry with log artifacts for read_file tool
toolRegistry := tools.NewRegistry(data.LogArtifacts)

// Prepare template variables
// Prepare template variables from collected data
vars := map[string]any{
"Summary": data.Summary,
"TopScenarios": data.TopScenarios,
Expand All @@ -108,6 +123,9 @@ func (e *Engine) Run(ctx context.Context) (*analysisengine.Result, error) {
"LogArtifacts": data.LogArtifacts,
"ConfigSummary": data.ConfigSummary,
}
if data.ClusterInfo != nil {
vars["ClusterInfo"] = data.ClusterInfo
}

// Render prompt using prompt store
userPrompt, llmConfig, err := e.promptStore.RenderPrompt(krknAIPromptTemplate, vars)
Expand All @@ -134,10 +152,19 @@ func (e *Engine) Run(ctx context.Context) (*analysisengine.Result, error) {
return nil, fmt.Errorf("LLM analysis failed: %w", err)
}

content := result.Content
if e.config.ReportFormat == "html" {
var err error
content, err = markdownToHTML(content)
if err != nil {
return nil, fmt.Errorf("failed to convert markdown to HTML: %w", err)
}
}
Comment on lines +156 to +162
Copy link

@coderabbitai coderabbitai bot Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Reject unsupported ReportFormat values explicitly.

Line 156 only handles "html". Typos or invalid values silently fall back to raw content, which is hard to diagnose.

💡 Suggested guard 
 	content := result.Content
-	if e.config.ReportFormat == "html" {
-		var err error
-		content, err = markdownToHTML(content)
-		if err != nil {
-			return nil, fmt.Errorf("failed to convert markdown to HTML: %w", err)
-		}
-	}
+	switch e.config.ReportFormat {
+	case "", "json", "markdown":
+		// keep markdown content as-is
+	case "html":
+		var err error
+		content, err = markdownToHTML(content)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert markdown to HTML: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("unsupported report format %q", e.config.ReportFormat)
+	}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if e.config.ReportFormat == "html" {
var err error
content, err = markdownToHTML(content)
if err != nil {
return nil, fmt.Errorf("failed to convert markdown to HTML: %w", err)
}
}
switch e.config.ReportFormat {
case "", "json", "markdown":
// keep markdown content as-is
case "html":
var err error
content, err = markdownToHTML(content)
if err != nil {
return nil, fmt.Errorf("failed to convert markdown to HTML: %w", err)
}
default:
return nil, fmt.Errorf("unsupported report format %q", e.config.ReportFormat)
}
🤖 Prompt for AI Agents 
Verify each finding against the current code and only fix it if needed.

In `@pkg/krknai/analysisengine/engine.go` around lines 156 - 162, Validate
ReportFormat explicitly before conversion: if e.config.ReportFormat is "html"
call markdownToHTML(content) as you already do, but add a guard such as if
e.config.ReportFormat != "" && e.config.ReportFormat != "html" { return nil,
fmt.Errorf("unsupported ReportFormat: %q", e.config.ReportFormat) } so typos or
invalid values on e.config.ReportFormat are rejected instead of silently falling
back to raw; keep the markdownToHTML call and its error handling unchanged.


// Build analysis result
analysisResult := &analysisengine.Result{
Status: "completed",
Content: result.Content,
Content: content,
Prompt: userPrompt,
Metadata: map[string]any{
"analysis_type": "krknai",
Expand Down Expand Up @@ -181,6 +208,7 @@ func (e *Engine) writeSummary(result *analysisengine.Result, data *krknAggregato
summary := map[string]any{
"timestamp": time.Now().Format(time.RFC3339),
"analysis_type": "krknai",
"cluster_info": data.ClusterInfo,
"run_summary": map[string]any{
"total_scenarios": data.Summary.TotalScenarioCount,
"successful_scenarios": data.Summary.SuccessfulScenarioCount,
Expand Down Expand Up @@ -212,6 +240,29 @@ func (e *Engine) writeSummary(result *analysisengine.Result, data *krknAggregato
return nil
}

func markdownToHTML(content string) (string, error) {
htmlTmplBytes, err := krknPrompts.ReadFile(htmlTemplatePath)
if err != nil {
return "", fmt.Errorf("failed to read HTML template: %w", err)
}

tmpl, err := template.New("report").Parse(string(htmlTmplBytes))
if err != nil {
return "", fmt.Errorf("failed to parse HTML template: %w", err)
}

p := parser.NewWithExtensions(parser.CommonExtensions | parser.AutoHeadingIDs)
renderer := mdhtml.NewRenderer(mdhtml.RendererOptions{Flags: mdhtml.CommonFlags | mdhtml.HrefTargetBlank})
body := markdown.ToHTML([]byte(content), p, renderer)

var buf bytes.Buffer
if err := tmpl.Execute(&buf, struct{ Body template.HTML }{Body: template.HTML(body)}); err != nil {
return "", fmt.Errorf("failed to execute HTML template: %w", err)
}

return buf.String(), nil
}

// sendNotifications sends analysis results to configured reporters.
func (e *Engine) sendNotifications(ctx context.Context, result *analysisengine.Result) {
reporterResult := &reporter.AnalysisResult{
Expand Down
Loading