Merge pull request openshift#218 from jeefy/SDCICD-51

SDCICD-51: Support arbitrary metrics derived from log data

Author: openshift-merge-robot

common/e2e.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -206,6 +207,29 @@ func runTestsInPhase(t *testing.T, phase string, description string) {
 			}
 		}
 	}
+
+	logMetricsRegexs := make(map[string]*regexp.Regexp)
+	for name, match := range cfg.LogMetrics {
+		logMetricsRegexs[name] = regexp.MustCompile(match)
+	}
+
+	files, err = ioutil.ReadDir(cfg.ReportDir)
+	if err != nil {
+		t.Fatalf("error reading phase directory: %s", err.Error())
+	}
+
+	for _, file := range files {
+		if logFileRegex.MatchString(file.Name()) {
+			data, err := ioutil.ReadFile(filepath.Join(cfg.ReportDir, file.Name()))
+			if err != nil {
+				t.Fatalf("error opening log file %s: %s", file.Name(), err.Error())
+			}
+			for name, matchRegex := range logMetricsRegexs {
+				matches := matchRegex.Find(data)
+				metadata.Instance.IncrementLogMetric(name, len(matches))
+			}
+		}
+	}
 }
 
 // checkBeforeMetricsGeneration runs a variety of checks before generating metrics.

common/metrics.go

@@ -32,7 +32,7 @@ const (
 	eventMetricName    string = cicdPrefix + "event"
 )
 
-var junitFileRegex *regexp.Regexp
+var junitFileRegex, logFileRegex *regexp.Regexp
 
 // Metrics is the metrics object which can parse jUnit and JSON metadata and produce Prometheus metrics.
 type Metrics struct {
@@ -87,6 +87,7 @@ func NewMetrics() *Metrics {
 
 func init() {
 	junitFileRegex = regexp.MustCompile("^junit.*\\.xml$")
+	logFileRegex = regexp.MustCompile("^.*\\.(log|txt)$")
 }
 
 // WritePrometheusFile collects data and writes it out in the prometheus export file format (https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md)

pkg/config/config.go

@@ -29,6 +29,10 @@ type Config struct {
 
 	// DryRun lets you run osde2e all the way up to the e2e tests then skips them.
 	DryRun bool `json:"dry_run,omitempty" env:"DRY_RUN" sect:"tests"  yaml:"dryRun"`
+
+	// LogMetrics lets you define a metric name and a regex to apply on the build log
+	// For every match in the build log, the metric with that name will increment
+	LogMetrics map[string]string `json:"log-metrics" yaml:"logMetrics"`
 }
 
 // KubeConfig stores information required to talk to the Kube API

pkg/metadata/metadata.go

@@ -31,15 +31,17 @@ type Metadata struct {
 	UpgradeVersionSource string `json:"upgrade-version-source,omitempty"`
 
 	// Metrics
-	TimeToOCMReportingInstalled float64 `json:"time-to-ocm-reporting-installed,string"`
-	TimeToClusterReady          float64 `json:"time-to-cluster-ready,string"`
+	TimeToOCMReportingInstalled float64        `json:"time-to-ocm-reporting-installed,string"`
+	TimeToClusterReady          float64        `json:"time-to-cluster-ready,string"`
+	LogMetrics                  map[string]int `json:"log-metrics"`
 }
 
 // Instance is the global metadata instance
 var Instance *Metadata
 
 func init() {
 	Instance = &Metadata{}
+	Instance.LogMetrics = make(map[string]int)
 }
 
 // Next are a bunch of setter functions that allow us
@@ -94,6 +96,17 @@ func (m *Metadata) SetTimeToClusterReady(timeToClusterReady float64) {
 	m.WriteToJSON(config.Instance.ReportDir)
 }
 
+// IncrementLogMetric adds a supplied number to a log metric or sets the metric to
+// the value if it doesn't exist already
+func (m *Metadata) IncrementLogMetric(metric string, value int) {
+	if _, ok := m.LogMetrics[metric]; ok {
+		m.LogMetrics[metric] += value
+	} else {
+		m.LogMetrics[metric] = value
+	}
+	m.WriteToJSON(config.Instance.ReportDir)
+}
+
 // WriteToJSON will marshall the metadata struct and write it into the given file.
 func (m *Metadata) WriteToJSON(reportDir string) (err error) {
 	var data []byte

pkg/metadata/metadata_test.go

@@ -10,11 +10,31 @@ import (
 	"testing"
 )
 
+// generateExpected marshals/unmarshals a supplied metadata object.
+// Rather than hand-curating our own expected data, this functionally
+// achieves the same test result but it's cleaner.
+func generateExpected(m *Metadata) map[string]interface{} {
+	tmp, err := json.Marshal(m)
+	if err != nil {
+		fmt.Printf("%s", err.Error())
+		return nil
+	}
+
+	data := make(map[string]interface{})
+
+	err = json.Unmarshal(tmp, &data)
+	if err != nil {
+		fmt.Printf("%s\n", err.Error())
+		return nil
+	}
+
+	return data
+}
+
 func TestMetadata(t *testing.T) {
 	tests := []struct {
-		name     string
-		m        *Metadata
-		expected map[string]interface{}
+		name string
+		m    *Metadata
 	}{
 		{
 			name: "test all fields",
@@ -27,15 +47,6 @@ func TestMetadata(t *testing.T) {
 				TimeToOCMReportingInstalled: 123.45,
 				TimeToClusterReady:          456.78,
 			},
-			expected: map[string]interface{}{
-				"cluster-id":                      "test-id",
-				"cluster-name":                    "test-name",
-				"cluster-version":                 "test-version",
-				"environment":                     "test-environment",
-				"upgrade-version":                 "test-upgrade",
-				"time-to-ocm-reporting-installed": "123.45",
-				"time-to-cluster-ready":           "456.78",
-			},
 		},
 		{
 			name: "omit upgrade version",
@@ -47,25 +58,32 @@ func TestMetadata(t *testing.T) {
 				TimeToOCMReportingInstalled: 123.45,
 				TimeToClusterReady:          456.78,
 			},
-			expected: map[string]interface{}{
-				"cluster-id":                      "test-id",
-				"cluster-name":                    "test-name",
-				"cluster-version":                 "test-version",
-				"environment":                     "test-environment",
-				"time-to-ocm-reporting-installed": "123.45",
-				"time-to-cluster-ready":           "456.78",
+		},
+		{
+			name: "log metrics exists",
+			m: &Metadata{
+				ClusterID:                   "test-id",
+				ClusterName:                 "test-name",
+				ClusterVersion:              "test-version",
+				Environment:                 "test-environment",
+				UpgradeVersion:              "test-upgrade",
+				TimeToOCMReportingInstalled: 123.45,
+				TimeToClusterReady:          456.78,
+				LogMetrics: map[string]int{
+					"some-metric": 5,
+				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		if err := writeAndTestMetadata(test.m, test.expected); err != nil {
+		if err := writeAndTestMetadata(test.m); err != nil {
 			t.Errorf("%s: error while testing metadata: %v", test.name, err)
 		}
 	}
 }
 
-func writeAndTestMetadata(m *Metadata, expected map[string]interface{}) (err error) {
+func writeAndTestMetadata(m *Metadata) (err error) {
 	var tempDir string
 	if tempDir, err = ioutil.TempDir("", ""); err != nil {
 		return err
@@ -87,7 +105,7 @@ func writeAndTestMetadata(m *Metadata, expected map[string]interface{}) (err err
 		return err
 	}
 
-	if !reflect.DeepEqual(expected, readData) {
+	if !reflect.DeepEqual(generateExpected(m), readData) {
 		return fmt.Errorf("expected and generated JSON do not match")
 	}