diff --git a/CHANGELOG.md b/CHANGELOG.md index e9669e838e..0d8cbff9e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http This is included by default. ([#1859](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1859)) - The `WithEnv` function no longer parses `OTEL_SERVICE_NAME` or `OTEL_TRACES_EXPORTER`. Use the `Handler` from `go.opentelemtry.io/auto/pipeline/otelsdk` with its own `WithEnv` to replace functionality. ([#1859](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1859)) +- Instrument spans created with the OpenTelemetry trace API from an empty context. ([#2001](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/2001)) - Upgrade OpenTelemetry semantic conventions to `v1.30.0`. ([#2032](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/2032)) ### Removed diff --git a/instrumentation.go b/instrumentation.go index f968f7f257..a77211d06f 100644 --- a/instrumentation.go +++ b/instrumentation.go @@ -22,6 +22,7 @@ import ( kafkaConsumer "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer" kafkaProducer "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer" autosdk "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk" + otelTrace "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace" otelTraceGlobal "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal" grpcClient "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/google.golang.org/grpc/client" grpcServer "go.opentelemetry.io/auto/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server" @@ -73,6 +74,7 @@ func NewInstrumentation( kafkaProducer.New(c.logger, Version()), kafkaConsumer.New(c.logger, Version()), autosdk.New(c.logger), + otelTrace.New(c.logger), otelTraceGlobal.New(c.logger), } diff --git a/internal/include/sdk.h b/internal/include/sdk.h new file mode 100644 index 0000000000..3413e0e36c --- /dev/null +++ b/internal/include/sdk.h @@ -0,0 +1,64 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#ifndef SDK_H +#define SDK_H + +#include "trace/span_context.h" + +#define MAX_CONCURRENT 50 +// TODO: tune this. This is a just a guess, but we should be able to determine +// the maximum size of a span (based on limits) and set this. Ideally, we could +// also look into a tiered allocation strategy so we do not over allocate +// space (i.e. small, medium, large data sizes). +#define MAX_SIZE 2048 + +// Injected constants +volatile const u64 span_context_trace_id_pos; +volatile const u64 span_context_span_id_pos; +volatile const u64 span_context_trace_flags_pos; + +struct otel_span_t { + struct span_context sc; + struct span_context psc; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void*); + __type(value, struct otel_span_t); + __uint(max_entries, MAX_CONCURRENT); +} active_spans_by_span_ptr SEC(".maps"); + +static __always_inline long write_span_context(void *go_sc, struct span_context *sc) { + if (go_sc == NULL) { + bpf_printk("write_span_context: NULL go_sc"); + return -1; + } + + void *tid = (void *)(go_sc + span_context_trace_id_pos); + long ret = bpf_probe_write_user(tid, &sc->TraceID, TRACE_ID_SIZE); + if (ret != 0) { + bpf_printk("write_span_context: failed to write trace ID: %ld", ret); + return -2; + } + + void *sid = (void *)(go_sc + span_context_span_id_pos); + ret = bpf_probe_write_user(sid, &sc->SpanID, SPAN_ID_SIZE); + if (ret != 0) { + bpf_printk("write_span_context: failed to write span ID: %ld", ret); + return -3; + } + + void *flags = (void *)(go_sc + span_context_trace_flags_pos); + ret = bpf_probe_write_user(flags, &sc->TraceFlags, TRACE_FLAGS_SIZE); + if (ret != 0) { + bpf_printk("write_span_context: failed to write trace flags: %ld", ret); + return -4; + } + + return 0; +} + + +#endif // SDK_H diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c index 2b14d1f355..c92a273242 100644 --- a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c @@ -4,36 +4,13 @@ #include "arguments.h" #include "go_context.h" #include "go_types.h" +#include "sdk.h" #include "trace/span_context.h" #include "trace/start_span.h" #include "trace/span_output.h" char __license[] SEC("license") = "Dual MIT/GPL"; -#define MAX_CONCURRENT 50 -// TODO: tune this. This is a just a guess, but we should be able to determine -// the maximum size of a span (based on limits) and set this. Ideally, we could -// also look into a tiered allocation strategy so we do not over allocate -// space (i.e. small, medium, large data sizes). -#define MAX_SIZE 2048 - -// Injected const. -volatile const u64 span_context_trace_id_pos; -volatile const u64 span_context_span_id_pos; -volatile const u64 span_context_trace_flags_pos; - -struct otel_span_t { - struct span_context sc; - struct span_context psc; -}; - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct otel_span_t); - __uint(max_entries, MAX_CONCURRENT); -} active_spans_by_span_ptr SEC(".maps"); - struct event_t { u32 size; char data[MAX_SIZE]; @@ -46,36 +23,6 @@ struct { __uint(max_entries, 1); } new_event SEC(".maps"); -static __always_inline long write_span_context(void *go_sc, struct span_context *sc) { - if (go_sc == NULL) { - bpf_printk("write_span_context: NULL go_sc"); - return -1; - } - - void *tid = (void *)(go_sc + span_context_trace_id_pos); - long ret = bpf_probe_write_user(tid, &sc->TraceID, TRACE_ID_SIZE); - if (ret != 0) { - bpf_printk("write_span_context: failed to write trace ID: %ld", ret); - return -2; - } - - void *sid = (void *)(go_sc + span_context_span_id_pos); - ret = bpf_probe_write_user(sid, &sc->SpanID, SPAN_ID_SIZE); - if (ret != 0) { - bpf_printk("write_span_context: failed to write span ID: %ld", ret); - return -3; - } - - void *flags = (void *)(go_sc + span_context_trace_flags_pos); - ret = bpf_probe_write_user(flags, &sc->TraceFlags, TRACE_FLAGS_SIZE); - if (ret != 0) { - bpf_printk("write_span_context: failed to write trace flags: %ld", ret); - return -4; - } - - return 0; -} - // This instrumentation attaches a uprobe to the following function: // func (t *tracer) start(ctx context.Context, spanPtr *span, parentSpanCtx *trace.SpanContext, sampled *bool, spanCtx *trace.SpanContext) { // https://github.com/open-telemetry/opentelemetry-go-instrumentation/blob/effdec9ac23e56e9e9655663d386600e62b10871/sdk/trace.go#L56-L66 diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c new file mode 100644 index 0000000000..10466b3da6 --- /dev/null +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c @@ -0,0 +1,182 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#include "arguments.h" +#include "go_context.h" +#include "go_types.h" +#include "sdk.h" +#include "trace/span_context.h" +#include "trace/start_span.h" +#include "trace/span_output.h" + +char __license[] SEC("license") = "Dual MIT/GPL"; + +// Records state of our write to auto-instrumentation flag. +bool wrote_flag = false; + +struct control_t { + u64 kind; // Required to be 1. +}; + +struct event_t { + u64 kind; // Required to be 0. + u32 size; + char data[MAX_SIZE]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(struct event_t)); + __uint(max_entries, 1); +} new_event SEC(".maps"); + +// This instrumentation attaches uprobe to the following function: +// func (noopSpan) tracerProvider(autoEnabled *bool) TracerProvider +// https://github.com/open-telemetry/opentelemetry-go/blob/2e8d5a99340b1e11ca6b19bcdfcbfe9cd0c2c385/trace/noop.go#L98C1-L98C65 +SEC("uprobe/tracerProvider") +int uprobe_tracerProvider(struct pt_regs *ctx) { + if (wrote_flag) { + // Already wrote flag value. + return 0; + } + + void *flag_ptr = get_argument(ctx, 3); + if (flag_ptr == NULL) { + bpf_printk("invalid flag_ptr: NULL"); + return -1; + } + + bool true_value = true; + long res = bpf_probe_write_user(flag_ptr, &true_value, sizeof(bool)); + if (res != 0) { + bpf_printk("failed to write bool flag value: %ld", res); + return -2; + } + + wrote_flag = true; + + // Signal this uprobe should be unloaded. + struct control_t ctrl = {1}; + return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t)); +} + +// This instrumentation attaches a uprobe to the following function: +// func (t *autoTracer) start(ctx context.Context, spanPtr *autoSpan, psc *SpanContext, sampled *bool, sc *SpanContext) +// https://github.com/open-telemetry/opentelemetry-go/blob/2e8d5a99340b1e11ca6b19bcdfcbfe9cd0c2c385/trace/auto.go#L81-L92 +SEC("uprobe/Tracer_start") +int uprobe_Tracer_start(struct pt_regs *ctx) { + struct go_iface go_context = {0}; + get_Go_context(ctx, 2, 0, true, &go_context); + + struct otel_span_t otel_span; + __builtin_memset(&otel_span, 0, sizeof(struct otel_span_t)); + + start_span_params_t params = { + .ctx = ctx, + .go_context = &go_context, + .psc = &otel_span.psc, + .sc = &otel_span.sc, + .get_parent_span_context_fn = NULL, + .get_parent_span_context_arg = NULL, // Default to new root. + }; + + start_span(¶ms); + + void *parent_span_context = get_argument(ctx, 5); + long rc = write_span_context(parent_span_context, &otel_span.psc); + if (rc != 0) { + bpf_printk("failed to write parent span context: %ld", rc); + } + + if (!is_sampled(params.sc)) { + // Default SDK behaviour is to sample everything. Only set the sampled + // value to false when needed. + void *sampled_ptr_val = get_argument(ctx, 6); + if (sampled_ptr_val == NULL) { + bpf_printk("nil sampled pointer"); + } else { + bool false_val = false; + rc = bpf_probe_write_user(sampled_ptr_val, &false_val, sizeof(bool)); + if (rc != 0) { + bpf_printk("bpf_probe_write_user: failed to write sampled value: %ld", rc); + } else { + bpf_printk("wrote sampled value"); + } + } + } + + void *span_context_ptr_val = get_argument(ctx, 7); + rc = write_span_context(span_context_ptr_val, &otel_span.sc); + if (rc != 0) { + bpf_printk("failed to write span context: %ld", rc); + } + + void *span_ptr_val = get_argument(ctx, 4); + bpf_map_update_elem(&active_spans_by_span_ptr, &span_ptr_val, &otel_span, 0); + start_tracking_span(go_context.data, &otel_span.sc); + + return 0; +} + +// This instrumentation attaches a uprobe to the following function: +// func (*autoSpan) ended(buf []byte) {} +// https://github.com/open-telemetry/opentelemetry-go/blob/2e8d5a99340b1e11ca6b19bcdfcbfe9cd0c2c385/trace/auto.go#L435-L448 +SEC("uprobe/Span_ended") +int uprobe_Span_ended(struct pt_regs *ctx) { + void *span_ptr = get_argument(ctx, 1); + struct otel_span_t *span = bpf_map_lookup_elem(&active_spans_by_span_ptr, &span_ptr); + if (span == NULL) { + return 0; + } + bool sampled = is_sampled(&span->sc); + stop_tracking_span(&span->sc, &span->psc); + bpf_map_delete_elem(&active_spans_by_span_ptr, &span_ptr); + + // Do not output un-sampled span data. + if (!sampled) return 0; + + u64 len = (u64)get_argument(ctx, 3); + if (len > MAX_SIZE) { + bpf_printk("span data too large: %d", len); + return -1; + } + if (len == 0) { + bpf_printk("empty span data"); + return 0; + } + + void *data_ptr = get_argument(ctx, 2); + if (data_ptr == NULL) { + bpf_printk("empty span data"); + return 0; + } + + u32 key = 0; + struct event_t *event = bpf_map_lookup_elem(&new_event, &key); + if (event == NULL) { + bpf_printk("failed to initialize new event"); + return -2; + } + event->size = (u32)len; + + if (event->size < MAX_SIZE) { + long rc = bpf_probe_read(&event->data, event->size, data_ptr); + if (rc < 0) { + bpf_printk("failed to read encoded span data"); + return -3; + } + } else { + bpf_printk("read too large: %d", event->size); + return -4; + } + + // Do not send the whole size.buf if it is not needed. + u64 size = sizeof(event->kind) + sizeof(event->size) + event->size; + // Make the verifier happy, ensure no unbounded memory access. + if (size < sizeof(struct event_t)+1) { + return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, size); + } + bpf_printk("write too large: %d", event->size); + return -5; +} diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_arm64_bpfel.go b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_arm64_bpfel.go new file mode 100644 index 0000000000..9ecfd66b86 --- /dev/null +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_arm64_bpfel.go @@ -0,0 +1,193 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64 + +package sdk + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +type bpfOtelSpanT struct { + Sc bpfSpanContext + Psc bpfSpanContext +} + +type bpfSliceArrayBuff struct{ Buff [1024]uint8 } + +type bpfSpanContext struct { + TraceID [16]uint8 + SpanID [8]uint8 + TraceFlags uint8 + Padding [7]uint8 +} + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs + bpfVariableSpecs +} + +// bpfProgramSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + UprobeSpanEnded *ebpf.ProgramSpec `ebpf:"uprobe_Span_ended"` + UprobeTracerStart *ebpf.ProgramSpec `ebpf:"uprobe_Tracer_start"` + UprobeTracerProvider *ebpf.ProgramSpec `ebpf:"uprobe_tracerProvider"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + ActiveSpansBySpanPtr *ebpf.MapSpec `ebpf:"active_spans_by_span_ptr"` + AllocMap *ebpf.MapSpec `ebpf:"alloc_map"` + Events *ebpf.MapSpec `ebpf:"events"` + GoContextToSc *ebpf.MapSpec `ebpf:"go_context_to_sc"` + NewEvent *ebpf.MapSpec `ebpf:"new_event"` + ProbeActiveSamplerMap *ebpf.MapSpec `ebpf:"probe_active_sampler_map"` + SamplersConfigMap *ebpf.MapSpec `ebpf:"samplers_config_map"` + SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` + TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` +} + +// bpfVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfVariableSpecs struct { + EndAddr *ebpf.VariableSpec `ebpf:"end_addr"` + Hex *ebpf.VariableSpec `ebpf:"hex"` + SpanContextSpanIdPos *ebpf.VariableSpec `ebpf:"span_context_span_id_pos"` + SpanContextTraceFlagsPos *ebpf.VariableSpec `ebpf:"span_context_trace_flags_pos"` + SpanContextTraceIdPos *ebpf.VariableSpec `ebpf:"span_context_trace_id_pos"` + StartAddr *ebpf.VariableSpec `ebpf:"start_addr"` + TotalCpus *ebpf.VariableSpec `ebpf:"total_cpus"` + WroteFlag *ebpf.VariableSpec `ebpf:"wrote_flag"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps + bpfVariables +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + ActiveSpansBySpanPtr *ebpf.Map `ebpf:"active_spans_by_span_ptr"` + AllocMap *ebpf.Map `ebpf:"alloc_map"` + Events *ebpf.Map `ebpf:"events"` + GoContextToSc *ebpf.Map `ebpf:"go_context_to_sc"` + NewEvent *ebpf.Map `ebpf:"new_event"` + ProbeActiveSamplerMap *ebpf.Map `ebpf:"probe_active_sampler_map"` + SamplersConfigMap *ebpf.Map `ebpf:"samplers_config_map"` + SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` + TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.ActiveSpansBySpanPtr, + m.AllocMap, + m.Events, + m.GoContextToSc, + m.NewEvent, + m.ProbeActiveSamplerMap, + m.SamplersConfigMap, + m.SliceArrayBuffMap, + m.TrackedSpansBySc, + ) +} + +// bpfVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfVariables struct { + EndAddr *ebpf.Variable `ebpf:"end_addr"` + Hex *ebpf.Variable `ebpf:"hex"` + SpanContextSpanIdPos *ebpf.Variable `ebpf:"span_context_span_id_pos"` + SpanContextTraceFlagsPos *ebpf.Variable `ebpf:"span_context_trace_flags_pos"` + SpanContextTraceIdPos *ebpf.Variable `ebpf:"span_context_trace_id_pos"` + StartAddr *ebpf.Variable `ebpf:"start_addr"` + TotalCpus *ebpf.Variable `ebpf:"total_cpus"` + WroteFlag *ebpf.Variable `ebpf:"wrote_flag"` +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + UprobeSpanEnded *ebpf.Program `ebpf:"uprobe_Span_ended"` + UprobeTracerStart *ebpf.Program `ebpf:"uprobe_Tracer_start"` + UprobeTracerProvider *ebpf.Program `ebpf:"uprobe_tracerProvider"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.UprobeSpanEnded, + p.UprobeTracerStart, + p.UprobeTracerProvider, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed bpf_arm64_bpfel.o +var _BpfBytes []byte diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_x86_bpfel.go b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_x86_bpfel.go new file mode 100644 index 0000000000..26413e4e11 --- /dev/null +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf_x86_bpfel.go @@ -0,0 +1,193 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 + +package sdk + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +type bpfOtelSpanT struct { + Sc bpfSpanContext + Psc bpfSpanContext +} + +type bpfSliceArrayBuff struct{ Buff [1024]uint8 } + +type bpfSpanContext struct { + TraceID [16]uint8 + SpanID [8]uint8 + TraceFlags uint8 + Padding [7]uint8 +} + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs + bpfVariableSpecs +} + +// bpfProgramSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + UprobeSpanEnded *ebpf.ProgramSpec `ebpf:"uprobe_Span_ended"` + UprobeTracerStart *ebpf.ProgramSpec `ebpf:"uprobe_Tracer_start"` + UprobeTracerProvider *ebpf.ProgramSpec `ebpf:"uprobe_tracerProvider"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + ActiveSpansBySpanPtr *ebpf.MapSpec `ebpf:"active_spans_by_span_ptr"` + AllocMap *ebpf.MapSpec `ebpf:"alloc_map"` + Events *ebpf.MapSpec `ebpf:"events"` + GoContextToSc *ebpf.MapSpec `ebpf:"go_context_to_sc"` + NewEvent *ebpf.MapSpec `ebpf:"new_event"` + ProbeActiveSamplerMap *ebpf.MapSpec `ebpf:"probe_active_sampler_map"` + SamplersConfigMap *ebpf.MapSpec `ebpf:"samplers_config_map"` + SliceArrayBuffMap *ebpf.MapSpec `ebpf:"slice_array_buff_map"` + TrackedSpansBySc *ebpf.MapSpec `ebpf:"tracked_spans_by_sc"` +} + +// bpfVariableSpecs contains global variables before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfVariableSpecs struct { + EndAddr *ebpf.VariableSpec `ebpf:"end_addr"` + Hex *ebpf.VariableSpec `ebpf:"hex"` + SpanContextSpanIdPos *ebpf.VariableSpec `ebpf:"span_context_span_id_pos"` + SpanContextTraceFlagsPos *ebpf.VariableSpec `ebpf:"span_context_trace_flags_pos"` + SpanContextTraceIdPos *ebpf.VariableSpec `ebpf:"span_context_trace_id_pos"` + StartAddr *ebpf.VariableSpec `ebpf:"start_addr"` + TotalCpus *ebpf.VariableSpec `ebpf:"total_cpus"` + WroteFlag *ebpf.VariableSpec `ebpf:"wrote_flag"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps + bpfVariables +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + ActiveSpansBySpanPtr *ebpf.Map `ebpf:"active_spans_by_span_ptr"` + AllocMap *ebpf.Map `ebpf:"alloc_map"` + Events *ebpf.Map `ebpf:"events"` + GoContextToSc *ebpf.Map `ebpf:"go_context_to_sc"` + NewEvent *ebpf.Map `ebpf:"new_event"` + ProbeActiveSamplerMap *ebpf.Map `ebpf:"probe_active_sampler_map"` + SamplersConfigMap *ebpf.Map `ebpf:"samplers_config_map"` + SliceArrayBuffMap *ebpf.Map `ebpf:"slice_array_buff_map"` + TrackedSpansBySc *ebpf.Map `ebpf:"tracked_spans_by_sc"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.ActiveSpansBySpanPtr, + m.AllocMap, + m.Events, + m.GoContextToSc, + m.NewEvent, + m.ProbeActiveSamplerMap, + m.SamplersConfigMap, + m.SliceArrayBuffMap, + m.TrackedSpansBySc, + ) +} + +// bpfVariables contains all global variables after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfVariables struct { + EndAddr *ebpf.Variable `ebpf:"end_addr"` + Hex *ebpf.Variable `ebpf:"hex"` + SpanContextSpanIdPos *ebpf.Variable `ebpf:"span_context_span_id_pos"` + SpanContextTraceFlagsPos *ebpf.Variable `ebpf:"span_context_trace_flags_pos"` + SpanContextTraceIdPos *ebpf.Variable `ebpf:"span_context_trace_id_pos"` + StartAddr *ebpf.Variable `ebpf:"start_addr"` + TotalCpus *ebpf.Variable `ebpf:"total_cpus"` + WroteFlag *ebpf.Variable `ebpf:"wrote_flag"` +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + UprobeSpanEnded *ebpf.Program `ebpf:"uprobe_Span_ended"` + UprobeTracerStart *ebpf.Program `ebpf:"uprobe_Tracer_start"` + UprobeTracerProvider *ebpf.Program `ebpf:"uprobe_tracerProvider"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.UprobeSpanEnded, + p.UprobeTracerStart, + p.UprobeTracerProvider, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +// +//go:embed bpf_x86_bpfel.o +var _BpfBytes []byte diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/probe.go b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/probe.go new file mode 100644 index 0000000000..ede37e7fcb --- /dev/null +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/probe.go @@ -0,0 +1,210 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package sdk provides an auto-instrumentation probe for the built-in auto-SDK +// in the go.opentelemetry.io/otel/trace package. +package sdk + +import ( + "bytes" + "encoding/binary" + "fmt" + "log/slog" + + "github.com/Masterminds/semver/v3" + "github.com/cilium/ebpf/perf" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.opentelemetry.io/otel/trace" + + "go.opentelemetry.io/auto/internal/pkg/instrumentation/probe" + "go.opentelemetry.io/auto/internal/pkg/structfield" +) + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target amd64,arm64 bpf ./bpf/probe.bpf.c + +// New returns a new [probe.Probe]. +func New(logger *slog.Logger) probe.Probe { + id := probe.ID{ + SpanKind: trace.SpanKindClient, + InstrumentedPkg: "go.opentelemetry.io/otel/trace", + } + + // Minimum version of go.opentelemetry.io/otel/trace that added an + // auto-instrumentation SDK implementation for non-recording spans. + otelWithAutoSDK := probe.PackageConstraints{ + Package: "go.opentelemetry.io/otel/trace", + Constraints: func() *semver.Constraints { + c, err := semver.NewConstraint(">= 1.35.1-0") + if err != nil { + panic(err) + } + return c + }(), + FailureMode: probe.FailureModeIgnore, + } + + uprobeTracerProvider := &probe.Uprobe{ + Sym: "go.opentelemetry.io/otel/trace.noopSpan.tracerProvider", + EntryProbe: "uprobe_tracerProvider", + PackageConstraints: []probe.PackageConstraints{ + otelWithAutoSDK, + }, + } + + c := &converter{ + logger: logger, + uprobeTracerProvider: uprobeTracerProvider, + } + return &probe.TraceProducer[bpfObjects, event]{ + Base: probe.Base[bpfObjects, event]{ + ID: id, + Logger: logger, + Consts: []probe.Const{ + probe.AllocationConst{}, + probe.StructFieldConst{ + Key: "span_context_trace_id_pos", + ID: structfield.NewID( + "go.opentelemetry.io/otel", + "go.opentelemetry.io/otel/trace", + "SpanContext", + "traceID", + ), + }, + probe.StructFieldConst{ + Key: "span_context_span_id_pos", + ID: structfield.NewID( + "go.opentelemetry.io/otel", + "go.opentelemetry.io/otel/trace", + "SpanContext", + "spanID", + ), + }, + probe.StructFieldConst{ + Key: "span_context_trace_flags_pos", + ID: structfield.NewID( + "go.opentelemetry.io/otel", + "go.opentelemetry.io/otel/trace", + "SpanContext", + "traceFlags", + ), + }, + }, + Uprobes: []*probe.Uprobe{ + uprobeTracerProvider, + { + Sym: "go.opentelemetry.io/otel/trace.(*autoTracer).start", + EntryProbe: "uprobe_Tracer_start", + PackageConstraints: []probe.PackageConstraints{ + otelWithAutoSDK, + }, + }, + { + Sym: "go.opentelemetry.io/otel/trace.(*autoSpan).ended", + EntryProbe: "uprobe_Span_ended", + PackageConstraints: []probe.PackageConstraints{ + otelWithAutoSDK, + }, + }, + }, + SpecFn: loadBpf, + ProcessRecord: c.decodeEvent, + }, + ProcessFn: c.processFn, + } +} + +type event struct { + Size uint32 + SpanData []byte +} + +type recordKind uint64 + +const ( + recordKindTelemetry recordKind = iota + recordKindConrol +) + +type converter struct { + logger *slog.Logger + + uprobeTracerProvider *probe.Uprobe +} + +func (c *converter) decodeEvent(record perf.Record) (*event, error) { + c.logger.Debug( + "decoding event", + "len", + len(record.RawSample), + "CPU", + record.CPU, + "remaining", + record.Remaining, + "lost", + record.LostSamples, + ) + + reader := bytes.NewReader(record.RawSample) + + var kind recordKind + err := binary.Read(reader, binary.LittleEndian, &kind) + if err != nil { + c.logger.Error("failed read kind", "error", err) + return nil, err + } + + var e *event + switch kind { + case recordKindTelemetry: + e = new(event) + + err = binary.Read(reader, binary.LittleEndian, &e.Size) + if err != nil { + c.logger.Error("failed to decode size", "error", err) + break + } + c.logger.Debug("decoded size", "size", e.Size) + + e.SpanData = make([]byte, e.Size) + _, err = reader.Read(e.SpanData) + if err != nil { + c.logger.Error("failed to read span data", "error", err) + break + } + c.logger.Debug("decoded span data", "size", e.Size) + case recordKindConrol: + if c.uprobeTracerProvider != nil { + err = c.uprobeTracerProvider.Close() + c.uprobeTracerProvider = nil + } + c.logger.Debug("unloading noopSpan.tracerProvider uprobe") + default: + err = fmt.Errorf("unknown record kind: %d", kind) + } + return e, err +} + +func (c *converter) processFn(e *event) (pcommon.InstrumentationScope, string, ptrace.SpanSlice) { + var m ptrace.JSONUnmarshaler + traces, err := m.UnmarshalTraces(e.SpanData[:e.Size]) + if err != nil { + c.logger.Error("failed to unmarshal span data", "error", err) + return pcommon.InstrumentationScope{}, "", ptrace.SpanSlice{} + } + + rs := traces.ResourceSpans() + if rs.Len() == 0 { + c.logger.Error("empty ResourceSpans") + return pcommon.InstrumentationScope{}, "", ptrace.SpanSlice{} + } + + ss := rs.At(0).ScopeSpans() + if ss.Len() == 0 { + c.logger.Error("empty ScopeSpans") + return pcommon.InstrumentationScope{}, "", ptrace.SpanSlice{} + } + + s := ss.At(0) + return s.Scope(), s.SchemaUrl(), s.Spans() +} diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/probe.go b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/probe.go index 1aaff7e8d8..0dc3a61698 100644 --- a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/probe.go +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/probe.go @@ -40,7 +40,7 @@ const ( // Minimum version of go.opentelemetry.io/otel that supports using the // go.opentelemetry.io/auto/sdk in the global API. - minAutoSDK = "1.33.0" + minAutoSDK = "1.33.0-0" ) func must(c *semver.Constraints, err error) *semver.Constraints { @@ -51,7 +51,7 @@ func must(c *semver.Constraints, err error) *semver.Constraints { } var ( - goMapsVersion = semver.New(1, 24, 0, "", "") + goMapsVersion = semver.New(1, 24, 0, "", "0") otelWithAutoSDK = probe.PackageConstraints{ Package: "go.opentelemetry.io/otel", diff --git a/internal/test/e2e/go.mod b/internal/test/e2e/go.mod index 8071dbf267..a377f954ac 100644 --- a/internal/test/e2e/go.mod +++ b/internal/test/e2e/go.mod @@ -12,8 +12,8 @@ require ( go.opentelemetry.io/auto v0.21.0 go.opentelemetry.io/auto/sdk v1.1.0 go.opentelemetry.io/collector/pdata v1.30.0 - go.opentelemetry.io/otel v1.35.0 - go.opentelemetry.io/otel/trace v1.35.0 + go.opentelemetry.io/otel v1.35.1-0.20250319215043-90a9d1d93604 + go.opentelemetry.io/otel/trace v1.35.1-0.20250319215043-90a9d1d93604 go.uber.org/goleak v1.3.0 google.golang.org/grpc v1.72.0 google.golang.org/grpc/examples v0.0.0-20250421233052-f7d488de751d diff --git a/internal/test/e2e/go.sum b/internal/test/e2e/go.sum index e937df1b36..58d38a0642 100644 --- a/internal/test/e2e/go.sum +++ b/internal/test/e2e/go.sum @@ -177,8 +177,8 @@ go.opentelemetry.io/contrib/exporters/autoexport v0.60.0 h1:GuQXpvSXNjpswpweIem8 go.opentelemetry.io/contrib/exporters/autoexport v0.60.0/go.mod h1:CkmxekdHco4d7thFJNPQ7Mby4jMBgZUclnrxT4e+ryk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.35.1-0.20250319215043-90a9d1d93604 h1:baE3vOBAOuU+9ETouibi+daF14xEiJ3BXoi7K0jnJIc= +go.opentelemetry.io/otel v1.35.1-0.20250319215043-90a9d1d93604/go.mod h1:dytzy8H+Ym99GtJtLWMBV2+l9Ida7zqNuFDJ+CGRQ20= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.11.0 h1:HMUytBT3uGhPKYY/u/G5MR9itrlSO2SMOsSD3Tk3k7A= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.11.0/go.mod h1:hdDXsiNLmdW/9BF2jQpnHHlhFajpWCEYfM6e5m2OAZg= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 h1:C/Wi2F8wEmbxJ9Kuzw/nhP+Z9XaHYMkyDmXy6yR2cjw= @@ -211,8 +211,8 @@ go.opentelemetry.io/otel/sdk/log v0.11.0 h1:7bAOpjpGglWhdEzP8z0VXc4jObOiDEwr3IYb go.opentelemetry.io/otel/sdk/log v0.11.0/go.mod h1:dndLTxZbwBstZoqsJB3kGsRPkpAgaJrWfQg3lhlHFFY= go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/trace v1.35.1-0.20250319215043-90a9d1d93604 h1:/v53KmoEX9YRQ6Fbh2MNoWys6Z+5/8JkqP1WqNdn2Ik= +go.opentelemetry.io/otel/trace v1.35.1-0.20250319215043-90a9d1d93604/go.mod h1:v525kGRZc7GDmy0BwoFtoPfYvyWKsB54mdPNeKwkBBM= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/internal/test/e2e/nonrecording/cmd/main.go b/internal/test/e2e/nonrecording/cmd/main.go new file mode 100644 index 0000000000..19bd4ac81d --- /dev/null +++ b/internal/test/e2e/nonrecording/cmd/main.go @@ -0,0 +1,122 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package nonrecording is a testing application for the +// [go.opentelemetry.io/otel/trace] package. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "go.opentelemetry.io/auto/internal/test/trigger" +) + +const ( + pkgName = "go.opentelemetry.io/auto/internal/test/e2e/nonrecording" + pkgVer = "v1.23.42" + schemaURL = "https://some_schema" +) + +// Y2K (January 1, 2000). +var y2k = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + +type app struct { + tracer trace.Tracer +} + +func (a *app) Run(ctx context.Context, user string, admin bool, in <-chan msg) error { + opts := []trace.SpanStartOption{ + trace.WithAttributes( + attribute.String("user", user), + attribute.Bool("admin", admin), + ), + trace.WithTimestamp(y2k.Add(500 * time.Microsecond)), + trace.WithSpanKind(trace.SpanKindServer), + } + _, span := a.tracer.Start(ctx, "Run", opts...) + defer span.End(trace.WithTimestamp(y2k.Add(1 * time.Second))) + + for m := range in { + span.AddLink(trace.Link{ + SpanContext: m.SpanContext, + Attributes: []attribute.KeyValue{attribute.String("data", m.Data)}, + }) + } + + return errors.New("broken") +} + +type msg struct { + SpanContext trace.SpanContext + Data string +} + +func sig(ctx context.Context) <-chan msg { + tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer( + pkgName, + trace.WithInstrumentationVersion(pkgVer), + trace.WithSchemaURL(schemaURL), + ) + + ts := y2k.Add(10 * time.Microsecond) + _, span := tracer.Start(ctx, "sig", trace.WithTimestamp(ts)) + defer span.End(trace.WithTimestamp(ts.Add(100 * time.Microsecond))) + + out := make(chan msg, 1) + out <- msg{SpanContext: span.SpanContext(), Data: "Hello World"} + close(out) + + return out +} + +func main() { + var trig trigger.Flag + flag.Var(&trig, "trigger", trig.Docs()) + flag.Parse() + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + // Wait for auto-instrumentation. + err := trig.Wait(ctx) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // Auto-instrumentation should integrate with the non-recording span + // returned, by default, from the OpenTelemetry Go trace API when + // trace.SpanFromContext is called on a context without any active span. + tracer := trace.SpanFromContext(ctx).TracerProvider().Tracer( + pkgName, + trace.WithInstrumentationVersion(pkgVer), + trace.WithSchemaURL(schemaURL), + ) + app := app{tracer: tracer} + + ctx, span := tracer.Start(ctx, "main", trace.WithTimestamp(y2k)) + + // Ensure the OTel trace API SDK for auto-instrumentation is full featured. + err = app.Run(ctx, "Alice", true, sig(ctx)) + if err != nil { + span.SetStatus(codes.Error, "application error") + span.RecordError( + err, + trace.WithAttributes(attribute.Int("impact", 11)), + trace.WithTimestamp(y2k.Add(2*time.Second)), + trace.WithStackTrace(true), + ) + } + + span.End(trace.WithTimestamp(y2k.Add(5 * time.Second))) +} diff --git a/internal/test/e2e/nonrecording/integration_test.go b/internal/test/e2e/nonrecording/integration_test.go new file mode 100644 index 0000000000..63f71f71ae --- /dev/null +++ b/internal/test/e2e/nonrecording/integration_test.go @@ -0,0 +1,15 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package nonrecording provides an integration test for the trace API probe. +package nonrecording + +import ( + "testing" + + "go.opentelemetry.io/auto/internal/test/e2e" +) + +func TestIntegration(t *testing.T) { + e2e.TestIntegration(t, "./cmd", ".") +} diff --git a/internal/test/e2e/nonrecording/verify.bats b/internal/test/e2e/nonrecording/verify.bats new file mode 100644 index 0000000000..1ff3d0ded8 --- /dev/null +++ b/internal/test/e2e/nonrecording/verify.bats @@ -0,0 +1,154 @@ +#!/usr/bin/env bats + +load ../../test_helpers/utilities.sh + +SCOPE="go.opentelemetry.io/auto/internal/test/e2e/nonrecording" + +@test "nonrecording :: includes service.name in resource attributes" { + result=$(resource_attributes_received | jq "select(.key == \"service.name\").value.stringValue") + assert_equal "$result" '"sample-app"' +} + +@test "nonrecording :: include tracer name in scope" { + result=$(spans_received | jq ".scopeSpans[].scope.name") + assert_equal "$result" "\"$SCOPE\"" +} + +@test "nonrecording :: include tracer version in scope" { + result=$(spans_received | jq ".scopeSpans[].scope.version") + assert_equal "$result" '"v1.23.42"' +} + +@test "nonrecording :: include schema url" { + result=$(spans_received | jq ".scopeSpans[].schemaUrl") + assert_equal "$result" '"https://some_schema"' +} + +@test "nonrecording :: main span :: trace ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"main\")" | jq ".traceId") + assert_regex "$trace_id" ${MATCH_A_TRACE_ID} +} + +@test "nonrecording :: main span :: span ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"main\")" | jq ".spanId") + assert_regex "$trace_id" ${MATCH_A_SPAN_ID} +} + +@test "nonrecording :: main span :: start time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"main\")" | jq ".startTimeUnixNano") + assert_regex "$timestamp" "946684800000000000" +} + +@test "nonrecording :: main span :: end time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"main\")" | jq ".endTimeUnixNano") + assert_regex "$timestamp" "946684805000000000" +} + +@test "nonrecording :: main span :: event" { + event=$(span_events ${SCOPE} "main") + + assert_equal $(echo "$event" | jq ".timeUnixNano") '"946684802000000000"' + assert_equal $(echo "$event" | jq ".name") '"exception"' + + attrs=$(echo "$event" | jq ".attributes[]") + + impact=$(echo "$attrs" | jq "select(.key == \"impact\").value.intValue") + assert_equal "$impact" '"11"' + + type=$(echo "$attrs" | jq "select(.key == \"exception.type\").value.stringValue") + assert_equal "$type" '"*errors.errorString"' + + msg=$(echo "$attrs" | jq "select(.key == \"exception.message\").value.stringValue") + assert_equal "$msg" '"broken"' + + st=$(echo "$attrs" | jq "select(.key == \"exception.stacktrace\")") + assert_not_empty "$st" +} + +@test "nonrecording :: main span :: status" { + status=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"main\")" | jq ".status") + assert_equal "$(echo $status | jq ".code")" "2" + assert_equal "$(echo $status | jq ".message")" '"application error"' +} + +@test "nonrecording :: sig span :: trace ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".traceId") + assert_regex "$trace_id" ${MATCH_A_TRACE_ID} +} + +@test "nonrecording :: sig span :: span ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".spanId") + assert_regex "$trace_id" ${MATCH_A_SPAN_ID} +} + +@test "nonrecording :: sig span :: parent span ID" { + parent_span_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".parentSpanId") + assert_regex "$parent_span_id" ${MATCH_A_SPAN_ID} +} + +@test "nonrecording :: sig span :: start time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".startTimeUnixNano") + assert_regex "$timestamp" "946684800000010000" +} + +@test "nonrecording :: sig span :: end time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".endTimeUnixNano") + assert_regex "$timestamp" "946684800000110000" +} + +@test "nonrecording :: Run span :: trace ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".traceId") + assert_regex "$trace_id" ${MATCH_A_TRACE_ID} +} + +@test "nonrecording :: Run span :: span ID" { + trace_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".spanId") + assert_regex "$trace_id" ${MATCH_A_SPAN_ID} +} + +@test "nonrecording :: Run span :: parent span ID" { + parent_span_id=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".parentSpanId") + assert_regex "$parent_span_id" ${MATCH_A_SPAN_ID} +} + +@test "nonrecording :: Run span :: start time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".startTimeUnixNano") + assert_regex "$timestamp" "946684800000500000" +} + +@test "nonrecording :: Run span :: end time" { + timestamp=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".endTimeUnixNano") + assert_regex "$timestamp" "946684801000000000" +} + +@test "nonrecording :: Run span :: kind" { + kind=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"Run\")" | jq ".kind") + assert_equal "$kind" "2" +} + +@test "nonrecording :: Run span :: attribute :: user" { + result=$(span_attributes_for ${SCOPE} | jq "select(.key == \"user\").value.stringValue") + assert_equal "$result" '"Alice"' +} + +@test "nonrecording :: Run span :: attribute :: admin" { + result=$(span_attributes_for ${SCOPE} | jq "select(.key == \"admin\").value.boolValue") + assert_equal "$result" 'true' +} + +@test "nonrecording :: Run span :: link :: traceID" { + want=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".traceId") + got=$(span_links ${SCOPE} "Run" | jq ".traceId") + assert_equal "$got" "$want" +} + +@test "nonrecording :: Run span :: link :: spanID" { + want=$(spans_from_scope_named ${SCOPE} | jq "select(.name == \"sig\")" | jq ".spanId") + got=$(span_links ${SCOPE} "Run" | jq ".spanId") + assert_equal "$got" "$want" +} + +@test "nonrecording :: Run span :: link :: attributes" { + got=$(span_links ${SCOPE} "Run" | jq ".attributes[] | select(.key == \"data\").value.stringValue") + assert_equal "$got" '"Hello World"' +}