From c6d71b2bd46352ec02e893143511a7472ab51052 Mon Sep 17 00:00:00 2001 From: Rafael Roquetto Date: Mon, 18 Aug 2025 18:19:13 -0600 Subject: [PATCH 1/3] Add clang-format --- .clang-format | 13 + Makefile | 5 + internal/include/alloc.h | 57 +- internal/include/arguments.h | 6 +- internal/include/common.h | 166 ++-- internal/include/go_context.h | 57 +- internal/include/go_net.h | 6 +- internal/include/go_types.h | 79 +- internal/include/libbpf/bpf_helper_defs.h | 662 +++++++++----- internal/include/libbpf/bpf_helpers.h | 314 +++---- internal/include/libbpf/bpf_tracing.h | 812 ++++++++++++------ internal/include/otel_types.h | 209 ++--- internal/include/sdk.h | 3 +- internal/include/trace/sampling.h | 83 +- internal/include/trace/span_context.h | 13 +- internal/include/trace/span_output.h | 6 +- internal/include/trace/start_span.h | 3 +- internal/include/uprobe.h | 38 +- internal/include/utils.h | 49 +- .../bpf/database/sql/bpf/probe.bpf.c | 8 +- .../kafka-go/consumer/bpf/probe.bpf.c | 58 +- .../kafka-go/producer/bpf/probe.bpf.c | 43 +- .../auto/sdk/bpf/probe.bpf.c | 7 +- .../otel/trace/bpf/probe.bpf.c | 10 +- .../otel/traceglobal/bpf/probe.bpf.c | 263 +++--- .../grpc/client/bpf/probe.bpf.c | 47 +- .../grpc/server/bpf/probe.bpf.c | 70 +- .../bpf/net/http/client/bpf/probe.bpf.c | 96 ++- .../bpf/net/http/server/bpf/probe.bpf.c | 165 ++-- 29 files changed, 1948 insertions(+), 1400 deletions(-) create mode 100644 .clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000000..cf296d6ec8 --- /dev/null +++ b/.clang-format @@ -0,0 +1,13 @@ +{ + BasedOnStyle: LLVM, + BreakStringLiterals: true, + AllowShortFunctionsOnASingleLine: InlineOnly, + ColumnLimit: 100, + IndentWidth: 4, + SortIncludes: false, + ReflowComments: false, + BinPackArguments: false, + BinPackParameters: false, + TabWidth: 4, + PointerAlignment: Right, +} diff --git a/Makefile b/Makefile index 3003d12ce9..7251749ce9 100644 --- a/Makefile +++ b/Makefile @@ -303,3 +303,8 @@ MARKDOWNIMAGE := $(shell awk '$$4=="markdown" {print $$2}' $(DEPENDENCIES_DOCKER .PHONY: markdown-lint markdown-lint: docker run --rm -u $(DOCKER_USER) -v "$(CURDIR):$(WORKDIR)" $(MARKDOWNIMAGE) -c $(WORKDIR)/.markdownlint.yaml -p $(WORKDIR)/.markdownlintignore $(WORKDIR)/**/*.md + +.PHONY: clang-format +clang-format: + find ./internal -type f -name "*.c" | xargs -P 0 -n 1 clang-format -i + find ./internal -type f -name "*.h" | xargs -P 0 -n 1 clang-format -i diff --git a/internal/include/alloc.h b/internal/include/alloc.h index 6405eacfb0..47ec97963a 100644 --- a/internal/include/alloc.h +++ b/internal/include/alloc.h @@ -15,8 +15,7 @@ volatile const u64 total_cpus; volatile const u64 start_addr; volatile const u64 end_addr; -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_HASH); __type(key, s32); __type(value, u64); @@ -24,65 +23,50 @@ struct __uint(pinning, LIBBPF_PIN_BY_NAME); } alloc_map SEC(".maps"); -static __always_inline u64 get_area_start() -{ +static __always_inline u64 get_area_start() { s64 partition_size = (end_addr - start_addr) / total_cpus; u32 current_cpu = bpf_get_smp_processor_id(); s32 start_index = 0; u64 *start = (u64 *)bpf_map_lookup_elem(&alloc_map, &start_index); - if (start == NULL || *start == 0) - { + if (start == NULL || *start == 0) { u64 current_start_addr = start_addr + (partition_size * current_cpu); bpf_map_update_elem(&alloc_map, &start_index, ¤t_start_addr, BPF_ANY); return current_start_addr; - } - else - { + } else { return *start; } } -static __always_inline u64 get_area_end(u64 start) -{ +static __always_inline u64 get_area_end(u64 start) { s64 partition_size = (end_addr - start_addr) / total_cpus; s32 end_index = 1; u64 *end = (u64 *)bpf_map_lookup_elem(&alloc_map, &end_index); - if (end == NULL || *end == 0) - { + if (end == NULL || *end == 0) { u64 current_end_addr = start + partition_size; bpf_map_update_elem(&alloc_map, &end_index, ¤t_end_addr, BPF_ANY); return current_end_addr; - } - else - { + } else { return *end; } } -static __always_inline s32 bound_number(s32 num, s32 min, s32 max) -{ - if (num < min) - { +static __always_inline s32 bound_number(s32 num, s32 min, s32 max) { + if (num < min) { return min; - } - else if (num > max) - { + } else if (num > max) { return max; } return num; } -static __always_inline void *write_target_data(void *data, s32 size) -{ - if (!data || data == NULL) - { +static __always_inline void *write_target_data(void *data, s32 size) { + if (!data || data == NULL) { return NULL; } u64 start = get_area_start(); u64 end = get_area_end(start); - if (end - start < size) - { + if (end - start < size) { bpf_printk("reached end of CPU memory block, going to the start again"); s32 start_index = 0; bpf_map_delete_elem(&alloc_map, &start_index); @@ -93,8 +77,7 @@ static __always_inline void *write_target_data(void *data, s32 size) size = bound_number(size, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE); u64 page_offset = (u64)target & 0xFFF; u64 dist_to_next_page = 4096 - page_offset; - if (dist_to_next_page < size) - { + if (dist_to_next_page < size) { target += dist_to_next_page; } u64 target_u = (u64)target; @@ -104,8 +87,7 @@ static __always_inline void *write_target_data(void *data, s32 size) } long success = bpf_probe_write_user(target, data, size); - if (success == 0) - { + if (success == 0) { s32 start_index = 0; // Update the start position of this chunk, taking into account possible adjustments // we made to be page aligned @@ -118,10 +100,11 @@ static __always_inline void *write_target_data(void *data, s32 size) bpf_map_update_elem(&alloc_map, &start_index, &updated_start, BPF_ANY); return target; - } - else - { - bpf_printk("failed to write to userspace, error code: %d, addr: %lx, size: %d", success, target, size); + } else { + bpf_printk("failed to write to userspace, error code: %d, addr: %lx, size: %d", + success, + target, + size); return NULL; } } diff --git a/internal/include/arguments.h b/internal/include/arguments.h index 88ef73aeb6..7149689bc5 100644 --- a/internal/include/arguments.h +++ b/internal/include/arguments.h @@ -45,10 +45,8 @@ #endif -static __always_inline void *get_argument(struct pt_regs *ctx, int index) -{ - switch (index) - { +static __always_inline void *get_argument(struct pt_regs *ctx, int index) { + switch (index) { case 1: return (void *)GO_PARAM1(ctx); case 2: diff --git a/internal/include/common.h b/internal/include/common.h index 5900de9488..235846e81e 100644 --- a/internal/include/common.h +++ b/internal/include/common.h @@ -26,45 +26,43 @@ typedef __u32 __be32; typedef __u64 __be64; typedef __u32 __wsum; -enum bpf_map_type -{ - BPF_MAP_TYPE_UNSPEC = 0, - BPF_MAP_TYPE_HASH = 1, - BPF_MAP_TYPE_ARRAY = 2, - BPF_MAP_TYPE_PROG_ARRAY = 3, - BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, - BPF_MAP_TYPE_PERCPU_HASH = 5, - BPF_MAP_TYPE_PERCPU_ARRAY = 6, - BPF_MAP_TYPE_STACK_TRACE = 7, - BPF_MAP_TYPE_CGROUP_ARRAY = 8, - BPF_MAP_TYPE_LRU_HASH = 9, - BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, - BPF_MAP_TYPE_LPM_TRIE = 11, - BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, - BPF_MAP_TYPE_HASH_OF_MAPS = 13, - BPF_MAP_TYPE_DEVMAP = 14, - BPF_MAP_TYPE_SOCKMAP = 15, - BPF_MAP_TYPE_CPUMAP = 16, - BPF_MAP_TYPE_XSKMAP = 17, - BPF_MAP_TYPE_SOCKHASH = 18, - BPF_MAP_TYPE_CGROUP_STORAGE = 19, - BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, - BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, - BPF_MAP_TYPE_QUEUE = 22, - BPF_MAP_TYPE_STACK = 23, - BPF_MAP_TYPE_SK_STORAGE = 24, - BPF_MAP_TYPE_DEVMAP_HASH = 25, - BPF_MAP_TYPE_STRUCT_OPS = 26, - BPF_MAP_TYPE_RINGBUF = 27, - BPF_MAP_TYPE_INODE_STORAGE = 28, +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, }; -enum -{ - BPF_ANY = 0, - BPF_NOEXIST = 1, - BPF_EXIST = 2, - BPF_F_LOCK = 4, +enum { + BPF_ANY = 0, + BPF_NOEXIST = 1, + BPF_EXIST = 2, + BPF_F_LOCK = 4, }; /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and @@ -75,65 +73,65 @@ enum #if defined(__TARGET_ARCH_x86) struct pt_regs { - /* + /* * C ABI says these regs are callee-preserved. They aren't saved on kernel entry * unless syscall needs a complete, fully filled "struct pt_regs". */ - long unsigned int r15; - long unsigned int r14; - long unsigned int r13; - long unsigned int r12; - long unsigned int bp; - long unsigned int bx; - /* These regs are callee-clobbered. Always saved on kernel entry. */ - long unsigned int r11; - long unsigned int r10; - long unsigned int r9; - long unsigned int r8; - long unsigned int ax; - long unsigned int cx; - long unsigned int dx; - long unsigned int si; - long unsigned int di; - /* + long unsigned int r15; + long unsigned int r14; + long unsigned int r13; + long unsigned int r12; + long unsigned int bp; + long unsigned int bx; + /* These regs are callee-clobbered. Always saved on kernel entry. */ + long unsigned int r11; + long unsigned int r10; + long unsigned int r9; + long unsigned int r8; + long unsigned int ax; + long unsigned int cx; + long unsigned int dx; + long unsigned int si; + long unsigned int di; + /* * On syscall entry, this is syscall#. On CPU exception, this is error code. * On hw interrupt, it's IRQ number: */ - long unsigned int orig_ax; - /* Return frame for iretq */ - long unsigned int ip; - long unsigned int cs; - long unsigned int flags; - long unsigned int sp; - long unsigned int ss; - /* top of stack page */ + long unsigned int orig_ax; + /* Return frame for iretq */ + long unsigned int ip; + long unsigned int cs; + long unsigned int flags; + long unsigned int sp; + long unsigned int ss; + /* top of stack page */ }; #elif defined(__TARGET_ARCH_arm64) struct user_pt_regs { - __u64 regs[31]; - __u64 sp; - __u64 pc; - __u64 pstate; + __u64 regs[31]; + __u64 sp; + __u64 pc; + __u64 pstate; }; struct pt_regs { - union { - struct user_pt_regs user_regs; - struct { - u64 regs[31]; - u64 sp; - u64 pc; - u64 pstate; - }; - }; - u64 orig_x0; - s32 syscallno; - u32 unused2; - u64 orig_addr_limit; - u64 pmr_save; - u64 stackframe[2]; - u64 lockdep_hardirqs; - u64 exit_rcu; + union { + struct user_pt_regs user_regs; + struct { + u64 regs[31]; + u64 sp; + u64 pc; + u64 pstate; + }; + }; + u64 orig_x0; + s32 syscallno; + u32 unused2; + u64 orig_addr_limit; + u64 pmr_save; + u64 stackframe[2]; + u64 lockdep_hardirqs; + u64 exit_rcu; }; #endif diff --git a/internal/include/go_context.h b/internal/include/go_context.h index 9747d31ed1..b7be763585 100644 --- a/internal/include/go_context.h +++ b/internal/include/go_context.h @@ -11,8 +11,7 @@ #define MAX_DISTANCE 100 #define MAX_CONCURRENT_SPANS 1000 -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, void *); __type(value, struct span_context); @@ -20,8 +19,7 @@ struct __uint(pinning, LIBBPF_PIN_BY_NAME); } go_context_to_sc SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, struct span_context); __type(value, void *); @@ -31,16 +29,13 @@ struct static __always_inline void *get_parent_go_context(struct go_iface *go_context, void *map) { void *data = go_context->data; - for (int i = 0; i < MAX_DISTANCE; i++) - { - if (data == NULL) - { + for (int i = 0; i < MAX_DISTANCE; i++) { + if (data == NULL) { break; } - + void *found_in_map = bpf_map_lookup_elem(map, &data); - if (found_in_map != NULL) - { + if (found_in_map != NULL) { return data; } @@ -54,14 +49,12 @@ static __always_inline void *get_parent_go_context(struct go_iface *go_context, static __always_inline struct span_context *get_parent_span_context(struct go_iface *go_context) { void *parent_go_ctx = get_parent_go_context(go_context, &go_context_to_sc); - if (parent_go_ctx == NULL) - { + if (parent_go_ctx == NULL) { return NULL; } struct span_context *parent_sc = bpf_map_lookup_elem(&go_context_to_sc, &parent_go_ctx); - if (parent_sc == NULL) - { + if (parent_sc == NULL) { return NULL; } @@ -71,15 +64,13 @@ static __always_inline struct span_context *get_parent_span_context(struct go_if static __always_inline void start_tracking_span(void *contextContext, struct span_context *sc) { long err = 0; err = bpf_map_update_elem(&go_context_to_sc, &contextContext, sc, BPF_ANY); - if (err != 0) - { + if (err != 0) { bpf_printk("Failed to update tracked_spans map: %ld", err); return; } err = bpf_map_update_elem(&tracked_spans_by_sc, sc, &contextContext, BPF_ANY); - if (err != 0) - { + if (err != 0) { bpf_printk("Failed to update tracked_spans_by_sc map: %ld", err); return; } @@ -92,26 +83,22 @@ static __always_inline void stop_tracking_span(struct span_context *sc, struct s } void *ctx = bpf_map_lookup_elem(&tracked_spans_by_sc, sc); - if (ctx == NULL) - { + if (ctx == NULL) { // The span context is not tracked, nothing to do. This can happen for outgoing spans. return; } void *parent_ctx = ((psc == NULL) ? NULL : bpf_map_lookup_elem(&tracked_spans_by_sc, psc)); - if (parent_ctx == NULL) - { + if (parent_ctx == NULL) { // No parent span, delete the context bpf_map_delete_elem(&go_context_to_sc, ctx); - } else - { + } else { void *ctx_val = 0; bpf_probe_read_user(&ctx_val, sizeof(ctx_val), ctx); void *parent_ctx_val = 0; bpf_probe_read_user(&parent_ctx_val, sizeof(parent_ctx_val), parent_ctx); - if (ctx_val != parent_ctx_val) - { + if (ctx_val != parent_ctx_val) { // Parent with different context, delete the context bpf_map_delete_elem(&go_context_to_sc, ctx); } else { @@ -128,22 +115,28 @@ static __always_inline void stop_tracking_span(struct span_context *sc, struct s // In case the context.Context is passed as an argument, // this is the argument index of the pointer (starting from 1). // In case the context.Context is a member of a struct, -// this is the argument index of the struct pointer (starting from 1). +// this is the argument index of the struct pointer (starting from 1). // context_offset: // In case the context.Context is a member of a struct, // this is the offset of the context.Context member in the struct. // passed_as_arg: // Indicates whether context.Context is passed as an argument or is a member of a struct -static __always_inline void get_Go_context(void *ctx, int context_pos, const volatile u64 context_offset, bool passed_as_arg, struct go_iface *contextContext) { +static __always_inline void get_Go_context(void *ctx, + int context_pos, + const volatile u64 context_offset, + bool passed_as_arg, + struct go_iface *contextContext) { // Read the argument which is either the context.Context type pointer or pointer to a struct containing the context.Context void *ctx_type_or_struct = get_argument(ctx, context_pos); if (passed_as_arg) { contextContext->type = ctx_type_or_struct; contextContext->data = get_argument(ctx, context_pos + 1); } else { - void *context_struct_ptr = (void*)(ctx_type_or_struct + context_offset); - bpf_probe_read(&contextContext->type, sizeof(contextContext->type),context_struct_ptr ); - bpf_probe_read(&contextContext->data, sizeof(contextContext->data), get_go_interface_instance(context_struct_ptr)); + void *context_struct_ptr = (void *)(ctx_type_or_struct + context_offset); + bpf_probe_read(&contextContext->type, sizeof(contextContext->type), context_struct_ptr); + bpf_probe_read(&contextContext->data, + sizeof(contextContext->data), + get_go_interface_instance(context_struct_ptr)); } } diff --git a/internal/include/go_net.h b/internal/include/go_net.h index 0d7baf0260..fe58c49c53 100644 --- a/internal/include/go_net.h +++ b/internal/include/go_net.h @@ -22,7 +22,8 @@ type TCPAddr struct { const volatile u64 TCPAddr_IP_offset; const volatile u64 TCPAddr_Port_offset; -static __always_inline long get_tcp_net_addr_from_tcp_addr(struct pt_regs *ctx, net_addr_t *addr, void* tcpAddr_ptr) { +static __always_inline long +get_tcp_net_addr_from_tcp_addr(struct pt_regs *ctx, net_addr_t *addr, void *tcpAddr_ptr) { go_slice_t ip; long res = bpf_probe_read_user(&ip, sizeof(ip), (void *)(tcpAddr_ptr + TCPAddr_IP_offset)); if (res != 0) { @@ -46,7 +47,8 @@ static __always_inline long get_tcp_net_addr_from_tcp_addr(struct pt_regs *ctx, return res; } - res = bpf_probe_read_user(&addr->port, sizeof(addr->port), (void *)(tcpAddr_ptr + TCPAddr_Port_offset)); + res = bpf_probe_read_user( + &addr->port, sizeof(addr->port), (void *)(tcpAddr_ptr + TCPAddr_Port_offset)); if (res != 0) { bpf_printk("failed to read port"); } diff --git a/internal/include/go_types.h b/internal/include/go_types.h index 3351533d5c..ae49e939ae 100644 --- a/internal/include/go_types.h +++ b/internal/include/go_types.h @@ -12,21 +12,18 @@ Keep a power of 2 to help with masks */ #define MAX_SLICE_ARRAY_SIZE 1024 -typedef struct go_string -{ +typedef struct go_string { char *str; s64 len; } go_string_t; -typedef struct go_slice -{ +typedef struct go_slice { void *array; s64 len; s64 cap; } go_slice_t; -typedef struct go_iface -{ +typedef struct go_iface { void *type; void *data; } go_iface_t; @@ -36,21 +33,19 @@ typedef struct go_iface // a map bucket struct definition with the given key and value types // for more details about the structure of a map bucket see: // https://github.com/golang/go/blob/639cc0dcc0948dd02c9d5fc12fbed730a21ebebc/src/runtime/map.go#L143 -#define MAP_BUCKET_DEFINITION(key_type, value_type) \ -MAP_BUCKET_TYPE(key_type, value_type) { \ - char tophash[8]; \ - key_type keys[8]; \ - value_type values[8]; \ - void *overflow; \ -}; - -struct slice_array_buff -{ +#define MAP_BUCKET_DEFINITION(key_type, value_type) \ + MAP_BUCKET_TYPE(key_type, value_type) { \ + char tophash[8]; \ + key_type keys[8]; \ + value_type values[8]; \ + void *overflow; \ + }; + +struct slice_array_buff { unsigned char buff[MAX_SLICE_ARRAY_SIZE]; }; -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __type(key, u32); __type(value, struct slice_array_buff); @@ -60,13 +55,11 @@ struct // In Go, interfaces are represented as a pair of pointers: a pointer to the // interface data, and a pointer to the interface table. // See: runtime.iface in https://golang.org/src/runtime/runtime2.go -static __always_inline void* get_go_interface_instance(void *iface) -{ - return (void*)(iface + 8); +static __always_inline void *get_go_interface_instance(void *iface) { + return (void *)(iface + 8); } -static __always_inline struct go_string write_user_go_string(char *str, u32 len) -{ +static __always_inline struct go_string write_user_go_string(char *str, u32 len) { // Copy chars to userspace struct go_string new_string = {.str = NULL, .len = 0}; char *addr = write_target_data((void *)str, len); @@ -88,8 +81,8 @@ static __always_inline struct go_string write_user_go_string(char *str, u32 len) return new_string; } -static __always_inline void append_item_to_slice(void *new_item, u32 item_size, void *slice_user_ptr) -{ +static __always_inline void +append_item_to_slice(void *new_item, u32 item_size, void *slice_user_ptr) { // read the slice descriptor struct go_slice slice = {0}; bpf_probe_read(&slice, sizeof(slice), slice_user_ptr); @@ -97,40 +90,36 @@ static __always_inline void append_item_to_slice(void *new_item, u32 item_size, u64 slice_len = slice.len; u64 slice_cap = slice.cap; - if (slice_len < slice_cap && slice.array != NULL) - { + if (slice_len < slice_cap && slice.array != NULL) { // Room available on current array, append to the underlying array res = bpf_probe_write_user(slice.array + (item_size * slice_len), new_item, item_size); - } - else - { + } else { // No room on current array - try to copy new one of size item_size * (len + 1) u32 alloc_size = item_size * slice_len; - if (alloc_size >= MAX_SLICE_ARRAY_SIZE) - { + if (alloc_size >= MAX_SLICE_ARRAY_SIZE) { return; } - + // Get temporary buffer u32 index = 0; struct slice_array_buff *map_buff = bpf_map_lookup_elem(&slice_array_buff_map, &index); - if (!map_buff) - { + if (!map_buff) { return; } - + unsigned char *new_slice_array = map_buff->buff; // help the verifier alloc_size &= (MAX_SLICE_ARRAY_SIZE - 1); - if (alloc_size + item_size > MAX_SLICE_ARRAY_SIZE) - { + if (alloc_size + item_size > MAX_SLICE_ARRAY_SIZE) { // No room for new item return; } // Append to buffer if (slice.array != NULL) { bpf_probe_read_user(new_slice_array, alloc_size, slice.array); - bpf_printk("append_item_to_slice: copying %d bytes to new array from address 0x%llx", alloc_size, slice.array); + bpf_printk("append_item_to_slice: copying %d bytes to new array from address 0x%llx", + alloc_size, + slice.array); } copy_byte_arrays(new_item, new_slice_array + alloc_size, item_size); @@ -138,8 +127,7 @@ static __always_inline void append_item_to_slice(void *new_item, u32 item_size, u32 new_array_size = alloc_size + item_size; void *new_array = write_target_data(new_slice_array, new_array_size); - if (new_array == NULL) - { + if (new_array == NULL) { bpf_printk("append_item_to_slice: failed to copy new array to userspace"); return; } @@ -152,15 +140,14 @@ static __always_inline void append_item_to_slice(void *new_item, u32 item_size, // Update len slice.len++; long success = bpf_probe_write_user(slice_user_ptr, &slice, sizeof(slice)); - if (success != 0) - { + if (success != 0) { bpf_printk("append_item_to_slice: failed to update slice in userspace"); return; } } -static __always_inline bool get_go_string_from_user_ptr(void *user_str_ptr, char *dst, u64 max_len) -{ +static __always_inline bool +get_go_string_from_user_ptr(void *user_str_ptr, char *dst, u64 max_len) { if (user_str_ptr == NULL || max_len == 0) { return false; } @@ -174,7 +161,7 @@ static __always_inline bool get_go_string_from_user_ptr(void *user_str_ptr, char u64 size_to_read = user_str.len > max_len ? max_len : user_str.len; __builtin_memset(dst, 0, max_len); - + success = bpf_probe_read_user(dst, size_to_read, user_str.str); if (success != 0) { return false; diff --git a/internal/include/libbpf/bpf_helper_defs.h b/internal/include/libbpf/bpf_helper_defs.h index 6f13d1f1d8..77cf642236 100644 --- a/internal/include/libbpf/bpf_helper_defs.h +++ b/internal/include/libbpf/bpf_helper_defs.h @@ -53,7 +53,7 @@ struct ipv6hdr; * Map value associated to *key*, or **NULL** if no entry was * found. */ -static void *(* const bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; +static void *(*const bpf_map_lookup_elem)(void *map, const void *key) = (void *)1; /* * bpf_map_update_elem @@ -75,7 +75,10 @@ static void *(* const bpf_map_lookup_elem)(void *map, const void *key) = (void * * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_map_update_elem)(void *map, const void *key, const void *value, __u64 flags) = (void *) 2; +static long (*const bpf_map_update_elem)(void *map, + const void *key, + const void *value, + __u64 flags) = (void *)2; /* * bpf_map_delete_elem @@ -85,7 +88,7 @@ static long (* const bpf_map_update_elem)(void *map, const void *key, const void * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; +static long (*const bpf_map_delete_elem)(void *map, const void *key) = (void *)3; /* * bpf_probe_read @@ -99,7 +102,7 @@ static long (* const bpf_map_delete_elem)(void *map, const void *key) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 4; +static long (*const bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *)4; /* * bpf_ktime_get_ns @@ -111,7 +114,7 @@ static long (* const bpf_probe_read)(void *dst, __u32 size, const void *unsafe_p * Returns * Current *ktime*. */ -static __u64 (* const bpf_ktime_get_ns)(void) = (void *) 5; +static __u64 (*const bpf_ktime_get_ns)(void) = (void *)5; /* * bpf_trace_printk @@ -174,7 +177,7 @@ static __u64 (* const bpf_ktime_get_ns)(void) = (void *) 5; * The number of bytes written to the buffer, or a negative error * in case of failure. */ -static long (* const bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) 6; +static long (*const bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *)6; /* * bpf_get_prandom_u32 @@ -190,7 +193,7 @@ static long (* const bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = ( * Returns * A random 32-bit unsigned value. */ -static __u32 (* const bpf_get_prandom_u32)(void) = (void *) 7; +static __u32 (*const bpf_get_prandom_u32)(void) = (void *)7; /* * bpf_get_smp_processor_id @@ -203,7 +206,7 @@ static __u32 (* const bpf_get_prandom_u32)(void) = (void *) 7; * Returns * The SMP id of the processor running the program. */ -static __u32 (* const bpf_get_smp_processor_id)(void) = (void *) 8; +static __u32 (*const bpf_get_smp_processor_id)(void) = (void *)8; /* * bpf_skb_store_bytes @@ -224,7 +227,8 @@ static __u32 (* const bpf_get_smp_processor_id)(void) = (void *) 8; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *) 9; +static long (*const bpf_skb_store_bytes)( + struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *)9; /* * bpf_l3_csum_replace @@ -253,7 +257,8 @@ static long (* const bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, c * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *) 10; +static long (*const bpf_l3_csum_replace)( + struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *)10; /* * bpf_l4_csum_replace @@ -289,7 +294,8 @@ static long (* const bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, _ * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *) 11; +static long (*const bpf_l4_csum_replace)( + struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *)11; /* * bpf_tail_call @@ -324,7 +330,7 @@ static long (* const bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, _ * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *) 12; +static long (*const bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *)12; /* * bpf_clone_redirect @@ -354,7 +360,9 @@ static long (* const bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index * error indicates a potential drop or congestion in the target * device. The particular positive error codes are not defined. */ -static long (* const bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 flags) = (void *) 13; +static long (*const bpf_clone_redirect)(struct __sk_buff *skb, + __u32 ifindex, + __u64 flags) = (void *)13; /* * bpf_get_current_pid_tgid @@ -367,7 +375,7 @@ static long (* const bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, _ * *current_task*\ **->tgid << 32 \|** * *current_task*\ **->pid**. */ -static __u64 (* const bpf_get_current_pid_tgid)(void) = (void *) 14; +static __u64 (*const bpf_get_current_pid_tgid)(void) = (void *)14; /* * bpf_get_current_uid_gid @@ -378,7 +386,7 @@ static __u64 (* const bpf_get_current_pid_tgid)(void) = (void *) 14; * A 64-bit integer containing the current GID and UID, and * created as such: *current_gid* **<< 32 \|** *current_uid*. */ -static __u64 (* const bpf_get_current_uid_gid)(void) = (void *) 15; +static __u64 (*const bpf_get_current_uid_gid)(void) = (void *)15; /* * bpf_get_current_comm @@ -393,7 +401,7 @@ static __u64 (* const bpf_get_current_uid_gid)(void) = (void *) 15; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; +static long (*const bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *)16; /* * bpf_get_cgroup_classid @@ -423,7 +431,7 @@ static long (* const bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void * Returns * The classid, or 0 for the default unconfigured classid. */ -static __u32 (* const bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; +static __u32 (*const bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *)17; /* * bpf_skb_vlan_push @@ -443,7 +451,9 @@ static __u32 (* const bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 vlan_tci) = (void *) 18; +static long (*const bpf_skb_vlan_push)(struct __sk_buff *skb, + __be16 vlan_proto, + __u16 vlan_tci) = (void *)18; /* * bpf_skb_vlan_pop @@ -459,7 +469,7 @@ static long (* const bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; +static long (*const bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *)19; /* * bpf_skb_get_tunnel_key @@ -514,7 +524,10 @@ static long (* const bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 20; +static long (*const bpf_skb_get_tunnel_key)(struct __sk_buff *skb, + struct bpf_tunnel_key *key, + __u32 size, + __u64 flags) = (void *)20; /* * bpf_skb_set_tunnel_key @@ -558,7 +571,10 @@ static long (* const bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_t * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 21; +static long (*const bpf_skb_set_tunnel_key)(struct __sk_buff *skb, + struct bpf_tunnel_key *key, + __u32 size, + __u64 flags) = (void *)21; /* * bpf_perf_event_read @@ -591,7 +607,7 @@ static long (* const bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_t * The value of the perf event counter read from the map, or a * negative error code in case of failure. */ -static __u64 (* const bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; +static __u64 (*const bpf_perf_event_read)(void *map, __u64 flags) = (void *)22; /* * bpf_redirect @@ -618,7 +634,7 @@ static __u64 (* const bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22 * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on * error. */ -static long (* const bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; +static long (*const bpf_redirect)(__u32 ifindex, __u64 flags) = (void *)23; /* * bpf_get_route_realm @@ -646,7 +662,7 @@ static long (* const bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; * The realm of the route for the packet associated to *skb*, or 0 * if none was found. */ -static __u32 (* const bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; +static __u32 (*const bpf_get_route_realm)(struct __sk_buff *skb) = (void *)24; /* * bpf_perf_event_output @@ -695,7 +711,8 @@ static __u32 (* const bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 25; +static long (*const bpf_perf_event_output)( + void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *)25; /* * bpf_skb_load_bytes @@ -716,7 +733,10 @@ static long (* const bpf_perf_event_output)(void *ctx, void *map, __u64 flags, v * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 len) = (void *) 26; +static long (*const bpf_skb_load_bytes)(const void *skb, + __u32 offset, + void *to, + __u32 len) = (void *)26; /* * bpf_get_stackid @@ -762,7 +782,7 @@ static long (* const bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to * The positive or null stack id on success, or a negative error * in case of failure. */ -static long (* const bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; +static long (*const bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *)27; /* * bpf_csum_diff @@ -793,7 +813,8 @@ static long (* const bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void * The checksum result, or a negative error code in case of * failure. */ -static __s64 (* const bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *) 28; +static __s64 (*const bpf_csum_diff)( + __be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *)28; /* * bpf_skb_get_tunnel_opt @@ -815,7 +836,9 @@ static __s64 (* const bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, * Returns * The size of the option data retrieved. */ -static long (* const bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 29; +static long (*const bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, + void *opt, + __u32 size) = (void *)29; /* * bpf_skb_set_tunnel_opt @@ -829,7 +852,9 @@ static long (* const bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, _ * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 30; +static long (*const bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, + void *opt, + __u32 size) = (void *)30; /* * bpf_skb_change_proto @@ -860,7 +885,9 @@ static long (* const bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, _ * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 flags) = (void *) 31; +static long (*const bpf_skb_change_proto)(struct __sk_buff *skb, + __be16 proto, + __u64 flags) = (void *)31; /* * bpf_skb_change_type @@ -891,7 +918,7 @@ static long (* const bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) 32; +static long (*const bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *)32; /* * bpf_skb_under_cgroup @@ -906,7 +933,9 @@ static long (* const bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = ( * * 1, if the *skb* succeeded the cgroup2 descendant test. * * A negative error code, if an error occurred. */ -static long (* const bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 index) = (void *) 33; +static long (*const bpf_skb_under_cgroup)(struct __sk_buff *skb, + void *map, + __u32 index) = (void *)33; /* * bpf_get_hash_recalc @@ -926,7 +955,7 @@ static long (* const bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u * Returns * The 32-bit hash. */ -static __u32 (* const bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; +static __u32 (*const bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *)34; /* * bpf_get_current_task @@ -936,7 +965,7 @@ static __u32 (* const bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; * Returns * A pointer to the current task struct. */ -static __u64 (* const bpf_get_current_task)(void) = (void *) 35; +static __u64 (*const bpf_get_current_task)(void) = (void *)35; /* * bpf_probe_write_user @@ -959,7 +988,7 @@ static __u64 (* const bpf_get_current_task)(void) = (void *) 35; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *) 36; +static long (*const bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *)36; /* * bpf_current_task_under_cgroup @@ -975,7 +1004,7 @@ static long (* const bpf_probe_write_user)(void *dst, const void *src, __u32 len * * 0, if current task does not belong to the cgroup2. * * A negative error code, if an error occurred. */ -static long (* const bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37; +static long (*const bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *)37; /* * bpf_skb_change_tail @@ -1003,7 +1032,9 @@ static long (* const bpf_current_task_under_cgroup)(void *map, __u32 index) = (v * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 38; +static long (*const bpf_skb_change_tail)(struct __sk_buff *skb, + __u32 len, + __u64 flags) = (void *)38; /* * bpf_skb_pull_data @@ -1044,7 +1075,7 @@ static long (* const bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u6 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39; +static long (*const bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *)39; /* * bpf_csum_update @@ -1060,7 +1091,7 @@ static long (* const bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (voi * The checksum on success, or a negative error code in case of * failure. */ -static __s64 (* const bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 40; +static __s64 (*const bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *)40; /* * bpf_set_hash_invalid @@ -1074,7 +1105,7 @@ static __s64 (* const bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (vo * Returns * void. */ -static void (* const bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; +static void (*const bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *)41; /* * bpf_get_numa_node_id @@ -1089,7 +1120,7 @@ static void (* const bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; * Returns * The id of current NUMA node. */ -static long (* const bpf_get_numa_node_id)(void) = (void *) 42; +static long (*const bpf_get_numa_node_id)(void) = (void *)42; /* * bpf_skb_change_head @@ -1114,7 +1145,9 @@ static long (* const bpf_get_numa_node_id)(void) = (void *) 42; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 43; +static long (*const bpf_skb_change_head)(struct __sk_buff *skb, + __u32 len, + __u64 flags) = (void *)43; /* * bpf_xdp_adjust_head @@ -1133,7 +1166,7 @@ static long (* const bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u6 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) 44; +static long (*const bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *)44; /* * bpf_probe_read_str @@ -1150,7 +1183,7 @@ static long (* const bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (v * including the trailing NUL character. On error, a negative * value. */ -static long (* const bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 45; +static long (*const bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *)45; /* * bpf_get_socket_cookie @@ -1167,7 +1200,7 @@ static long (* const bpf_probe_read_str)(void *dst, __u32 size, const void *unsa * A 8-byte long unique number on success, or 0 if the socket * field is missing inside *skb*. */ -static __u64 (* const bpf_get_socket_cookie)(void *ctx) = (void *) 46; +static __u64 (*const bpf_get_socket_cookie)(void *ctx) = (void *)46; /* * bpf_get_socket_uid @@ -1181,7 +1214,7 @@ static __u64 (* const bpf_get_socket_cookie)(void *ctx) = (void *) 46; * is returned (note that **overflowuid** might also be the actual * UID value for the socket). */ -static __u32 (* const bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; +static __u32 (*const bpf_get_socket_uid)(struct __sk_buff *skb) = (void *)47; /* * bpf_set_hash @@ -1192,7 +1225,7 @@ static __u32 (* const bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; * Returns * 0 */ -static long (* const bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; +static long (*const bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *)48; /* * bpf_setsockopt @@ -1232,7 +1265,8 @@ static long (* const bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_setsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 49; +static long (*const bpf_setsockopt)( + void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *)49; /* * bpf_skb_adjust_room @@ -1293,7 +1327,10 @@ static long (* const bpf_setsockopt)(void *bpf_socket, int level, int optname, v * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 mode, __u64 flags) = (void *) 50; +static long (*const bpf_skb_adjust_room)(struct __sk_buff *skb, + __s32 len_diff, + __u32 mode, + __u64 flags) = (void *)50; /* * bpf_redirect_map @@ -1322,7 +1359,7 @@ static long (* const bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, * **XDP_REDIRECT** on success, or the value of the two lower bits * of the *flags* argument on error. */ -static long (* const bpf_redirect_map)(void *map, __u64 key, __u64 flags) = (void *) 51; +static long (*const bpf_redirect_map)(void *map, __u64 key, __u64 flags) = (void *)51; /* * bpf_sk_redirect_map @@ -1337,7 +1374,10 @@ static long (* const bpf_redirect_map)(void *map, __u64 key, __u64 flags) = (voi * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (* const bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, __u64 flags) = (void *) 52; +static long (*const bpf_sk_redirect_map)(struct __sk_buff *skb, + void *map, + __u32 key, + __u64 flags) = (void *)52; /* * bpf_sock_map_update @@ -1360,7 +1400,10 @@ static long (* const bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u3 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 53; +static long (*const bpf_sock_map_update)(struct bpf_sock_ops *skops, + void *map, + void *key, + __u64 flags) = (void *)53; /* * bpf_xdp_adjust_meta @@ -1393,7 +1436,7 @@ static long (* const bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54; +static long (*const bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *)54; /* * bpf_perf_event_read_value @@ -1447,7 +1490,10 @@ static long (* const bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (v * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 55; +static long (*const bpf_perf_event_read_value)(void *map, + __u64 flags, + struct bpf_perf_event_value *buf, + __u32 buf_size) = (void *)55; /* * bpf_perf_prog_read_value @@ -1462,7 +1508,9 @@ static long (* const bpf_perf_event_read_value)(void *map, __u64 flags, struct b * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 56; +static long (*const bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, + struct bpf_perf_event_value *buf, + __u32 buf_size) = (void *)56; /* * bpf_getsockopt @@ -1489,7 +1537,8 @@ static long (* const bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_getsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 57; +static long (*const bpf_getsockopt)( + void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *)57; /* * bpf_override_return @@ -1518,7 +1567,7 @@ static long (* const bpf_getsockopt)(void *bpf_socket, int level, int optname, v * Returns * 0 */ -static long (* const bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58; +static long (*const bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *)58; /* * bpf_sock_ops_cb_flags_set @@ -1566,7 +1615,8 @@ static long (* const bpf_override_return)(struct pt_regs *regs, __u64 rc) = (voi * be set is returned (which comes down to 0 if all bits were set * as required). */ -static long (* const bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argval) = (void *) 59; +static long (*const bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, + int argval) = (void *)59; /* * bpf_msg_redirect_map @@ -1584,7 +1634,10 @@ static long (* const bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, i * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (* const bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, __u64 flags) = (void *) 60; +static long (*const bpf_msg_redirect_map)(struct sk_msg_md *msg, + void *map, + __u32 key, + __u64 flags) = (void *)60; /* * bpf_msg_apply_bytes @@ -1622,7 +1675,7 @@ static long (* const bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u * Returns * 0 */ -static long (* const bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 61; +static long (*const bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *)61; /* * bpf_msg_cork_bytes @@ -1644,7 +1697,7 @@ static long (* const bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = * Returns * 0 */ -static long (* const bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 62; +static long (*const bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *)62; /* * bpf_msg_pull_data @@ -1679,7 +1732,10 @@ static long (* const bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = ( * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, __u64 flags) = (void *) 63; +static long (*const bpf_msg_pull_data)(struct sk_msg_md *msg, + __u32 start, + __u32 end, + __u64 flags) = (void *)63; /* * bpf_bind @@ -1701,7 +1757,9 @@ static long (* const bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u3 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) = (void *) 64; +static long (*const bpf_bind)(struct bpf_sock_addr *ctx, + struct sockaddr *addr, + int addr_len) = (void *)64; /* * bpf_xdp_adjust_tail @@ -1719,7 +1777,7 @@ static long (* const bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) 65; +static long (*const bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *)65; /* * bpf_skb_get_xfrm_state @@ -1739,7 +1797,11 @@ static long (* const bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (v * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct bpf_xfrm_state *xfrm_state, __u32 size, __u64 flags) = (void *) 66; +static long (*const bpf_skb_get_xfrm_state)(struct __sk_buff *skb, + __u32 index, + struct bpf_xfrm_state *xfrm_state, + __u32 size, + __u64 flags) = (void *)66; /* * bpf_get_stack @@ -1786,7 +1848,7 @@ static long (* const bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. */ -static long (* const bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67; +static long (*const bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *)67; /* * bpf_skb_load_bytes_relative @@ -1812,7 +1874,8 @@ static long (* const bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flag * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *) 68; +static long (*const bpf_skb_load_bytes_relative)( + const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *)68; /* * bpf_fib_lookup @@ -1864,7 +1927,10 @@ static long (* const bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU * was exceeded and output params->mtu_result contains the MTU. */ -static long (* const bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen, __u32 flags) = (void *) 69; +static long (*const bpf_fib_lookup)(void *ctx, + struct bpf_fib_lookup *params, + int plen, + __u32 flags) = (void *)69; /* * bpf_sock_hash_update @@ -1887,7 +1953,10 @@ static long (* const bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, i * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 70; +static long (*const bpf_sock_hash_update)(struct bpf_sock_ops *skops, + void *map, + void *key, + __u64 flags) = (void *)70; /* * bpf_msg_redirect_hash @@ -1905,7 +1974,10 @@ static long (* const bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (* const bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key, __u64 flags) = (void *) 71; +static long (*const bpf_msg_redirect_hash)(struct sk_msg_md *msg, + void *map, + void *key, + __u64 flags) = (void *)71; /* * bpf_sk_redirect_hash @@ -1923,7 +1995,10 @@ static long (* const bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, vo * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (* const bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, __u64 flags) = (void *) 72; +static long (*const bpf_sk_redirect_hash)(struct __sk_buff *skb, + void *map, + void *key, + __u64 flags) = (void *)72; /* * bpf_lwt_push_encap @@ -1964,7 +2039,10 @@ static long (* const bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, voi * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, __u32 len) = (void *) 73; +static long (*const bpf_lwt_push_encap)(struct __sk_buff *skb, + __u32 type, + void *hdr, + __u32 len) = (void *)73; /* * bpf_lwt_seg6_store_bytes @@ -1983,7 +2061,10 @@ static long (* const bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len) = (void *) 74; +static long (*const bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, + __u32 offset, + const void *from, + __u32 len) = (void *)74; /* * bpf_lwt_seg6_adjust_srh @@ -2003,7 +2084,9 @@ static long (* const bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offs * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s32 delta) = (void *) 75; +static long (*const bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, + __u32 offset, + __s32 delta) = (void *)75; /* * bpf_lwt_seg6_action @@ -2036,7 +2119,10 @@ static long (* const bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offse * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *param, __u32 param_len) = (void *) 76; +static long (*const bpf_lwt_seg6_action)(struct __sk_buff *skb, + __u32 action, + void *param, + __u32 param_len) = (void *)76; /* * bpf_rc_repeat @@ -2059,7 +2145,7 @@ static long (* const bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, v * Returns * 0 */ -static long (* const bpf_rc_repeat)(void *ctx) = (void *) 77; +static long (*const bpf_rc_repeat)(void *ctx) = (void *)77; /* * bpf_rc_keydown @@ -2089,7 +2175,10 @@ static long (* const bpf_rc_repeat)(void *ctx) = (void *) 77; * Returns * 0 */ -static long (* const bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void *) 78; +static long (*const bpf_rc_keydown)(void *ctx, + __u32 protocol, + __u64 scancode, + __u32 toggle) = (void *)78; /* * bpf_skb_cgroup_id @@ -2109,7 +2198,7 @@ static long (* const bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; +static __u64 (*const bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *)79; /* * bpf_get_current_cgroup_id @@ -2121,7 +2210,7 @@ static __u64 (* const bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; * A 64-bit integer containing the current cgroup id based * on the cgroup within which the current task is running. */ -static __u64 (* const bpf_get_current_cgroup_id)(void) = (void *) 80; +static __u64 (*const bpf_get_current_cgroup_id)(void) = (void *)80; /* * bpf_get_local_storage @@ -2143,7 +2232,7 @@ static __u64 (* const bpf_get_current_cgroup_id)(void) = (void *) 80; * Returns * A pointer to the local storage area. */ -static void *(* const bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; +static void *(*const bpf_get_local_storage)(void *map, __u64 flags) = (void *)81; /* * bpf_sk_select_reuseport @@ -2156,7 +2245,10 @@ static void *(* const bpf_get_local_storage)(void *map, __u64 flags) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, void *key, __u64 flags) = (void *) 82; +static long (*const bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, + void *map, + void *key, + __u64 flags) = (void *)82; /* * bpf_skb_ancestor_cgroup_id @@ -2178,7 +2270,8 @@ static long (* const bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, voi * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_level) = (void *) 83; +static __u64 (*const bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, + int ancestor_level) = (void *)83; /* * bpf_sk_lookup_tcp @@ -2219,7 +2312,11 @@ static __u64 (* const bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int anc * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(* const bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 84; +static struct bpf_sock *(*const bpf_sk_lookup_tcp)(void *ctx, + struct bpf_sock_tuple *tuple, + __u32 tuple_size, + __u64 netns, + __u64 flags) = (void *)84; /* * bpf_sk_lookup_udp @@ -2260,7 +2357,11 @@ static struct bpf_sock *(* const bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_t * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(* const bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 85; +static struct bpf_sock *(*const bpf_sk_lookup_udp)(void *ctx, + struct bpf_sock_tuple *tuple, + __u32 tuple_size, + __u64 netns, + __u64 flags) = (void *)85; /* * bpf_sk_release @@ -2272,7 +2373,7 @@ static struct bpf_sock *(* const bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_t * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_sk_release)(void *sock) = (void *) 86; +static long (*const bpf_sk_release)(void *sock) = (void *)86; /* * bpf_map_push_elem @@ -2286,7 +2387,7 @@ static long (* const bpf_sk_release)(void *sock) = (void *) 86; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *) 87; +static long (*const bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *)87; /* * bpf_map_pop_elem @@ -2296,7 +2397,7 @@ static long (* const bpf_map_push_elem)(void *map, const void *value, __u64 flag * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_map_pop_elem)(void *map, void *value) = (void *) 88; +static long (*const bpf_map_pop_elem)(void *map, void *value) = (void *)88; /* * bpf_map_peek_elem @@ -2306,7 +2407,7 @@ static long (* const bpf_map_pop_elem)(void *map, void *value) = (void *) 88; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_map_peek_elem)(void *map, void *value) = (void *) 89; +static long (*const bpf_map_peek_elem)(void *map, void *value) = (void *)89; /* * bpf_msg_push_data @@ -2326,7 +2427,10 @@ static long (* const bpf_map_peek_elem)(void *map, void *value) = (void *) 89; * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 90; +static long (*const bpf_msg_push_data)(struct sk_msg_md *msg, + __u32 start, + __u32 len, + __u64 flags) = (void *)90; /* * bpf_msg_pop_data @@ -2342,7 +2446,10 @@ static long (* const bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u3 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 91; +static long (*const bpf_msg_pop_data)(struct sk_msg_md *msg, + __u32 start, + __u32 len, + __u64 flags) = (void *)91; /* * bpf_rc_pointer_rel @@ -2360,7 +2467,7 @@ static long (* const bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 * Returns * 0 */ -static long (* const bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *) 92; +static long (*const bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *)92; /* * bpf_spin_lock @@ -2412,7 +2519,7 @@ static long (* const bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = * Returns * 0 */ -static long (* const bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; +static long (*const bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *)93; /* * bpf_spin_unlock @@ -2423,7 +2530,7 @@ static long (* const bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; * Returns * 0 */ -static long (* const bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; +static long (*const bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *)94; /* * bpf_sk_fullsock @@ -2435,7 +2542,7 @@ static long (* const bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; * A **struct bpf_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_sock *(* const bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; +static struct bpf_sock *(*const bpf_sk_fullsock)(struct bpf_sock *sk) = (void *)95; /* * bpf_tcp_sock @@ -2447,7 +2554,7 @@ static struct bpf_sock *(* const bpf_sk_fullsock)(struct bpf_sock *sk) = (void * * A **struct bpf_tcp_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_tcp_sock *(* const bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; +static struct bpf_tcp_sock *(*const bpf_tcp_sock)(struct bpf_sock *sk) = (void *)96; /* * bpf_skb_ecn_set_ce @@ -2461,7 +2568,7 @@ static struct bpf_tcp_sock *(* const bpf_tcp_sock)(struct bpf_sock *sk) = (void * 1 if the **CE** flag is set (either by the current helper call * or because it was already present), 0 if it is not set. */ -static long (* const bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; +static long (*const bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *)97; /* * bpf_get_listener_sock @@ -2473,7 +2580,7 @@ static long (* const bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; * A **struct bpf_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_sock *(* const bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) 98; +static struct bpf_sock *(*const bpf_get_listener_sock)(struct bpf_sock *sk) = (void *)98; /* * bpf_skc_lookup_tcp @@ -2496,7 +2603,11 @@ static struct bpf_sock *(* const bpf_get_listener_sock)(struct bpf_sock *sk) = ( * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(* const bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 99; +static struct bpf_sock *(*const bpf_skc_lookup_tcp)(void *ctx, + struct bpf_sock_tuple *tuple, + __u32 tuple_size, + __u64 netns, + __u64 flags) = (void *)99; /* * bpf_tcp_check_syncookie @@ -2516,7 +2627,8 @@ static struct bpf_sock *(* const bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_ * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative * error otherwise. */ -static long (* const bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 100; +static long (*const bpf_tcp_check_syncookie)( + void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *)100; /* * bpf_sysctl_get_name @@ -2536,7 +2648,10 @@ static long (* const bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len * **-E2BIG** if the buffer wasn't big enough (*buf* will contain * truncated name in this case). */ -static long (* const bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len, __u64 flags) = (void *) 101; +static long (*const bpf_sysctl_get_name)(struct bpf_sysctl *ctx, + char *buf, + unsigned long buf_len, + __u64 flags) = (void *)101; /* * bpf_sysctl_get_current_value @@ -2559,7 +2674,9 @@ static long (* const bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, uns * **-EINVAL** if current value was unavailable, e.g. because * sysctl is uninitialized and read returns -EIO for it. */ -static long (* const bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 102; +static long (*const bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, + char *buf, + unsigned long buf_len) = (void *)102; /* * bpf_sysctl_get_new_value @@ -2580,7 +2697,9 @@ static long (* const bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char * * **-EINVAL** if sysctl is being read. */ -static long (* const bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 103; +static long (*const bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, + char *buf, + unsigned long buf_len) = (void *)103; /* * bpf_sysctl_set_new_value @@ -2601,7 +2720,9 @@ static long (* const bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf * * **-EINVAL** if sysctl is being read. */ -static long (* const bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, unsigned long buf_len) = (void *) 104; +static long (*const bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, + const char *buf, + unsigned long buf_len) = (void *)104; /* * bpf_strtol @@ -2629,7 +2750,10 @@ static long (* const bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const cha * * **-ERANGE** if resulting value was out of range. */ -static long (* const bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, long *res) = (void *) 105; +static long (*const bpf_strtol)(const char *buf, + unsigned long buf_len, + __u64 flags, + long *res) = (void *)105; /* * bpf_strtoul @@ -2656,7 +2780,10 @@ static long (* const bpf_strtol)(const char *buf, unsigned long buf_len, __u64 f * * **-ERANGE** if resulting value was out of range. */ -static long (* const bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, unsigned long *res) = (void *) 106; +static long (*const bpf_strtoul)(const char *buf, + unsigned long buf_len, + __u64 flags, + unsigned long *res) = (void *)106; /* * bpf_sk_storage_get @@ -2691,7 +2818,10 @@ static long (* const bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 * **NULL** if not found or there was an error in adding * a new bpf-local-storage. */ -static void *(* const bpf_sk_storage_get)(void *map, void *sk, void *value, __u64 flags) = (void *) 107; +static void *(*const bpf_sk_storage_get)(void *map, + void *sk, + void *value, + __u64 flags) = (void *)107; /* * bpf_sk_storage_delete @@ -2704,7 +2834,7 @@ static void *(* const bpf_sk_storage_get)(void *map, void *sk, void *value, __u6 * **-ENOENT** if the bpf-local-storage cannot be found. * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). */ -static long (* const bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; +static long (*const bpf_sk_storage_delete)(void *map, void *sk) = (void *)108; /* * bpf_send_signal @@ -2723,7 +2853,7 @@ static long (* const bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; * * **-EAGAIN** if bpf program can try again. */ -static long (* const bpf_send_signal)(__u32 sig) = (void *) 109; +static long (*const bpf_send_signal)(__u32 sig) = (void *)109; /* * bpf_tcp_gen_syncookie @@ -2754,7 +2884,8 @@ static long (* const bpf_send_signal)(__u32 sig) = (void *) 109; * * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 */ -static __s64 (* const bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 110; +static __s64 (*const bpf_tcp_gen_syncookie)( + void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *)110; /* * bpf_skb_output @@ -2782,7 +2913,8 @@ static __s64 (* const bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 111; +static long (*const bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = + (void *)111; /* * bpf_probe_read_user @@ -2793,7 +2925,9 @@ static long (* const bpf_skb_output)(void *ctx, void *map, __u64 flags, void *da * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 112; +static long (*const bpf_probe_read_user)(void *dst, + __u32 size, + const void *unsafe_ptr) = (void *)112; /* * bpf_probe_read_kernel @@ -2804,7 +2938,9 @@ static long (* const bpf_probe_read_user)(void *dst, __u32 size, const void *uns * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 113; +static long (*const bpf_probe_read_kernel)(void *dst, + __u32 size, + const void *unsafe_ptr) = (void *)113; /* * bpf_probe_read_user_str @@ -2852,7 +2988,9 @@ static long (* const bpf_probe_read_kernel)(void *dst, __u32 size, const void *u * including the trailing NUL character. On error, a negative * value. */ -static long (* const bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114; +static long (*const bpf_probe_read_user_str)(void *dst, + __u32 size, + const void *unsafe_ptr) = (void *)114; /* * bpf_probe_read_kernel_str @@ -2864,7 +3002,9 @@ static long (* const bpf_probe_read_user_str)(void *dst, __u32 size, const void * On success, the strictly positive length of the string, including * the trailing NUL character. On error, a negative value. */ -static long (* const bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 115; +static long (*const bpf_probe_read_kernel_str)(void *dst, + __u32 size, + const void *unsafe_ptr) = (void *)115; /* * bpf_tcp_send_ack @@ -2875,7 +3015,7 @@ static long (* const bpf_probe_read_kernel_str)(void *dst, __u32 size, const voi * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; +static long (*const bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *)116; /* * bpf_send_signal_thread @@ -2893,7 +3033,7 @@ static long (* const bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; * * **-EAGAIN** if bpf program can try again. */ -static long (* const bpf_send_signal_thread)(__u32 sig) = (void *) 117; +static long (*const bpf_send_signal_thread)(__u32 sig) = (void *)117; /* * bpf_jiffies64 @@ -2903,7 +3043,7 @@ static long (* const bpf_send_signal_thread)(__u32 sig) = (void *) 117; * Returns * The 64 bit jiffies */ -static __u64 (* const bpf_jiffies64)(void) = (void *) 118; +static __u64 (*const bpf_jiffies64)(void) = (void *)118; /* * bpf_read_branch_records @@ -2926,7 +3066,10 @@ static __u64 (* const bpf_jiffies64)(void) = (void *) 118; * * **-ENOENT** if architecture does not support branch records. */ -static long (* const bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *buf, __u32 size, __u64 flags) = (void *) 119; +static long (*const bpf_read_branch_records)(struct bpf_perf_event_data *ctx, + void *buf, + __u32 size, + __u64 flags) = (void *)119; /* * bpf_get_ns_current_pid_tgid @@ -2942,7 +3085,10 @@ static long (* const bpf_read_branch_records)(struct bpf_perf_event_data *ctx, v * * **-ENOENT** if pidns does not exists for the current task. */ -static long (* const bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidns_info *nsdata, __u32 size) = (void *) 120; +static long (*const bpf_get_ns_current_pid_tgid)(__u64 dev, + __u64 ino, + struct bpf_pidns_info *nsdata, + __u32 size) = (void *)120; /* * bpf_xdp_output @@ -2970,7 +3116,8 @@ static long (* const bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct b * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 121; +static long (*const bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = + (void *)121; /* * bpf_get_netns_cookie @@ -2987,7 +3134,7 @@ static long (* const bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *da * Returns * A 8-byte long opaque number. */ -static __u64 (* const bpf_get_netns_cookie)(void *ctx) = (void *) 122; +static __u64 (*const bpf_get_netns_cookie)(void *ctx) = (void *)122; /* * bpf_get_current_ancestor_cgroup_id @@ -3009,7 +3156,7 @@ static __u64 (* const bpf_get_netns_cookie)(void *ctx) = (void *) 122; * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *) 123; +static __u64 (*const bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *)123; /* * bpf_sk_assign @@ -3041,7 +3188,7 @@ static __u64 (* const bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = * **-EOPNOTSUPP** if the operation is not supported, for example * a call from outside of TC ingress. */ -static long (* const bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) 124; +static long (*const bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *)124; /* * bpf_ktime_get_boot_ns @@ -3053,7 +3200,7 @@ static long (* const bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) * Returns * Current *ktime*. */ -static __u64 (* const bpf_ktime_get_boot_ns)(void) = (void *) 125; +static __u64 (*const bpf_ktime_get_boot_ns)(void) = (void *)125; /* * bpf_seq_printf @@ -3086,7 +3233,11 @@ static __u64 (* const bpf_ktime_get_boot_ns)(void) = (void *) 125; * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. */ -static long (* const bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 126; +static long (*const bpf_seq_printf)(struct seq_file *m, + const char *fmt, + __u32 fmt_size, + const void *data, + __u32 data_len) = (void *)126; /* * bpf_seq_write @@ -3100,7 +3251,7 @@ static long (* const bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. */ -static long (* const bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *) 127; +static long (*const bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *)127; /* * bpf_sk_cgroup_id @@ -3118,7 +3269,7 @@ static long (* const bpf_seq_write)(struct seq_file *m, const void *data, __u32 * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_sk_cgroup_id)(void *sk) = (void *) 128; +static __u64 (*const bpf_sk_cgroup_id)(void *sk) = (void *)128; /* * bpf_sk_ancestor_cgroup_id @@ -3140,7 +3291,7 @@ static __u64 (* const bpf_sk_cgroup_id)(void *sk) = (void *) 128; * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void *) 129; +static __u64 (*const bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void *)129; /* * bpf_ringbuf_output @@ -3161,7 +3312,10 @@ static __u64 (* const bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 flags) = (void *) 130; +static long (*const bpf_ringbuf_output)(void *ringbuf, + void *data, + __u64 size, + __u64 flags) = (void *)130; /* * bpf_ringbuf_reserve @@ -3173,7 +3327,7 @@ static long (* const bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, * Valid pointer with *size* bytes of memory available; NULL, * otherwise. */ -static void *(* const bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *) 131; +static void *(*const bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *)131; /* * bpf_ringbuf_submit @@ -3191,7 +3345,7 @@ static void *(* const bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flag * Returns * Nothing. Always succeeds. */ -static void (* const bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; +static void (*const bpf_ringbuf_submit)(void *data, __u64 flags) = (void *)132; /* * bpf_ringbuf_discard @@ -3209,7 +3363,7 @@ static void (* const bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132 * Returns * Nothing. Always succeeds. */ -static void (* const bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; +static void (*const bpf_ringbuf_discard)(void *data, __u64 flags) = (void *)133; /* * bpf_ringbuf_query @@ -3230,7 +3384,7 @@ static void (* const bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 13 * Returns * Requested value, or 0, if *flags* are not recognized. */ -static __u64 (* const bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; +static __u64 (*const bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *)134; /* * bpf_csum_level @@ -3266,7 +3420,7 @@ static __u64 (* const bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) * is returned or the error code -EACCES in case the skb is not * subject to CHECKSUM_UNNECESSARY. */ -static long (* const bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135; +static long (*const bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *)135; /* * bpf_skc_to_tcp6_sock @@ -3276,7 +3430,7 @@ static long (* const bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp6_sock *(* const bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; +static struct tcp6_sock *(*const bpf_skc_to_tcp6_sock)(void *sk) = (void *)136; /* * bpf_skc_to_tcp_sock @@ -3286,7 +3440,7 @@ static struct tcp6_sock *(* const bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136 * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_sock *(* const bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; +static struct tcp_sock *(*const bpf_skc_to_tcp_sock)(void *sk) = (void *)137; /* * bpf_skc_to_tcp_timewait_sock @@ -3296,7 +3450,7 @@ static struct tcp_sock *(* const bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_timewait_sock *(* const bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *) 138; +static struct tcp_timewait_sock *(*const bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *)138; /* * bpf_skc_to_tcp_request_sock @@ -3306,7 +3460,7 @@ static struct tcp_timewait_sock *(* const bpf_skc_to_tcp_timewait_sock)(void *sk * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_request_sock *(* const bpf_skc_to_tcp_request_sock)(void *sk) = (void *) 139; +static struct tcp_request_sock *(*const bpf_skc_to_tcp_request_sock)(void *sk) = (void *)139; /* * bpf_skc_to_udp6_sock @@ -3316,7 +3470,7 @@ static struct tcp_request_sock *(* const bpf_skc_to_tcp_request_sock)(void *sk) * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct udp6_sock *(* const bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; +static struct udp6_sock *(*const bpf_skc_to_udp6_sock)(void *sk) = (void *)140; /* * bpf_get_task_stack @@ -3355,7 +3509,10 @@ static struct udp6_sock *(* const bpf_skc_to_udp6_sock)(void *sk) = (void *) 140 * The non-negative copied *buf* length equal to or less than * *size* on success, or a negative error in case of failure. */ -static long (* const bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141; +static long (*const bpf_get_task_stack)(struct task_struct *task, + void *buf, + __u32 size, + __u64 flags) = (void *)141; /* * bpf_load_hdr_opt @@ -3422,7 +3579,10 @@ static long (* const bpf_get_task_stack)(struct task_struct *task, void *buf, __ * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (* const bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchby_res, __u32 len, __u64 flags) = (void *) 142; +static long (*const bpf_load_hdr_opt)(struct bpf_sock_ops *skops, + void *searchby_res, + __u32 len, + __u64 flags) = (void *)142; /* * bpf_store_hdr_opt @@ -3459,7 +3619,10 @@ static long (* const bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchb * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (* const bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void *from, __u32 len, __u64 flags) = (void *) 143; +static long (*const bpf_store_hdr_opt)(struct bpf_sock_ops *skops, + const void *from, + __u32 len, + __u64 flags) = (void *)143; /* * bpf_reserve_hdr_opt @@ -3485,7 +3648,9 @@ static long (* const bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void * * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (* const bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, __u64 flags) = (void *) 144; +static long (*const bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, + __u32 len, + __u64 flags) = (void *)144; /* * bpf_inode_storage_get @@ -3517,7 +3682,10 @@ static long (* const bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, * **NULL** if not found or there was an error in adding * a new bpf_local_storage. */ -static void *(* const bpf_inode_storage_get)(void *map, void *inode, void *value, __u64 flags) = (void *) 145; +static void *(*const bpf_inode_storage_get)(void *map, + void *inode, + void *value, + __u64 flags) = (void *)145; /* * bpf_inode_storage_delete @@ -3529,7 +3697,7 @@ static void *(* const bpf_inode_storage_get)(void *map, void *inode, void *value * * **-ENOENT** if the bpf_local_storage cannot be found. */ -static int (* const bpf_inode_storage_delete)(void *map, void *inode) = (void *) 146; +static int (*const bpf_inode_storage_delete)(void *map, void *inode) = (void *)146; /* * bpf_d_path @@ -3545,7 +3713,7 @@ static int (* const bpf_inode_storage_delete)(void *map, void *inode) = (void *) * including the trailing NUL character. On error, a negative * value. */ -static long (* const bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *) 147; +static long (*const bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *)147; /* * bpf_copy_from_user @@ -3556,7 +3724,7 @@ static long (* const bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = (void *) 148; +static long (*const bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = (void *)148; /* * bpf_snprintf_btf @@ -3596,7 +3764,8 @@ static long (* const bpf_copy_from_user)(void *dst, __u32 size, const void *user * written if output had to be truncated due to string size), * or a negative error in cases of failure. */ -static long (* const bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr *ptr, __u32 btf_ptr_size, __u64 flags) = (void *) 149; +static long (*const bpf_snprintf_btf)( + char *str, __u32 str_size, struct btf_ptr *ptr, __u32 btf_ptr_size, __u64 flags) = (void *)149; /* * bpf_seq_printf_btf @@ -3608,7 +3777,10 @@ static long (* const bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr * Returns * 0 on success or a negative error in case of failure. */ -static long (* const bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr, __u32 ptr_size, __u64 flags) = (void *) 150; +static long (*const bpf_seq_printf_btf)(struct seq_file *m, + struct btf_ptr *ptr, + __u32 ptr_size, + __u64 flags) = (void *)150; /* * bpf_skb_cgroup_classid @@ -3621,7 +3793,7 @@ static long (* const bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (* const bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) 151; +static __u64 (*const bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *)151; /* * bpf_redirect_neigh @@ -3646,7 +3818,10 @@ static __u64 (* const bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. */ -static long (* const bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh *params, int plen, __u64 flags) = (void *) 152; +static long (*const bpf_redirect_neigh)(__u32 ifindex, + struct bpf_redir_neigh *params, + int plen, + __u64 flags) = (void *)152; /* * bpf_per_cpu_ptr @@ -3667,7 +3842,7 @@ static long (* const bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh * * A pointer pointing to the kernel percpu variable on *cpu*, or * NULL, if *cpu* is invalid. */ -static void *(* const bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *) 153; +static void *(*const bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *)153; /* * bpf_this_cpu_ptr @@ -3683,7 +3858,7 @@ static void *(* const bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (voi * Returns * A pointer pointing to the kernel percpu variable on this cpu. */ -static void *(* const bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; +static void *(*const bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *)154; /* * bpf_redirect_peer @@ -3703,7 +3878,7 @@ static void *(* const bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. */ -static long (* const bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; +static long (*const bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *)155; /* * bpf_task_storage_get @@ -3735,7 +3910,10 @@ static long (* const bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 1 * **NULL** if not found or there was an error in adding * a new bpf_local_storage. */ -static void *(* const bpf_task_storage_get)(void *map, struct task_struct *task, void *value, __u64 flags) = (void *) 156; +static void *(*const bpf_task_storage_get)(void *map, + struct task_struct *task, + void *value, + __u64 flags) = (void *)156; /* * bpf_task_storage_delete @@ -3747,7 +3925,7 @@ static void *(* const bpf_task_storage_get)(void *map, struct task_struct *task, * * **-ENOENT** if the bpf_local_storage cannot be found. */ -static long (* const bpf_task_storage_delete)(void *map, struct task_struct *task) = (void *) 157; +static long (*const bpf_task_storage_delete)(void *map, struct task_struct *task) = (void *)157; /* * bpf_get_current_task_btf @@ -3759,7 +3937,7 @@ static long (* const bpf_task_storage_delete)(void *map, struct task_struct *tas * Returns * Pointer to the current task. */ -static struct task_struct *(* const bpf_get_current_task_btf)(void) = (void *) 158; +static struct task_struct *(*const bpf_get_current_task_btf)(void) = (void *)158; /* * bpf_bprm_opts_set @@ -3773,7 +3951,7 @@ static struct task_struct *(* const bpf_get_current_task_btf)(void) = (void *) 1 * Returns * **-EINVAL** if invalid *flags* are passed, zero otherwise. */ -static long (* const bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void *) 159; +static long (*const bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void *)159; /* * bpf_ktime_get_coarse_ns @@ -3787,7 +3965,7 @@ static long (* const bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) * Returns * Current *ktime*. */ -static __u64 (* const bpf_ktime_get_coarse_ns)(void) = (void *) 160; +static __u64 (*const bpf_ktime_get_coarse_ns)(void) = (void *)160; /* * bpf_ima_inode_hash @@ -3801,7 +3979,7 @@ static __u64 (* const bpf_ktime_get_coarse_ns)(void) = (void *) 160; * **-EOPNOTSUP** if IMA is disabled or **-EINVAL** if * invalid arguments are passed. */ -static long (* const bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = (void *) 161; +static long (*const bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = (void *)161; /* * bpf_sock_from_file @@ -3813,7 +3991,7 @@ static long (* const bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 s * A pointer to a struct socket on success or NULL if the file is * not a socket. */ -static struct socket *(* const bpf_sock_from_file)(struct file *file) = (void *) 162; +static struct socket *(*const bpf_sock_from_file)(struct file *file) = (void *)162; /* * bpf_check_mtu @@ -3884,7 +4062,8 @@ static struct socket *(* const bpf_sock_from_file)(struct file *file) = (void *) * * **BPF_MTU_CHK_RET_FRAG_NEEDED** * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** */ -static long (* const bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len_diff, __u64 flags) = (void *) 163; +static long (*const bpf_check_mtu)( + void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len_diff, __u64 flags) = (void *)163; /* * bpf_for_each_map_elem @@ -3917,7 +4096,10 @@ static long (* const bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __ * The number of traversed map elements for success, **-EINVAL** for * invalid **flags**. */ -static long (* const bpf_for_each_map_elem)(void *map, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 164; +static long (*const bpf_for_each_map_elem)(void *map, + void *callback_fn, + void *callback_ctx, + __u64 flags) = (void *)164; /* * bpf_snprintf @@ -3949,7 +4131,8 @@ static long (* const bpf_for_each_map_elem)(void *map, void *callback_fn, void * * * Or **-EBUSY** if the per-CPU memory copy buffer is busy. */ -static long (* const bpf_snprintf)(char *str, __u32 str_size, const char *fmt, __u64 *data, __u32 data_len) = (void *) 165; +static long (*const bpf_snprintf)( + char *str, __u32 str_size, const char *fmt, __u64 *data, __u32 data_len) = (void *)165; /* * bpf_sys_bpf @@ -3959,7 +4142,7 @@ static long (* const bpf_snprintf)(char *str, __u32 str_size, const char *fmt, _ * Returns * A syscall result. */ -static long (* const bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *) 166; +static long (*const bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *)166; /* * bpf_btf_find_by_name_kind @@ -3969,7 +4152,10 @@ static long (* const bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (voi * Returns * Returns btf_id and btf_obj_fd in lower and upper 32 bits. */ -static long (* const bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 kind, int flags) = (void *) 167; +static long (*const bpf_btf_find_by_name_kind)(char *name, + int name_sz, + __u32 kind, + int flags) = (void *)167; /* * bpf_sys_close @@ -3979,7 +4165,7 @@ static long (* const bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 k * Returns * A syscall result. */ -static long (* const bpf_sys_close)(__u32 fd) = (void *) 168; +static long (*const bpf_sys_close)(__u32 fd) = (void *)168; /* * bpf_timer_init @@ -4000,7 +4186,7 @@ static long (* const bpf_sys_close)(__u32 fd) = (void *) 168; * or pin such map in bpffs. When map is unpinned or file descriptor is * closed all timers in the map will be cancelled and freed. */ -static long (* const bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = (void *) 169; +static long (*const bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = (void *)169; /* * bpf_timer_set_callback @@ -4015,7 +4201,8 @@ static long (* const bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 f * or pin such map in bpffs. When map is unpinned or file descriptor is * closed all timers in the map will be cancelled and freed. */ -static long (* const bpf_timer_set_callback)(struct bpf_timer *timer, void *callback_fn) = (void *) 170; +static long (*const bpf_timer_set_callback)(struct bpf_timer *timer, + void *callback_fn) = (void *)170; /* * bpf_timer_start @@ -4053,7 +4240,9 @@ static long (* const bpf_timer_set_callback)(struct bpf_timer *timer, void *call * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier * or invalid *flags* are passed. */ -static long (* const bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u64 flags) = (void *) 171; +static long (*const bpf_timer_start)(struct bpf_timer *timer, + __u64 nsecs, + __u64 flags) = (void *)171; /* * bpf_timer_cancel @@ -4067,7 +4256,7 @@ static long (* const bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u6 * **-EDEADLK** if callback_fn tried to call bpf_timer_cancel() on its * own timer which would have led to a deadlock otherwise. */ -static long (* const bpf_timer_cancel)(struct bpf_timer *timer) = (void *) 172; +static long (*const bpf_timer_cancel)(struct bpf_timer *timer) = (void *)172; /* * bpf_get_func_ip @@ -4083,7 +4272,7 @@ static long (* const bpf_timer_cancel)(struct bpf_timer *timer) = (void *) 172; * 0 for kprobes placed within the function (not at the entry). * Address of the probe for uprobe and return uprobe. */ -static __u64 (* const bpf_get_func_ip)(void *ctx) = (void *) 173; +static __u64 (*const bpf_get_func_ip)(void *ctx) = (void *)173; /* * bpf_get_attach_cookie @@ -4102,7 +4291,7 @@ static __u64 (* const bpf_get_func_ip)(void *ctx) = (void *) 173; * Value specified by user at BPF link creation/attachment time * or 0, if it was not specified. */ -static __u64 (* const bpf_get_attach_cookie)(void *ctx) = (void *) 174; +static __u64 (*const bpf_get_attach_cookie)(void *ctx) = (void *)174; /* * bpf_task_pt_regs @@ -4112,7 +4301,7 @@ static __u64 (* const bpf_get_attach_cookie)(void *ctx) = (void *) 174; * Returns * A pointer to struct pt_regs. */ -static long (* const bpf_task_pt_regs)(struct task_struct *task) = (void *) 175; +static long (*const bpf_task_pt_regs)(struct task_struct *task) = (void *)175; /* * bpf_get_branch_snapshot @@ -4137,7 +4326,7 @@ static long (* const bpf_task_pt_regs)(struct task_struct *task) = (void *) 175; * * **-ENOENT** if architecture does not support branch records. */ -static long (* const bpf_get_branch_snapshot)(void *entries, __u32 size, __u64 flags) = (void *) 176; +static long (*const bpf_get_branch_snapshot)(void *entries, __u32 size, __u64 flags) = (void *)176; /* * bpf_trace_vprintk @@ -4151,7 +4340,10 @@ static long (* const bpf_get_branch_snapshot)(void *entries, __u32 size, __u64 f * The number of bytes written to the buffer, or a negative error * in case of failure. */ -static long (* const bpf_trace_vprintk)(const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 177; +static long (*const bpf_trace_vprintk)(const char *fmt, + __u32 fmt_size, + const void *data, + __u32 data_len) = (void *)177; /* * bpf_skc_to_unix_sock @@ -4161,7 +4353,7 @@ static long (* const bpf_trace_vprintk)(const char *fmt, __u32 fmt_size, const v * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct unix_sock *(* const bpf_skc_to_unix_sock)(void *sk) = (void *) 178; +static struct unix_sock *(*const bpf_skc_to_unix_sock)(void *sk) = (void *)178; /* * bpf_kallsyms_lookup_name @@ -4180,7 +4372,10 @@ static struct unix_sock *(* const bpf_skc_to_unix_sock)(void *sk) = (void *) 178 * * **-EPERM** if caller does not have permission to obtain kernel address. */ -static long (* const bpf_kallsyms_lookup_name)(const char *name, int name_sz, int flags, __u64 *res) = (void *) 179; +static long (*const bpf_kallsyms_lookup_name)(const char *name, + int name_sz, + int flags, + __u64 *res) = (void *)179; /* * bpf_find_vma @@ -4203,7 +4398,11 @@ static long (* const bpf_kallsyms_lookup_name)(const char *name, int name_sz, in * **-EBUSY** if failed to try lock mmap_lock. * **-EINVAL** for invalid **flags**. */ -static long (* const bpf_find_vma)(struct task_struct *task, __u64 addr, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 180; +static long (*const bpf_find_vma)(struct task_struct *task, + __u64 addr, + void *callback_fn, + void *callback_ctx, + __u64 flags) = (void *)180; /* * bpf_loop @@ -4231,7 +4430,10 @@ static long (* const bpf_find_vma)(struct task_struct *task, __u64 addr, void *c * The number of loops performed, **-EINVAL** for invalid **flags**, * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. */ -static long (* const bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 181; +static long (*const bpf_loop)(__u32 nr_loops, + void *callback_fn, + void *callback_ctx, + __u64 flags) = (void *)181; /* * bpf_strncmp @@ -4245,7 +4447,7 @@ static long (* const bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback * if the first **s1_sz** bytes of **s1** is found to be * less than, to match, or be greater than **s2**. */ -static long (* const bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = (void *) 182; +static long (*const bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = (void *)182; /* * bpf_get_func_arg @@ -4258,7 +4460,7 @@ static long (* const bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = * 0 on success. * **-EINVAL** if n >= argument register count of traced function. */ -static long (* const bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void *) 183; +static long (*const bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void *)183; /* * bpf_get_func_ret @@ -4271,7 +4473,7 @@ static long (* const bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void * 0 on success. * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. */ -static long (* const bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; +static long (*const bpf_get_func_ret)(void *ctx, __u64 *value) = (void *)184; /* * bpf_get_func_arg_cnt @@ -4283,7 +4485,7 @@ static long (* const bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; * Returns * The number of argument registers of the traced function. */ -static long (* const bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; +static long (*const bpf_get_func_arg_cnt)(void *ctx) = (void *)185; /* * bpf_get_retval @@ -4296,7 +4498,7 @@ static long (* const bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; * Returns * The BPF program's return value. */ -static int (* const bpf_get_retval)(void) = (void *) 186; +static int (*const bpf_get_retval)(void) = (void *)186; /* * bpf_set_retval @@ -4319,7 +4521,7 @@ static int (* const bpf_get_retval)(void) = (void *) 186; * Returns * 0 on success, or a negative error in case of failure. */ -static int (* const bpf_set_retval)(int retval) = (void *) 187; +static int (*const bpf_set_retval)(int retval) = (void *)187; /* * bpf_xdp_get_buff_len @@ -4329,7 +4531,7 @@ static int (* const bpf_set_retval)(int retval) = (void *) 187; * Returns * The total size of a given xdp buffer. */ -static __u64 (* const bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *) 188; +static __u64 (*const bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *)188; /* * bpf_xdp_load_bytes @@ -4342,7 +4544,10 @@ static __u64 (* const bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *) 18 * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_load_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 189; +static long (*const bpf_xdp_load_bytes)(struct xdp_md *xdp_md, + __u32 offset, + void *buf, + __u32 len) = (void *)189; /* * bpf_xdp_store_bytes @@ -4353,7 +4558,10 @@ static long (* const bpf_xdp_load_bytes)(struct xdp_md *xdp_md, __u32 offset, vo * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 190; +static long (*const bpf_xdp_store_bytes)(struct xdp_md *xdp_md, + __u32 offset, + void *buf, + __u32 len) = (void *)190; /* * bpf_copy_from_user_task @@ -4367,7 +4575,11 @@ static long (* const bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, v * 0 on success, or a negative error in case of failure. On error * *dst* buffer is zeroed out. */ -static long (* const bpf_copy_from_user_task)(void *dst, __u32 size, const void *user_ptr, struct task_struct *tsk, __u64 flags) = (void *) 191; +static long (*const bpf_copy_from_user_task)(void *dst, + __u32 size, + const void *user_ptr, + struct task_struct *tsk, + __u64 flags) = (void *)191; /* * bpf_skb_set_tstamp @@ -4401,7 +4613,9 @@ static long (* const bpf_copy_from_user_task)(void *dst, __u32 size, const void * **-EINVAL** for invalid input * **-EOPNOTSUPP** for unsupported protocol */ -static long (* const bpf_skb_set_tstamp)(struct __sk_buff *skb, __u64 tstamp, __u32 tstamp_type) = (void *) 192; +static long (*const bpf_skb_set_tstamp)(struct __sk_buff *skb, + __u64 tstamp, + __u32 tstamp_type) = (void *)192; /* * bpf_ima_file_hash @@ -4415,7 +4629,7 @@ static long (* const bpf_skb_set_tstamp)(struct __sk_buff *skb, __u64 tstamp, __ * **-EOPNOTSUP** if the hash calculation failed or **-EINVAL** if * invalid arguments are passed. */ -static long (* const bpf_ima_file_hash)(struct file *file, void *dst, __u32 size) = (void *) 193; +static long (*const bpf_ima_file_hash)(struct file *file, void *dst, __u32 size) = (void *)193; /* * bpf_kptr_xchg @@ -4430,7 +4644,7 @@ static long (* const bpf_ima_file_hash)(struct file *file, void *dst, __u32 size * corresponding release function, or moved into a BPF map before * program exit. */ -static void *(* const bpf_kptr_xchg)(void *map_value, void *ptr) = (void *) 194; +static void *(*const bpf_kptr_xchg)(void *map_value, void *ptr) = (void *)194; /* * bpf_map_lookup_percpu_elem @@ -4442,7 +4656,9 @@ static void *(* const bpf_kptr_xchg)(void *map_value, void *ptr) = (void *) 194; * Map value associated to *key* on *cpu*, or **NULL** if no entry * was found or *cpu* is invalid. */ -static void *(* const bpf_map_lookup_percpu_elem)(void *map, const void *key, __u32 cpu) = (void *) 195; +static void *(*const bpf_map_lookup_percpu_elem)(void *map, + const void *key, + __u32 cpu) = (void *)195; /* * bpf_skc_to_mptcp_sock @@ -4452,7 +4668,7 @@ static void *(* const bpf_map_lookup_percpu_elem)(void *map, const void *key, __ * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct mptcp_sock *(* const bpf_skc_to_mptcp_sock)(void *sk) = (void *) 196; +static struct mptcp_sock *(*const bpf_skc_to_mptcp_sock)(void *sk) = (void *)196; /* * bpf_dynptr_from_mem @@ -4467,7 +4683,10 @@ static struct mptcp_sock *(* const bpf_skc_to_mptcp_sock)(void *sk) = (void *) 1 * 0 on success, -E2BIG if the size exceeds DYNPTR_MAX_SIZE, * -EINVAL if flags is not 0. */ -static long (* const bpf_dynptr_from_mem)(void *data, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 197; +static long (*const bpf_dynptr_from_mem)(void *data, + __u32 size, + __u64 flags, + struct bpf_dynptr *ptr) = (void *)197; /* * bpf_ringbuf_reserve_dynptr @@ -4482,7 +4701,10 @@ static long (* const bpf_dynptr_from_mem)(void *data, __u32 size, __u64 flags, s * Returns * 0 on success, or a negative error in case of failure. */ -static long (* const bpf_ringbuf_reserve_dynptr)(void *ringbuf, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 198; +static long (*const bpf_ringbuf_reserve_dynptr)(void *ringbuf, + __u32 size, + __u64 flags, + struct bpf_dynptr *ptr) = (void *)198; /* * bpf_ringbuf_submit_dynptr @@ -4497,7 +4719,7 @@ static long (* const bpf_ringbuf_reserve_dynptr)(void *ringbuf, __u32 size, __u6 * Returns * Nothing. Always succeeds. */ -static void (* const bpf_ringbuf_submit_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 199; +static void (*const bpf_ringbuf_submit_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *)199; /* * bpf_ringbuf_discard_dynptr @@ -4511,7 +4733,7 @@ static void (* const bpf_ringbuf_submit_dynptr)(struct bpf_dynptr *ptr, __u64 fl * Returns * Nothing. Always succeeds. */ -static void (* const bpf_ringbuf_discard_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 200; +static void (*const bpf_ringbuf_discard_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *)200; /* * bpf_dynptr_read @@ -4525,7 +4747,8 @@ static void (* const bpf_ringbuf_discard_dynptr)(struct bpf_dynptr *ptr, __u64 f * of *src*'s data, -EINVAL if *src* is an invalid dynptr or if * *flags* is not 0. */ -static long (* const bpf_dynptr_read)(void *dst, __u32 len, const struct bpf_dynptr *src, __u32 offset, __u64 flags) = (void *) 201; +static long (*const bpf_dynptr_read)( + void *dst, __u32 len, const struct bpf_dynptr *src, __u32 offset, __u64 flags) = (void *)201; /* * bpf_dynptr_write @@ -4550,7 +4773,8 @@ static long (* const bpf_dynptr_read)(void *dst, __u32 len, const struct bpf_dyn * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs, * other errors correspond to errors returned by **bpf_skb_store_bytes**\ (). */ -static long (* const bpf_dynptr_write)(const struct bpf_dynptr *dst, __u32 offset, void *src, __u32 len, __u64 flags) = (void *) 202; +static long (*const bpf_dynptr_write)( + const struct bpf_dynptr *dst, __u32 offset, void *src, __u32 len, __u64 flags) = (void *)202; /* * bpf_dynptr_data @@ -4568,7 +4792,9 @@ static long (* const bpf_dynptr_write)(const struct bpf_dynptr *dst, __u32 offse * read-only, if the dynptr is invalid, or if the offset and length * is out of bounds. */ -static void *(* const bpf_dynptr_data)(const struct bpf_dynptr *ptr, __u32 offset, __u32 len) = (void *) 203; +static void *(*const bpf_dynptr_data)(const struct bpf_dynptr *ptr, + __u32 offset, + __u32 len) = (void *)203; /* * bpf_tcp_raw_gen_syncookie_ipv4 @@ -4592,7 +4818,9 @@ static void *(* const bpf_dynptr_data)(const struct bpf_dynptr *ptr, __u32 offse * * **-EINVAL** if *th_len* is invalid. */ -static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 204; +static __s64 (*const bpf_tcp_raw_gen_syncookie_ipv4)(struct iphdr *iph, + struct tcphdr *th, + __u32 th_len) = (void *)204; /* * bpf_tcp_raw_gen_syncookie_ipv6 @@ -4618,7 +4846,9 @@ static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv4)(struct iphdr *iph, struct * * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. */ -static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 205; +static __s64 (*const bpf_tcp_raw_gen_syncookie_ipv6)(struct ipv6hdr *iph, + struct tcphdr *th, + __u32 th_len) = (void *)205; /* * bpf_tcp_raw_check_syncookie_ipv4 @@ -4637,7 +4867,8 @@ static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv6)(struct ipv6hdr *iph, struc * * **-EACCES** if the SYN cookie is not valid. */ -static long (* const bpf_tcp_raw_check_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th) = (void *) 206; +static long (*const bpf_tcp_raw_check_syncookie_ipv4)(struct iphdr *iph, + struct tcphdr *th) = (void *)206; /* * bpf_tcp_raw_check_syncookie_ipv6 @@ -4658,7 +4889,8 @@ static long (* const bpf_tcp_raw_check_syncookie_ipv4)(struct iphdr *iph, struct * * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. */ -static long (* const bpf_tcp_raw_check_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th) = (void *) 207; +static long (*const bpf_tcp_raw_check_syncookie_ipv6)(struct ipv6hdr *iph, + struct tcphdr *th) = (void *)207; /* * bpf_ktime_get_tai_ns @@ -4673,7 +4905,7 @@ static long (* const bpf_tcp_raw_check_syncookie_ipv6)(struct ipv6hdr *iph, stru * Returns * Current *ktime*. */ -static __u64 (* const bpf_ktime_get_tai_ns)(void) = (void *) 208; +static __u64 (*const bpf_ktime_get_tai_ns)(void) = (void *)208; /* * bpf_user_ringbuf_drain @@ -4715,7 +4947,10 @@ static __u64 (* const bpf_ktime_get_tai_ns)(void) = (void *) 208; * larger than the size of the ring buffer, or which cannot fit * within a struct bpf_dynptr. */ -static long (* const bpf_user_ringbuf_drain)(void *map, void *callback_fn, void *ctx, __u64 flags) = (void *) 209; +static long (*const bpf_user_ringbuf_drain)(void *map, + void *callback_fn, + void *ctx, + __u64 flags) = (void *)209; /* * bpf_cgrp_storage_get @@ -4749,7 +4984,10 @@ static long (* const bpf_user_ringbuf_drain)(void *map, void *callback_fn, void * **NULL** if not found or there was an error in adding * a new bpf_local_storage. */ -static void *(* const bpf_cgrp_storage_get)(void *map, struct cgroup *cgroup, void *value, __u64 flags) = (void *) 210; +static void *(*const bpf_cgrp_storage_get)(void *map, + struct cgroup *cgroup, + void *value, + __u64 flags) = (void *)210; /* * bpf_cgrp_storage_delete @@ -4761,6 +4999,4 @@ static void *(* const bpf_cgrp_storage_get)(void *map, struct cgroup *cgroup, vo * * **-ENOENT** if the bpf_local_storage cannot be found. */ -static long (* const bpf_cgrp_storage_delete)(void *map, struct cgroup *cgroup) = (void *) 211; - - +static long (*const bpf_cgrp_storage_delete)(void *map, struct cgroup *cgroup) = (void *)211; diff --git a/internal/include/libbpf/bpf_helpers.h b/internal/include/libbpf/bpf_helpers.h index cd17f6d079..64c290d612 100644 --- a/internal/include/libbpf/bpf_helpers.h +++ b/internal/include/libbpf/bpf_helpers.h @@ -10,7 +10,7 @@ */ #include "bpf_helper_defs.h" -#define __uint(name, val) int (*name)[val] +#define __uint(name, val) int(*name)[val] #define __type(name, val) typeof(val) *name #define __array(name, val) typeof(val) *name[] #define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name @@ -34,11 +34,9 @@ #else -#define SEC(name) \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \ - __attribute__((section(name), used)) \ - _Pragma("GCC diagnostic pop") \ +#define SEC(name) \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \ + __attribute__((section(name), used)) _Pragma("GCC diagnostic pop") #endif @@ -84,15 +82,15 @@ * old-school approach which works with CO-RE correctly */ #undef offsetof -#define offsetof(type, member) ((unsigned long)&((type *)0)->member) +#define offsetof(type, member) ((unsigned long)&((type *)0)->member) /* redefined container_of() to ensure we use the above offsetof() macro */ #undef container_of -#define container_of(ptr, type, member) \ - ({ \ - void *__mptr = (void *)(ptr); \ - ((type *)(__mptr - offsetof(type, member))); \ - }) +#define container_of(ptr, type, member) \ + ({ \ + void *__mptr = (void *)(ptr); \ + ((type *)(__mptr - offsetof(type, member))); \ + }) /* * Compiler (optimization) barrier. @@ -131,20 +129,18 @@ * being compiled out. */ #ifndef __bpf_unreachable -# define __bpf_unreachable() __builtin_trap() +#define __bpf_unreachable() __builtin_trap() #endif /* * Helper function to perform a tail call with a constant/immediate map slot. */ #if __clang_major__ >= 8 && defined(__bpf__) -static __always_inline void -bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) -{ - if (!__builtin_constant_p(slot)) - __bpf_unreachable(); +static __always_inline void bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) { + if (!__builtin_constant_p(slot)) + __bpf_unreachable(); - /* + /* * Provide a hard guarantee that LLVM won't optimize setting r2 (map * pointer) and r3 (constant map index) from _different paths_ ending * up at the _same_ call insn as otherwise we won't be able to use the @@ -157,25 +153,26 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) * to mark them as clobber so that LLVM doesn't end up using them * before / after the call. */ - asm volatile("r1 = %[ctx]\n\t" - "r2 = %[map]\n\t" - "r3 = %[slot]\n\t" - "call 12" - :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot) - : "r0", "r1", "r2", "r3", "r4", "r5"); + asm volatile("r1 = %[ctx]\n\t" + "r2 = %[map]\n\t" + "r3 = %[slot]\n\t" + "call 12" ::[ctx] "r"(ctx), + [map] "r"(map), + [slot] "i"(slot) + : "r0", "r1", "r2", "r3", "r4", "r5"); } #endif enum libbpf_pin_type { - LIBBPF_PIN_NONE, - /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ - LIBBPF_PIN_BY_NAME, + LIBBPF_PIN_NONE, + /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ + LIBBPF_PIN_BY_NAME, }; enum libbpf_tristate { - TRI_NO = 0, - TRI_YES = 1, - TRI_MODULE = 2, + TRI_NO = 0, + TRI_YES = 1, + TRI_MODULE = 2, }; #define __kconfig __attribute__((section(".kconfig"))) @@ -184,10 +181,11 @@ enum libbpf_tristate { #define __kptr __attribute__((btf_type_tag("kptr"))) #define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr"))) -#define bpf_ksym_exists(sym) ({ \ - _Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \ - !!sym; \ -}) +#define bpf_ksym_exists(sym) \ + ({ \ + _Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \ + !!sym; \ + }) #define __arg_ctx __attribute__((btf_decl_tag("arg:ctx"))) #define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull"))) @@ -196,7 +194,7 @@ enum libbpf_tristate { #define __arg_arena __attribute((btf_decl_tag("arg:arena"))) #ifndef ___bpf_concat -#define ___bpf_concat(a, b) a ## b +#define ___bpf_concat(a, b) a##b #endif #ifndef ___bpf_apply #define ___bpf_apply(fn, n) ___bpf_concat(fn, n) @@ -205,61 +203,79 @@ enum libbpf_tristate { #define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N #endif #ifndef ___bpf_narg -#define ___bpf_narg(...) \ - ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) #endif -#define ___bpf_fill0(arr, p, x) do {} while (0) +#define ___bpf_fill0(arr, p, x) \ + do { \ + } while (0) #define ___bpf_fill1(arr, p, x) arr[p] = x -#define ___bpf_fill2(arr, p, x, args...) arr[p] = x; ___bpf_fill1(arr, p + 1, args) -#define ___bpf_fill3(arr, p, x, args...) arr[p] = x; ___bpf_fill2(arr, p + 1, args) -#define ___bpf_fill4(arr, p, x, args...) arr[p] = x; ___bpf_fill3(arr, p + 1, args) -#define ___bpf_fill5(arr, p, x, args...) arr[p] = x; ___bpf_fill4(arr, p + 1, args) -#define ___bpf_fill6(arr, p, x, args...) arr[p] = x; ___bpf_fill5(arr, p + 1, args) -#define ___bpf_fill7(arr, p, x, args...) arr[p] = x; ___bpf_fill6(arr, p + 1, args) -#define ___bpf_fill8(arr, p, x, args...) arr[p] = x; ___bpf_fill7(arr, p + 1, args) -#define ___bpf_fill9(arr, p, x, args...) arr[p] = x; ___bpf_fill8(arr, p + 1, args) -#define ___bpf_fill10(arr, p, x, args...) arr[p] = x; ___bpf_fill9(arr, p + 1, args) -#define ___bpf_fill11(arr, p, x, args...) arr[p] = x; ___bpf_fill10(arr, p + 1, args) -#define ___bpf_fill12(arr, p, x, args...) arr[p] = x; ___bpf_fill11(arr, p + 1, args) -#define ___bpf_fill(arr, args...) \ - ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) +#define ___bpf_fill2(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill1(arr, p + 1, args) +#define ___bpf_fill3(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill2(arr, p + 1, args) +#define ___bpf_fill4(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill3(arr, p + 1, args) +#define ___bpf_fill5(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill4(arr, p + 1, args) +#define ___bpf_fill6(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill5(arr, p + 1, args) +#define ___bpf_fill7(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill6(arr, p + 1, args) +#define ___bpf_fill8(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill7(arr, p + 1, args) +#define ___bpf_fill9(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill8(arr, p + 1, args) +#define ___bpf_fill10(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill9(arr, p + 1, args) +#define ___bpf_fill11(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill10(arr, p + 1, args) +#define ___bpf_fill12(arr, p, x, args...) \ + arr[p] = x; \ + ___bpf_fill11(arr, p + 1, args) +#define ___bpf_fill(arr, args...) ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ -#define BPF_SEQ_PRINTF(seq, fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_seq_printf(seq, ___fmt, sizeof(___fmt), \ - ___param, sizeof(___param)); \ -}) +#define BPF_SEQ_PRINTF(seq, fmt, args...) \ + ({ \ + static const char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args)]; \ + \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + bpf_seq_printf(seq, ___fmt, sizeof(___fmt), ___param, sizeof(___param)); \ + }) /* * BPF_SNPRINTF wraps the bpf_snprintf helper with variadic arguments instead of * an array of u64. */ -#define BPF_SNPRINTF(out, out_size, fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_snprintf(out, out_size, ___fmt, \ - ___param, sizeof(___param)); \ -}) +#define BPF_SNPRINTF(out, out_size, fmt, args...) \ + ({ \ + static const char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args)]; \ + \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + bpf_snprintf(out, out_size, ___fmt, ___param, sizeof(___param)); \ + }) #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD @@ -267,39 +283,47 @@ enum libbpf_tristate { #define BPF_PRINTK_FMT_MOD static const #endif -#define __bpf_printk(fmt, ...) \ -({ \ - BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), \ - ##__VA_ARGS__); \ -}) +#define __bpf_printk(fmt, ...) \ + ({ \ + BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ + }) /* * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments * instead of an array of u64. */ -#define __bpf_vprintk(fmt, args...) \ -({ \ - static const char ___fmt[] = fmt; \ - unsigned long long ___param[___bpf_narg(args)]; \ - \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - ___bpf_fill(___param, args); \ - _Pragma("GCC diagnostic pop") \ - \ - bpf_trace_vprintk(___fmt, sizeof(___fmt), \ - ___param, sizeof(___param)); \ -}) +#define __bpf_vprintk(fmt, args...) \ + ({ \ + static const char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args)]; \ + \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + bpf_trace_vprintk(___fmt, sizeof(___fmt), ___param, sizeof(___param)); \ + }) /* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args * Otherwise use __bpf_vprintk */ -#define ___bpf_pick_printk(...) \ - ___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ - __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ - __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\ - __bpf_printk /*1*/, __bpf_printk /*0*/) +#define ___bpf_pick_printk(...) \ + ___bpf_nth(_, \ + ##__VA_ARGS__, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_vprintk, \ + __bpf_printk /*3*/, \ + __bpf_printk /*2*/, \ + __bpf_printk /*1*/, \ + __bpf_printk /*0*/) /* Helper macro to print out debug messages */ #define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args) @@ -338,19 +362,19 @@ extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym; * extension: __attribute__((cleanup())), supported by both GCC and * Clang. */ -#define bpf_for_each(type, cur, args...) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ - cleanup(bpf_iter_##type##_destroy))), \ - /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_##type##_new(&___it, ##args), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_##type##_destroy, (void *)0); \ - /* iteration and termination check */ \ - (((cur) = bpf_iter_##type##_next(&___it))); \ -) +#define bpf_for_each(type, cur, args...) \ + for (/* initialize and define destructor */ \ + struct bpf_iter_##type ___it __attribute__(( \ + aligned(8), \ + /* enforce, just in case */, \ + cleanup( \ + bpf_iter_##type##_destroy))), /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = \ + (bpf_iter_##type##_new(&___it, ##args), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_##type##_destroy, \ + (void *)0); /* iteration and termination check */ \ + (((cur) = bpf_iter_##type##_next(&___it)));) #endif /* bpf_for_each */ #ifndef bpf_for @@ -367,23 +391,23 @@ extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym; * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() * loop bound variables and cleanup attribute, supported by GCC and Clang. */ -#define bpf_for(i, start, end) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ - cleanup(bpf_iter_num_destroy))), \ - /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_num_new(&___it, (start), (end)), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_num_destroy, (void *)0); \ - ({ \ - /* iteration step */ \ - int *___t = bpf_iter_num_next(&___it); \ - /* termination and bounds check */ \ - (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ - }); \ -) +#define bpf_for(i, start, end) \ + for (/* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__(( \ + aligned(8), /* enforce, just in case */ \ + cleanup( \ + bpf_iter_num_destroy))), /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = \ + (bpf_iter_num_new(&___it, (start), (end)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, \ + (void *)0); \ + ({ \ + /* iteration step */ \ + int *___t = bpf_iter_num_next(&___it); \ + /* termination and bounds check */ \ + (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ + });) #endif /* bpf_for */ #ifndef bpf_repeat @@ -392,19 +416,19 @@ extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym; * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() * loop bound variables and cleanup attribute, supported by GCC and Clang. */ -#define bpf_repeat(N) for ( \ - /* initialize and define destructor */ \ - struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ - cleanup(bpf_iter_num_destroy))), \ - /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ - *___p __attribute__((unused)) = ( \ - bpf_iter_num_new(&___it, 0, (N)), \ - /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ - /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ - (void)bpf_iter_num_destroy, (void *)0); \ - bpf_iter_num_next(&___it); \ - /* nothing here */ \ -) +#define bpf_repeat(N) \ + for (/* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__(( \ + aligned(8), /* enforce, just in case */ \ + cleanup( \ + bpf_iter_num_destroy))), /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = \ + (bpf_iter_num_new(&___it, 0, (N)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, \ + (void *)0); \ + bpf_iter_num_next(&___it); /* nothing here */ \ + ) #endif /* bpf_repeat */ #endif diff --git a/internal/include/libbpf/bpf_tracing.h b/internal/include/libbpf/bpf_tracing.h index 1c13f8e888..501e38a84a 100644 --- a/internal/include/libbpf/bpf_tracing.h +++ b/internal/include/libbpf/bpf_tracing.h @@ -6,68 +6,68 @@ /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ #if defined(__TARGET_ARCH_x86) - #define bpf_target_x86 - #define bpf_target_defined +#define bpf_target_x86 +#define bpf_target_defined #elif defined(__TARGET_ARCH_s390) - #define bpf_target_s390 - #define bpf_target_defined +#define bpf_target_s390 +#define bpf_target_defined #elif defined(__TARGET_ARCH_arm) - #define bpf_target_arm - #define bpf_target_defined +#define bpf_target_arm +#define bpf_target_defined #elif defined(__TARGET_ARCH_arm64) - #define bpf_target_arm64 - #define bpf_target_defined +#define bpf_target_arm64 +#define bpf_target_defined #elif defined(__TARGET_ARCH_mips) - #define bpf_target_mips - #define bpf_target_defined +#define bpf_target_mips +#define bpf_target_defined #elif defined(__TARGET_ARCH_powerpc) - #define bpf_target_powerpc - #define bpf_target_defined +#define bpf_target_powerpc +#define bpf_target_defined #elif defined(__TARGET_ARCH_sparc) - #define bpf_target_sparc - #define bpf_target_defined +#define bpf_target_sparc +#define bpf_target_defined #elif defined(__TARGET_ARCH_riscv) - #define bpf_target_riscv - #define bpf_target_defined +#define bpf_target_riscv +#define bpf_target_defined #elif defined(__TARGET_ARCH_arc) - #define bpf_target_arc - #define bpf_target_defined +#define bpf_target_arc +#define bpf_target_defined #elif defined(__TARGET_ARCH_loongarch) - #define bpf_target_loongarch - #define bpf_target_defined +#define bpf_target_loongarch +#define bpf_target_defined #else /* Fall back to what the compiler says */ #if defined(__x86_64__) - #define bpf_target_x86 - #define bpf_target_defined +#define bpf_target_x86 +#define bpf_target_defined #elif defined(__s390__) - #define bpf_target_s390 - #define bpf_target_defined +#define bpf_target_s390 +#define bpf_target_defined #elif defined(__arm__) - #define bpf_target_arm - #define bpf_target_defined +#define bpf_target_arm +#define bpf_target_defined #elif defined(__aarch64__) - #define bpf_target_arm64 - #define bpf_target_defined +#define bpf_target_arm64 +#define bpf_target_defined #elif defined(__mips__) - #define bpf_target_mips - #define bpf_target_defined +#define bpf_target_mips +#define bpf_target_defined #elif defined(__powerpc__) - #define bpf_target_powerpc - #define bpf_target_defined +#define bpf_target_powerpc +#define bpf_target_defined #elif defined(__sparc__) - #define bpf_target_sparc - #define bpf_target_defined +#define bpf_target_sparc +#define bpf_target_defined #elif defined(__riscv) && __riscv_xlen == 64 - #define bpf_target_riscv - #define bpf_target_defined +#define bpf_target_riscv +#define bpf_target_defined #elif defined(__arc__) - #define bpf_target_arc - #define bpf_target_defined +#define bpf_target_arc +#define bpf_target_defined #elif defined(__loongarch__) - #define bpf_target_loongarch - #define bpf_target_defined +#define bpf_target_loongarch +#define bpf_target_defined #endif /* no compiler target */ #endif @@ -162,7 +162,7 @@ */ struct pt_regs___s390 { - unsigned long orig_gpr2; + unsigned long orig_gpr2; }; /* s390 provides user_pt_regs instead of struct pt_regs to userspace */ @@ -180,11 +180,11 @@ struct pt_regs___s390 { #define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG #define __PT_PARM6_SYSCALL_REG gprs[7] #define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) -#define PT_REGS_PARM1_CORE_SYSCALL(x) \ - BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) +#define PT_REGS_PARM1_CORE_SYSCALL(x) \ + BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) #define __PT_RET_REG gprs[14] -#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ #define __PT_RC_REG gprs[2] #define __PT_SP_REG gprs[15] #define __PT_IP_REG psw.addr @@ -209,7 +209,7 @@ struct pt_regs___s390 { #define __PT_PARM7_SYSCALL_REG uregs[6] #define __PT_RET_REG uregs[14] -#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ #define __PT_RC_REG uregs[0] #define __PT_SP_REG uregs[13] #define __PT_IP_REG uregs[12] @@ -221,7 +221,7 @@ struct pt_regs___s390 { */ struct pt_regs___arm64 { - unsigned long orig_x0; + unsigned long orig_x0; }; /* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ @@ -242,11 +242,11 @@ struct pt_regs___arm64 { #define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG #define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG #define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) -#define PT_REGS_PARM1_CORE_SYSCALL(x) \ - BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) +#define PT_REGS_PARM1_CORE_SYSCALL(x) \ + BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) #define __PT_RET_REG regs[30] -#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ #define __PT_RC_REG regs[0] #define __PT_SP_REG sp #define __PT_IP_REG pc @@ -275,7 +275,7 @@ struct pt_regs___arm64 { #define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */ #define __PT_RET_REG regs[31] -#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ #define __PT_RC_REG regs[2] #define __PT_SP_REG regs[29] #define __PT_IP_REG cp0_epc @@ -512,20 +512,20 @@ struct pt_regs; #if defined(bpf_target_powerpc) -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) -#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP +#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) +#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP #elif defined(bpf_target_sparc) -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) -#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP +#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) +#define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP #else -#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) -#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) +#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ + ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) +#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ + ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) #endif @@ -560,52 +560,220 @@ struct pt_regs; #else /* defined(bpf_target_defined) */ -#define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_SP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_IP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_SP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_IP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) - -#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM1(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM2(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM3(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM4(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM5(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM6(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM7(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM8(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_RET(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_FP(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_RC(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_SP(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_IP(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) + +#define PT_REGS_PARM1_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM2_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM3_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM4_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM5_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM6_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM7_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM8_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_RET_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_FP_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_RC_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_SP_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_IP_CORE(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) + +#define BPF_KPROBE_READ_RET_IP(ip, ctx) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) + +#define PT_REGS_PARM1_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM2_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM3_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM4_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM5_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM6_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM7_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) + +#define PT_REGS_PARM1_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM2_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM3_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM4_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM5_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM6_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) +#define PT_REGS_PARM7_CORE_SYSCALL(x) \ + ({ \ + _Pragma(__BPF_TARGET_MISSING); \ + 0l; \ + }) #endif /* defined(bpf_target_defined) */ @@ -620,7 +788,7 @@ struct pt_regs; #endif #ifndef ___bpf_concat -#define ___bpf_concat(a, b) a ## b +#define ___bpf_concat(a, b) a##b #endif #ifndef ___bpf_apply #define ___bpf_apply(fn, n) ___bpf_concat(fn, n) @@ -632,20 +800,20 @@ struct pt_regs; #define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) #endif -#define ___bpf_ctx_cast0() ctx -#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] -#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] -#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] -#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] -#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] -#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] -#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] -#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] -#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] +#define ___bpf_ctx_cast0() ctx +#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] +#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] +#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] +#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] +#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] +#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] +#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] +#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] +#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] #define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9] #define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10] #define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11] -#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) +#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) /* * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and @@ -662,90 +830,194 @@ struct pt_regs; * This is useful when using BPF helpers that expect original context * as one of the parameters (e.g., for bpf_perf_event_output()). */ -#define BPF_PROG(name, args...) \ -name(unsigned long long *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx, ##args); \ -typeof(name(0)) name(unsigned long long *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_ctx_cast(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx, ##args) +#define BPF_PROG(name, args...) \ + name(unsigned long long *ctx); \ + static __always_inline typeof(name(0)) ____##name(unsigned long long *ctx, ##args); \ + typeof(name(0)) name(unsigned long long *ctx) { \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") return ____##name( \ + ___bpf_ctx_cast(args)); \ + _Pragma("GCC diagnostic pop") \ + } \ + static __always_inline typeof(name(0)) ____##name(unsigned long long *ctx, ##args) #ifndef ___bpf_nth2 -#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ - _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N +#define ___bpf_nth2(_, \ + _1, \ + _2, \ + _3, \ + _4, \ + _5, \ + _6, \ + _7, \ + _8, \ + _9, \ + _10, \ + _11, \ + _12, \ + _13, \ + _14, \ + _15, \ + _16, \ + _17, \ + _18, \ + _19, \ + _20, \ + _21, \ + _22, \ + _23, \ + _24, \ + N, \ + ...) \ + N #endif #ifndef ___bpf_narg2 -#define ___bpf_narg2(...) \ - ___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \ - 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0) +#define ___bpf_narg2(...) \ + ___bpf_nth2(_, \ + ##__VA_ARGS__, \ + 12, \ + 12, \ + 11, \ + 11, \ + 10, \ + 10, \ + 9, \ + 9, \ + 8, \ + 8, \ + 7, \ + 7, \ + 6, \ + 6, \ + 5, \ + 5, \ + 4, \ + 4, \ + 3, \ + 3, \ + 2, \ + 2, \ + 1, \ + 1, \ + 0) #endif -#define ___bpf_treg_cnt(t) \ - __builtin_choose_expr(sizeof(t) == 1, 1, \ - __builtin_choose_expr(sizeof(t) == 2, 1, \ - __builtin_choose_expr(sizeof(t) == 4, 1, \ - __builtin_choose_expr(sizeof(t) == 8, 1, \ - __builtin_choose_expr(sizeof(t) == 16, 2, \ - (void)0))))) - -#define ___bpf_reg_cnt0() (0) -#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) -#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) - -#define ___bpf_union_arg(t, x, n) \ - __builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \ - __builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \ - (void)0))))) +#define ___bpf_treg_cnt(t) \ + __builtin_choose_expr( \ + sizeof(t) == 1, \ + 1, \ + __builtin_choose_expr( \ + sizeof(t) == 2, \ + 1, \ + __builtin_choose_expr( \ + sizeof(t) == 4, \ + 1, \ + __builtin_choose_expr( \ + sizeof(t) == 8, 1, __builtin_choose_expr(sizeof(t) == 16, 2, (void)0))))) + +#define ___bpf_reg_cnt0() (0) +#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) + +#define ___bpf_union_arg(t, x, n) \ + __builtin_choose_expr( \ + sizeof(t) == 1, \ + ({ \ + union { \ + __u8 z[1]; \ + t x; \ + } ___t = {.z = {ctx[n]}}; \ + ___t.x; \ + }), \ + __builtin_choose_expr( \ + sizeof(t) == 2, \ + ({ \ + union { \ + __u16 z[1]; \ + t x; \ + } ___t = {.z = {ctx[n]}}; \ + ___t.x; \ + }), \ + __builtin_choose_expr( \ + sizeof(t) == 4, \ + ({ \ + union { \ + __u32 z[1]; \ + t x; \ + } ___t = {.z = {ctx[n]}}; \ + ___t.x; \ + }), \ + __builtin_choose_expr(sizeof(t) == 8, \ + ({ \ + union { \ + __u64 z[1]; \ + t x; \ + } ___t = {.z = {ctx[n]}}; \ + ___t.x; \ + }), \ + __builtin_choose_expr(sizeof(t) == 16, \ + ({ \ + union { \ + __u64 z[2]; \ + t x; \ + } ___t = { \ + .z = {ctx[n], ctx[n + 1]}}; \ + ___t.x; \ + }), \ + (void)0))))) #define ___bpf_ctx_arg0(n, args...) -#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) -#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) -#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) -#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) -#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) -#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) -#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) -#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) -#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) -#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) -#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) -#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) -#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) +#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) +#define ___bpf_ctx_arg2(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) +#define ___bpf_ctx_arg3(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) +#define ___bpf_ctx_arg4(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) +#define ___bpf_ctx_arg5(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) +#define ___bpf_ctx_arg6(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) +#define ___bpf_ctx_arg7(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) +#define ___bpf_ctx_arg8(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) +#define ___bpf_ctx_arg9(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) +#define ___bpf_ctx_arg10(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) +#define ___bpf_ctx_arg11(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) +#define ___bpf_ctx_arg12(n, t, x, args...) \ + , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) +#define ___bpf_ctx_arg(args...) \ + ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) #define ___bpf_ctx_decl0() -#define ___bpf_ctx_decl1(t, x) , t x -#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) -#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) -#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) -#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) -#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) -#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) -#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) -#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) -#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) -#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) -#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) -#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) +#define ___bpf_ctx_decl1(t, x) , t x +#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) +#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) +#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) +#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) +#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) +#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) +#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) +#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) +#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) +#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) +#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) +#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) /* * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct @@ -772,21 +1044,19 @@ ____##name(unsigned long long *ctx, ##args) * ... * } */ -#define BPF_PROG2(name, args...) \ -name(unsigned long long *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \ -typeof(name(0)) name(unsigned long long *ctx) \ -{ \ - return ____##name(ctx ___bpf_ctx_arg(args)); \ -} \ -static __always_inline typeof(name(0)) \ -____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) +#define BPF_PROG2(name, args...) \ + name(unsigned long long *ctx); \ + static __always_inline typeof(name(0)) ____##name( \ + unsigned long long *ctx ___bpf_ctx_decl(args)); \ + typeof(name(0)) name(unsigned long long *ctx) { \ + return ____##name(ctx ___bpf_ctx_arg(args)); \ + } \ + static __always_inline typeof(name(0)) ____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) struct pt_regs; -#define ___bpf_kprobe_args0() ctx -#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) +#define ___bpf_kprobe_args0() ctx +#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) #define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx) #define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx) #define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx) @@ -794,7 +1064,7 @@ struct pt_regs; #define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (void *)PT_REGS_PARM6(ctx) #define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (void *)PT_REGS_PARM7(ctx) #define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (void *)PT_REGS_PARM8(ctx) -#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) +#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) /* * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for @@ -806,22 +1076,19 @@ struct pt_regs; * Original struct pt_regs* context is preserved as 'ctx' argument. This might * be necessary when using BPF helpers like bpf_perf_event_output(). */ -#define BPF_KPROBE(name, args...) \ -name(struct pt_regs *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_kprobe_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args) - -#define ___bpf_kretprobe_args0() ctx -#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) +#define BPF_KPROBE(name, args...) \ + name(struct pt_regs *ctx); \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args); \ + typeof(name(0)) name(struct pt_regs *ctx) { \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") return ____##name( \ + ___bpf_kprobe_args(args)); \ + _Pragma("GCC diagnostic pop") \ + } \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) + +#define ___bpf_kretprobe_args0() ctx +#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) #define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) /* @@ -830,40 +1097,50 @@ ____##name(struct pt_regs *ctx, ##args) * arguments, because they will be clobbered by the time probed function * returns. */ -#define BPF_KRETPROBE(name, args...) \ -name(struct pt_regs *ctx); \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_kretprobe_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +#define BPF_KRETPROBE(name, args...) \ + name(struct pt_regs *ctx); \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args); \ + typeof(name(0)) name(struct pt_regs *ctx) { \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") return ____##name( \ + ___bpf_kretprobe_args(args)); \ + _Pragma("GCC diagnostic pop") \ + } \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) /* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ -#define ___bpf_syscall_args0() ctx -#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs) -#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs) -#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs) -#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs) -#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs) -#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (void *)PT_REGS_PARM6_SYSCALL(regs) -#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs) -#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) +#define ___bpf_syscall_args0() ctx +#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs) +#define ___bpf_syscall_args2(x, args...) \ + ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs) +#define ___bpf_syscall_args3(x, args...) \ + ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs) +#define ___bpf_syscall_args4(x, args...) \ + ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs) +#define ___bpf_syscall_args5(x, args...) \ + ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs) +#define ___bpf_syscall_args6(x, args...) \ + ___bpf_syscall_args5(args), (void *)PT_REGS_PARM6_SYSCALL(regs) +#define ___bpf_syscall_args7(x, args...) \ + ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs) +#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) /* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ -#define ___bpf_syswrap_args0() ctx -#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (void *)PT_REGS_PARM6_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (void *)PT_REGS_PARM7_CORE_SYSCALL(regs) -#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) +#define ___bpf_syswrap_args0() ctx +#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args2(x, args...) \ + ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args3(x, args...) \ + ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args4(x, args...) \ + ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args5(x, args...) \ + ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args6(x, args...) \ + ___bpf_syswrap_args5(args), (void *)PT_REGS_PARM6_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args7(x, args...) \ + ___bpf_syswrap_args6(args), (void *)PT_REGS_PARM7_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) /* * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for @@ -889,26 +1166,19 @@ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) * * This macro relies on BPF CO-RE support and virtual __kconfig externs. */ -#define BPF_KSYSCALL(name, args...) \ -name(struct pt_regs *ctx); \ -extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args); \ -typeof(name(0)) name(struct pt_regs *ctx) \ -{ \ - struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ - ? (struct pt_regs *)PT_REGS_PARM1(ctx) \ - : ctx; \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - if (LINUX_HAS_SYSCALL_WRAPPER) \ - return ____##name(___bpf_syswrap_args(args)); \ - else \ - return ____##name(___bpf_syscall_args(args)); \ - _Pragma("GCC diagnostic pop") \ -} \ -static __always_inline typeof(name(0)) \ -____##name(struct pt_regs *ctx, ##args) +#define BPF_KSYSCALL(name, args...) \ + name(struct pt_regs *ctx); \ + extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args); \ + typeof(name(0)) name(struct pt_regs *ctx) { \ + struct pt_regs *regs = \ + LINUX_HAS_SYSCALL_WRAPPER ? (struct pt_regs *)PT_REGS_PARM1(ctx) : ctx; \ + _Pragma("GCC diagnostic push") _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") if ( \ + LINUX_HAS_SYSCALL_WRAPPER) return ____##name(___bpf_syswrap_args(args)); \ + else return ____##name(___bpf_syscall_args(args)); \ + _Pragma("GCC diagnostic pop") \ + } \ + static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) #define BPF_KPROBE_SYSCALL BPF_KSYSCALL @@ -916,7 +1186,7 @@ ____##name(struct pt_regs *ctx, ##args) * but are named way less confusingly for SEC("uprobe") and SEC("uretprobe") * use cases. */ -#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args) -#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args) +#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args) +#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args) #endif diff --git a/internal/include/otel_types.h b/internal/include/otel_types.h index c8f92fd354..761e5ee5aa 100644 --- a/internal/include/otel_types.h +++ b/internal/include/otel_types.h @@ -23,121 +23,124 @@ volatile const u64 attr_type_stringslice; /* Definitions should mimic structs defined in go.opentelemetry.io/otel/attribute */ typedef struct go_otel_attr_value { - u64 vtype; - u64 numeric; - struct go_string string; - struct go_iface slice; + u64 vtype; + u64 numeric; + struct go_string string; + struct go_iface slice; } go_otel_attr_value_t; typedef struct go_otel_key_value { - struct go_string key; - go_otel_attr_value_t value; + struct go_string key; + go_otel_attr_value_t value; } go_otel_key_value_t; -#define OTEL_ATTRIBUTE_KEY_MAX_LEN (32) -#define OTEL_ATTRIBUTE_VALUE_MAX_LEN (128) -#define OTEL_ATTRUBUTE_MAX_COUNT (16) +#define OTEL_ATTRIBUTE_KEY_MAX_LEN (32) +#define OTEL_ATTRIBUTE_VALUE_MAX_LEN (128) +#define OTEL_ATTRUBUTE_MAX_COUNT (16) typedef struct otel_attirbute { - u16 val_length; - u8 vtype; - u8 reserved; - char key[OTEL_ATTRIBUTE_KEY_MAX_LEN]; - char value[OTEL_ATTRIBUTE_VALUE_MAX_LEN]; + u16 val_length; + u8 vtype; + u8 reserved; + char key[OTEL_ATTRIBUTE_KEY_MAX_LEN]; + char value[OTEL_ATTRIBUTE_VALUE_MAX_LEN]; } otel_attirbute_t; typedef struct otel_attributes { - otel_attirbute_t attrs[OTEL_ATTRUBUTE_MAX_COUNT]; - u8 valid_attrs; -}__attribute__((packed)) otel_attributes_t; - -static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_value_t *go_attr_value) -{ - u64 vtype = go_attr_value->vtype; - - // Constant size values - if (vtype == attr_type_bool || - vtype == attr_type_int64 || - vtype == attr_type_float64) { - bpf_probe_read(attr->value, sizeof(s64), &go_attr_value->numeric); - return true; - } - - // String values - if (vtype == attr_type_string) { - if (go_attr_value->string.len >= OTEL_ATTRIBUTE_VALUE_MAX_LEN) { - bpf_printk("Aattribute string value is too long\n"); - return false; - } - long res = bpf_probe_read_user(attr->value, go_attr_value->string.len & (OTEL_ATTRIBUTE_VALUE_MAX_LEN -1), go_attr_value->string.str); - return res == 0; - } - - // TODO (#525): handle slices - return false; + otel_attirbute_t attrs[OTEL_ATTRUBUTE_MAX_COUNT]; + u8 valid_attrs; +} __attribute__((packed)) otel_attributes_t; + +static __always_inline bool set_attr_value(otel_attirbute_t *attr, + go_otel_attr_value_t *go_attr_value) { + u64 vtype = go_attr_value->vtype; + + // Constant size values + if (vtype == attr_type_bool || vtype == attr_type_int64 || vtype == attr_type_float64) { + bpf_probe_read(attr->value, sizeof(s64), &go_attr_value->numeric); + return true; + } + + // String values + if (vtype == attr_type_string) { + if (go_attr_value->string.len >= OTEL_ATTRIBUTE_VALUE_MAX_LEN) { + bpf_printk("Aattribute string value is too long\n"); + return false; + } + long res = + bpf_probe_read_user(attr->value, + go_attr_value->string.len & (OTEL_ATTRIBUTE_VALUE_MAX_LEN - 1), + go_attr_value->string.str); + return res == 0; + } + + // TODO (#525): handle slices + return false; } -static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slice_len, otel_attributes_t *enc_attrs) -{ - if (attrs_buf == NULL){ - return; - } - - if (slice_len < 1) { - return; - } - - u8 num_attrs = slice_len < OTEL_ATTRUBUTE_MAX_COUNT ? slice_len : OTEL_ATTRUBUTE_MAX_COUNT; - go_otel_key_value_t *go_attr = (go_otel_key_value_t*)attrs_buf; - go_otel_attr_value_t go_attr_value = {0}; - struct go_string go_str = {0}; - u8 valid_attrs = enc_attrs->valid_attrs; - if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { - return; - } - - for (u8 go_attr_index = 0; go_attr_index < OTEL_ATTRUBUTE_MAX_COUNT; go_attr_index++) { - if (go_attr_index >= slice_len) { - break; - } - __builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t)); - // Read the value struct - bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value); - - if (go_attr_value.vtype == attr_type_invalid) { - continue; - } - - // Read the key string - bpf_probe_read(&go_str, sizeof(struct go_string), &go_attr[go_attr_index].key); - if (go_str.len >= OTEL_ATTRIBUTE_KEY_MAX_LEN) { - // key string is too large - bpf_printk("Attribute key string is too long\n"); - continue; - } - - // Need to check valid_attrs otherwise the ebpf verifier thinks it's possible to exceed - // the max register value for a downstream call, even though it's not possible with - // this same check at the end of the loop. - if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { - break; - } - - bpf_probe_read_user(enc_attrs->attrs[valid_attrs].key, go_str.len & (OTEL_ATTRIBUTE_KEY_MAX_LEN -1), go_str.str); - - if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) { - continue; - } - - enc_attrs->attrs[valid_attrs].vtype = go_attr_value.vtype; - valid_attrs++; - if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { - // No more space for attributes - break; - } - } - - enc_attrs->valid_attrs = valid_attrs; +static __always_inline void +convert_go_otel_attributes(void *attrs_buf, u64 slice_len, otel_attributes_t *enc_attrs) { + if (attrs_buf == NULL) { + return; + } + + if (slice_len < 1) { + return; + } + + u8 num_attrs = slice_len < OTEL_ATTRUBUTE_MAX_COUNT ? slice_len : OTEL_ATTRUBUTE_MAX_COUNT; + go_otel_key_value_t *go_attr = (go_otel_key_value_t *)attrs_buf; + go_otel_attr_value_t go_attr_value = {0}; + struct go_string go_str = {0}; + u8 valid_attrs = enc_attrs->valid_attrs; + if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { + return; + } + + for (u8 go_attr_index = 0; go_attr_index < OTEL_ATTRUBUTE_MAX_COUNT; go_attr_index++) { + if (go_attr_index >= slice_len) { + break; + } + __builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t)); + // Read the value struct + bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value); + + if (go_attr_value.vtype == attr_type_invalid) { + continue; + } + + // Read the key string + bpf_probe_read(&go_str, sizeof(struct go_string), &go_attr[go_attr_index].key); + if (go_str.len >= OTEL_ATTRIBUTE_KEY_MAX_LEN) { + // key string is too large + bpf_printk("Attribute key string is too long\n"); + continue; + } + + // Need to check valid_attrs otherwise the ebpf verifier thinks it's possible to exceed + // the max register value for a downstream call, even though it's not possible with + // this same check at the end of the loop. + if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { + break; + } + + bpf_probe_read_user(enc_attrs->attrs[valid_attrs].key, + go_str.len & (OTEL_ATTRIBUTE_KEY_MAX_LEN - 1), + go_str.str); + + if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) { + continue; + } + + enc_attrs->attrs[valid_attrs].vtype = go_attr_value.vtype; + valid_attrs++; + if (valid_attrs >= OTEL_ATTRUBUTE_MAX_COUNT) { + // No more space for attributes + break; + } + } + + enc_attrs->valid_attrs = valid_attrs; } #endif diff --git a/internal/include/sdk.h b/internal/include/sdk.h index 3413e0e36c..574577070b 100644 --- a/internal/include/sdk.h +++ b/internal/include/sdk.h @@ -25,7 +25,7 @@ struct otel_span_t { struct { __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); + __type(key, void *); __type(value, struct otel_span_t); __uint(max_entries, MAX_CONCURRENT); } active_spans_by_span_ptr SEC(".maps"); @@ -60,5 +60,4 @@ static __always_inline long write_span_context(void *go_sc, struct span_context return 0; } - #endif // SDK_H diff --git a/internal/include/trace/sampling.h b/internal/include/trace/sampling.h index 016b13e5c4..43778630df 100644 --- a/internal/include/trace/sampling.h +++ b/internal/include/trace/sampling.h @@ -46,35 +46,34 @@ typedef struct sampling_parameters { } sampling_parameters_t; struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(key_size, sizeof(sampler_id_t)); - __uint(value_size, sizeof(struct sampling_config)); - __uint(max_entries, MAX_SAMPLERS); + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(sampler_id_t)); + __uint(value_size, sizeof(struct sampling_config)); + __uint(max_entries, MAX_SAMPLERS); } samplers_config_map SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __uint(key_size, sizeof(u32)); - __uint(value_size, sizeof(sampler_id_t)); - __uint(max_entries, 1); + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(sampler_id_t)); + __uint(max_entries, 1); } probe_active_sampler_map SEC(".maps"); static const u8 FLAG_SAMPLED = 1; -static __always_inline bool trace_flags_is_sampled(u8 flags) -{ +static __always_inline bool trace_flags_is_sampled(u8 flags) { return ((flags & FLAG_SAMPLED) == FLAG_SAMPLED); } -static __always_inline bool is_sampled(struct span_context *ctx) -{ +static __always_inline bool is_sampled(struct span_context *ctx) { return trace_flags_is_sampled(ctx->TraceFlags); } // This value should be in sync with user-space code which configures the sampler -static const u64 sampling_rate_denominator = ((1ULL<<32) - 1); +static const u64 sampling_rate_denominator = ((1ULL << 32) - 1); -static __always_inline bool _traceIDRatioSampler_should_sample(u64 sampling_rate_numerator, u8 *trace_id) { +static __always_inline bool _traceIDRatioSampler_should_sample(u64 sampling_rate_numerator, + u8 *trace_id) { if (sampling_rate_numerator == 0) { return false; } @@ -89,20 +88,25 @@ static __always_inline bool _traceIDRatioSampler_should_sample(u64 sampling_rate return (trace_id_num >> 1) < trace_id_upper_bound; } -static __always_inline bool traceIDRatioSampler_should_sample(struct sampling_config* config, sampling_parameters_t *params) { - return _traceIDRatioSampler_should_sample(config->config_data.sampling_rate_numerator, params->trace_id); +static __always_inline bool traceIDRatioSampler_should_sample(struct sampling_config *config, + sampling_parameters_t *params) { + return _traceIDRatioSampler_should_sample(config->config_data.sampling_rate_numerator, + params->trace_id); } -static __always_inline bool alwaysOnSampler_should_sample(struct sampling_config* config, sampling_parameters_t *params) { +static __always_inline bool alwaysOnSampler_should_sample(struct sampling_config *config, + sampling_parameters_t *params) { return true; } -static __always_inline bool alwaysOffSampler_should_sample(struct sampling_config* config, sampling_parameters_t *params) { +static __always_inline bool alwaysOffSampler_should_sample(struct sampling_config *config, + sampling_parameters_t *params) { return false; } -static __always_inline bool parentBasedSampler_should_sample(struct sampling_config* config, sampling_parameters_t *params) { - sampler_id_t sampler_id; +static __always_inline bool parentBasedSampler_should_sample(struct sampling_config *config, + sampling_parameters_t *params) { + sampler_id_t sampler_id; if (params->psc == NULL) { sampler_id = config->config_data.parent_based.root; } else { @@ -127,20 +131,21 @@ static __always_inline bool parentBasedSampler_should_sample(struct sampling_con } switch (base_config->type) { - case ALWAYS_ON: - return alwaysOnSampler_should_sample(base_config, params); - case ALWAYS_OFF: - return alwaysOffSampler_should_sample(base_config, params); - case TRACE_ID_RATIO: - return traceIDRatioSampler_should_sample(base_config, params); - default: - return false; + case ALWAYS_ON: + return alwaysOnSampler_should_sample(base_config, params); + case ALWAYS_OFF: + return alwaysOffSampler_should_sample(base_config, params); + case TRACE_ID_RATIO: + return traceIDRatioSampler_should_sample(base_config, params); + default: + return false; } } static __always_inline bool should_sample(sampling_parameters_t *params) { u32 active_sampler_map_key = 0; - sampler_id_t *active_sampler_id = bpf_map_lookup_elem(&probe_active_sampler_map, &active_sampler_map_key); + sampler_id_t *active_sampler_id = + bpf_map_lookup_elem(&probe_active_sampler_map, &active_sampler_map_key); if (active_sampler_id == NULL) { bpf_printk("No active sampler found\n"); return false; @@ -153,16 +158,16 @@ static __always_inline bool should_sample(sampling_parameters_t *params) { } switch (config->type) { - case ALWAYS_ON: - return alwaysOnSampler_should_sample(config, params); - case ALWAYS_OFF: - return alwaysOffSampler_should_sample(config, params); - case TRACE_ID_RATIO: - return traceIDRatioSampler_should_sample(config, params); - case PARENT_BASED: - return parentBasedSampler_should_sample(config, params); - default: - return false; + case ALWAYS_ON: + return alwaysOnSampler_should_sample(config, params); + case ALWAYS_OFF: + return alwaysOffSampler_should_sample(config, params); + case TRACE_ID_RATIO: + return traceIDRatioSampler_should_sample(config, params); + case PARENT_BASED: + return parentBasedSampler_should_sample(config, params); + default: + return false; } } diff --git a/internal/include/trace/span_context.h b/internal/include/trace/span_context.h index e3db779b29..315e009827 100644 --- a/internal/include/trace/span_context.h +++ b/internal/include/trace/span_context.h @@ -16,9 +16,7 @@ #define TRACE_FLAGS_SIZE 1 #define TRACE_FLAGS_STRING_SIZE 2 - -struct span_context -{ +struct span_context { u8 TraceID[TRACE_ID_SIZE]; u8 SpanID[SPAN_ID_SIZE]; u8 TraceFlags; @@ -27,7 +25,8 @@ struct span_context // Fill the child span context based on the parent span context, // generating a new span id and copying the trace id and trace flags -static __always_inline void get_span_context_from_parent(struct span_context *parent, struct span_context *child) { +static __always_inline void get_span_context_from_parent(struct span_context *parent, + struct span_context *child) { copy_byte_arrays(parent->TraceID, child->TraceID, TRACE_ID_SIZE); generate_random_bytes(child->SpanID, SPAN_ID_SIZE); } @@ -38,8 +37,7 @@ static __always_inline void get_root_span_context(struct span_context *sc) { generate_random_bytes(sc->SpanID, SPAN_ID_SIZE); } -static __always_inline void span_context_to_w3c_string(struct span_context *ctx, char *buff) -{ +static __always_inline void span_context_to_w3c_string(struct span_context *ctx, char *buff) { // W3C format: version (2 chars) - trace id (32 chars) - span id (16 chars) - sampled (2 chars) char *out = buff; @@ -62,8 +60,7 @@ static __always_inline void span_context_to_w3c_string(struct span_context *ctx, bytes_to_hex_string(&ctx->TraceFlags, TRACE_FLAGS_SIZE, out); } -static __always_inline void w3c_string_to_span_context(char *str, struct span_context *ctx) -{ +static __always_inline void w3c_string_to_span_context(char *str, struct span_context *ctx) { u32 trace_id_start_pos = 3; u32 span_id_start_pos = 36; u32 trace_flags_start_pos = 53; diff --git a/internal/include/trace/span_output.h b/internal/include/trace/span_output.h index 119d58779a..170bc865d2 100644 --- a/internal/include/trace/span_output.h +++ b/internal/include/trace/span_output.h @@ -7,14 +7,14 @@ #ifndef _SPAN_OUTPUT_H_ #define _SPAN_OUTPUT_H_ -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); } events SEC(".maps"); // Output a record to the perf buffer. If the span context is sampled, the record is outputted. // Returns 0 on success, negative error code on failure. -static __always_inline long output_span_event(void *ctx, void *data, u64 size, struct span_context *sc) { +static __always_inline long +output_span_event(void *ctx, void *data, u64 size, struct span_context *sc) { bool sampled = (sc != NULL && is_sampled(sc)); if (sampled) { return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, data, size); diff --git a/internal/include/trace/start_span.h b/internal/include/trace/start_span.h index 8cd57c1807..7a9c127a11 100644 --- a/internal/include/trace/start_span.h +++ b/internal/include/trace/start_span.h @@ -32,7 +32,8 @@ typedef struct start_span_params { static __always_inline void start_span(start_span_params_t *params) { long found_parent = -1; if (params->get_parent_span_context_fn != NULL) { - found_parent = params->get_parent_span_context_fn(params->get_parent_span_context_arg, params->psc); + found_parent = + params->get_parent_span_context_fn(params->get_parent_span_context_arg, params->psc); } else { struct span_context *local_psc = get_parent_span_context(params->go_context); if (local_psc != NULL) { diff --git a/internal/include/uprobe.h b/internal/include/uprobe.h index fc47e73996..23b98e845b 100644 --- a/internal/include/uprobe.h +++ b/internal/include/uprobe.h @@ -10,10 +10,10 @@ #include "go_types.h" #include "trace/span_output.h" -#define BASE_SPAN_PROPERTIES \ - u64 start_time; \ - u64 end_time; \ - struct span_context sc; \ +#define BASE_SPAN_PROPERTIES \ + u64 start_time; \ + u64 end_time; \ + struct span_context sc; \ struct span_context psc; // Common flow for uprobe return: @@ -23,20 +23,20 @@ // 4. Submit the constructed event to the agent code using perf buffer events_map // 5. Delete the span from the global active spans map (in case the span is not tracked in the active spans map, this will be a no-op) // 6. Delete the span from the uprobe_context_map -#define UPROBE_RETURN(name, event_type, uprobe_context_map) \ -SEC("uprobe/##name##") \ -int uprobe_##name##_Returns(struct pt_regs *ctx) { \ - void *key = (void *)GOROUTINE(ctx); \ - event_type *event = bpf_map_lookup_elem(&uprobe_context_map, &key); \ - if (event == NULL) { \ - bpf_printk("event is NULL in ret probe"); \ - return 0; \ - } \ - event->end_time = bpf_ktime_get_ns(); \ - output_span_event(ctx, event, sizeof(event_type), &event->sc); \ - stop_tracking_span(&event->sc, &event->psc); \ - bpf_map_delete_elem(&uprobe_context_map, &key); \ - return 0; \ -} +#define UPROBE_RETURN(name, event_type, uprobe_context_map) \ + SEC("uprobe/##name##") \ + int uprobe_##name##_Returns(struct pt_regs *ctx) { \ + void *key = (void *)GOROUTINE(ctx); \ + event_type *event = bpf_map_lookup_elem(&uprobe_context_map, &key); \ + if (event == NULL) { \ + bpf_printk("event is NULL in ret probe"); \ + return 0; \ + } \ + event->end_time = bpf_ktime_get_ns(); \ + output_span_event(ctx, event, sizeof(event_type), &event->sc); \ + stop_tracking_span(&event->sc, &event->psc); \ + bpf_map_delete_elem(&uprobe_context_map, &key); \ + return 0; \ + } #endif diff --git a/internal/include/utils.h b/internal/include/utils.h index 7190561f2f..54c00dd6c8 100644 --- a/internal/include/utils.h +++ b/internal/include/utils.h @@ -6,13 +6,9 @@ #include "bpf_helpers.h" - -static __always_inline bool bpf_memcmp(char *s1, char *s2, s32 size) -{ - for (int i = 0; i < size; i++) - { - if (s1[i] != s2[i]) - { +static __always_inline bool bpf_memcmp(char *s1, char *s2, s32 size) { + for (int i = 0; i < size; i++) { + if (s1[i] != s2[i]) { return false; } } @@ -32,10 +28,8 @@ static __always_inline int bpf_memicmp(const char *s1, const char *s2, s32 size) return 0; } -static __always_inline void generate_random_bytes(unsigned char *buff, u32 size) -{ - for (int i = 0; i < (size / 4); i++) - { +static __always_inline void generate_random_bytes(unsigned char *buff, u32 size) { + for (int i = 0; i < (size / 4); i++) { u32 random = bpf_get_prandom_u32(); buff[(4 * i)] = (random >> 24) & 0xFF; buff[(4 * i) + 1] = (random >> 16) & 0xFF; @@ -45,21 +39,17 @@ static __always_inline void generate_random_bytes(unsigned char *buff, u32 size) } char hex[16] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; -static __always_inline void bytes_to_hex_string(unsigned char *pin, u32 size, char *out) -{ +static __always_inline void bytes_to_hex_string(unsigned char *pin, u32 size, char *out) { char *pout = out; int out_index = 0; - for (u32 i = 0; i < size; i++) - { + for (u32 i = 0; i < size; i++) { *pout++ = hex[(*pin >> 4) & 0xF]; *pout++ = hex[(*pin++) & 0xF]; } } -static __always_inline void hex_string_to_bytes(char *str, u32 size, unsigned char *out) -{ - for (int i = 0; i < (size / 2); i++) - { +static __always_inline void hex_string_to_bytes(char *str, u32 size, unsigned char *out) { + for (int i = 0; i < (size / 2); i++) { char ch0 = str[2 * i]; char ch1 = str[2 * i + 1]; u8 nib0 = (ch0 & 0xF) + (ch0 >> 6) | ((ch0 >> 3) & 0x8); @@ -68,28 +58,21 @@ static __always_inline void hex_string_to_bytes(char *str, u32 size, unsigned ch } } -static __always_inline void copy_byte_arrays(unsigned char *src, unsigned char *dst, u32 size) -{ - for (int i = 0; i < size; i++) - { +static __always_inline void copy_byte_arrays(unsigned char *src, unsigned char *dst, u32 size) { + for (int i = 0; i < size; i++) { dst[i] = src[i]; } } -static __always_inline void bpf_memset(unsigned char *dst, u32 size, unsigned char value) -{ - for (int i = 0; i < size; i++) - { +static __always_inline void bpf_memset(unsigned char *dst, u32 size, unsigned char value) { + for (int i = 0; i < size; i++) { dst[i] = value; } } -static __always_inline bool bpf_is_zero(unsigned char *buff, u32 size) -{ - for (int i = 0; i < size; i++) - { - if (buff[i] != 0) - { +static __always_inline bool bpf_is_zero(unsigned char *buff, u32 size) { + for (int i = 0; i < size; i++) { + if (buff[i] != 0) { return false; } } diff --git a/internal/pkg/instrumentation/bpf/database/sql/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/database/sql/bpf/probe.bpf.c index 8f21dca180..302c5b5331 100644 --- a/internal/pkg/instrumentation/bpf/database/sql/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/database/sql/bpf/probe.bpf.c @@ -19,10 +19,10 @@ struct sql_request_t { }; struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct sql_request_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, struct sql_request_t); + __uint(max_entries, MAX_CONCURRENT); } sql_events SEC(".maps"); // Injected in init diff --git a/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer/bpf/probe.bpf.c index 5a7641ccfd..21fe1621df 100644 --- a/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/consumer/bpf/probe.bpf.c @@ -25,31 +25,30 @@ struct kafka_request_t { char consumer_group[MAX_CONSUMER_GROUP_SIZE]; s64 offset; s64 partition; -}__attribute__((packed)); +} __attribute__((packed)); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct kafka_request_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, struct kafka_request_t); + __uint(max_entries, MAX_CONCURRENT); } kafka_events SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, void*); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, void *); + __uint(max_entries, MAX_CONCURRENT); } goroutine_to_go_context SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, void*); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, void *); + __uint(max_entries, MAX_CONCURRENT); } kafka_reader_to_conn SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct kafka_request_t)); @@ -74,7 +73,8 @@ volatile const u64 reader_config_group_id_pos; #define MAX_HEADERS 20 -static __always_inline long extract_span_context_from_headers(void *message, struct span_context *parent_span_context) { +static __always_inline long +extract_span_context_from_headers(void *message, struct span_context *parent_span_context) { // Read the headers slice descriptor void *headers = (void *)(message + message_headers_pos); struct go_slice headers_slice = {0}; @@ -125,15 +125,16 @@ int uprobe_FetchMessage(struct pt_regs *ctx) { get_Go_context(ctx, 2, 0, true, &go_context); void *goroutine = (void *)GOROUTINE(ctx); struct kafka_request_t *kafka_request = bpf_map_lookup_elem(&kafka_events, &goroutine); - if (kafka_request == NULL) - { + if (kafka_request == NULL) { // The current goroutine has no kafka request, // this can happen in the first time FetchMessage is called // Save the context for the return probe for in-process context propagation goto save_context; } - get_go_string_from_user_ptr((void *)(reader + reader_config_pos + reader_config_group_id_pos), kafka_request->consumer_group, sizeof(kafka_request->consumer_group)); + get_go_string_from_user_ptr((void *)(reader + reader_config_pos + reader_config_group_id_pos), + kafka_request->consumer_group, + sizeof(kafka_request->consumer_group)); kafka_request->end_time = bpf_ktime_get_ns(); output_span_event(ctx, kafka_request, sizeof(*kafka_request), &kafka_request->sc); @@ -156,9 +157,9 @@ int uprobe_FetchMessage_Returns(struct pt_regs *ctx) { in a hash map to be read by the entry probe of FetchMessage, which will end this span */ void *goroutine = (void *)GOROUTINE(ctx); u32 map_id = 0; - struct kafka_request_t *kafka_request = bpf_map_lookup_elem(&kafka_request_storage_map, &map_id); - if (kafka_request == NULL) - { + struct kafka_request_t *kafka_request = + bpf_map_lookup_elem(&kafka_request_storage_map, &map_id); + if (kafka_request == NULL) { bpf_printk("uuprobe/sendMessage: kafka_request is NULL"); return 0; } @@ -182,11 +183,16 @@ int uprobe_FetchMessage_Returns(struct pt_regs *ctx) { // Collecting message attributes // topic - get_go_string_from_user_ptr((void *)(message + message_topic_pos), kafka_request->topic, sizeof(kafka_request->topic)); + get_go_string_from_user_ptr( + (void *)(message + message_topic_pos), kafka_request->topic, sizeof(kafka_request->topic)); // partition - bpf_probe_read(&kafka_request->partition, sizeof(kafka_request->partition), (void *)(message + message_partition_pos)); + bpf_probe_read(&kafka_request->partition, + sizeof(kafka_request->partition), + (void *)(message + message_partition_pos)); // offset - bpf_probe_read(&kafka_request->offset, sizeof(kafka_request->offset), (void *)(message + message_offset_pos)); + bpf_probe_read(&kafka_request->offset, + sizeof(kafka_request->offset), + (void *)(message + message_offset_pos)); // Key is a byte slice, first read the slice descriptor struct go_slice key_slice = {0}; bpf_probe_read(&key_slice, sizeof(key_slice), (void *)(message + message_key_pos)); @@ -196,7 +202,7 @@ int uprobe_FetchMessage_Returns(struct pt_regs *ctx) { bpf_probe_read(kafka_request->key, size_to_read, key_slice.array); bpf_map_update_elem(&kafka_events, &goroutine, kafka_request, 0); - + // We are start tracking the consumer span in the return probe, // hence we can't read Go's context directly from the registers as we usually do. // Using the goroutine address as a key to the map that contains the context. diff --git a/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer/bpf/probe.bpf.c index bfe61141c9..bf2c8a4750 100644 --- a/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/github.com/segmentio/kafka-go/producer/bpf/probe.bpf.c @@ -38,17 +38,16 @@ struct kafka_request_t { struct message_attributes_t msgs[MAX_BATCH_SIZE]; char global_topic[MAX_TOPIC_SIZE]; u64 valid_messages; -}__attribute__((packed)); +} __attribute__((packed)); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct kafka_request_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, struct kafka_request_t); + __uint(max_entries, MAX_CONCURRENT); } kafka_events SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct kafka_request_t)); @@ -70,7 +69,8 @@ volatile const u64 message_time_pos; volatile const u64 writer_topic_pos; #ifndef NO_HEADER_PROPAGATION -static __always_inline int build_contxet_header(struct kafka_header_t *header, struct span_context *span_ctx) { +static __always_inline int build_contxet_header(struct kafka_header_t *header, + struct span_context *span_ctx) { if (header == NULL || span_ctx == NULL) { bpf_printk("build_contxt_header: Invalid arguments"); return -1; @@ -111,10 +111,12 @@ static __always_inline int inject_kafka_header(void *message, struct kafka_heade } #endif -static __always_inline long collect_kafka_attributes(void *message, struct message_attributes_t *attrs, bool collect_topic) { +static __always_inline long +collect_kafka_attributes(void *message, struct message_attributes_t *attrs, bool collect_topic) { if (collect_topic) { // Topic might be globally set for a writer, or per message - get_go_string_from_user_ptr((void *)(message + message_topic_pos), attrs->topic, sizeof(attrs->topic)); + get_go_string_from_user_ptr( + (void *)(message + message_topic_pos), attrs->topic, sizeof(attrs->topic)); } // Key is a byte slice, first read the slice @@ -140,16 +142,15 @@ int uprobe_WriteMessages(struct pt_regs *ctx) { void *key = (void *)GOROUTINE(ctx); void *kafka_request_ptr = bpf_map_lookup_elem(&kafka_events, &key); - if (kafka_request_ptr != NULL) - { + if (kafka_request_ptr != NULL) { bpf_printk("uprobe/WriteMessages already tracked with the current context"); return 0; } u32 zero_id = 0; - struct kafka_request_t *zero_kafka_request = bpf_map_lookup_elem(&kafka_request_storage_map, &zero_id); - if (zero_kafka_request == NULL) - { + struct kafka_request_t *zero_kafka_request = + bpf_map_lookup_elem(&kafka_request_storage_map, &zero_id); + if (zero_kafka_request == NULL) { bpf_printk("uuprobe/WriteMessages: zero_kafka_request is NULL"); return 0; } @@ -158,7 +159,8 @@ int uprobe_WriteMessages(struct pt_regs *ctx) { // Zero the span we are about to build, eBPF doesn't support memset of large structs (more than 1024 bytes) bpf_map_update_elem(&kafka_request_storage_map, &actual_id, zero_kafka_request, BPF_ANY); // Get a pointer to the zeroed span - struct kafka_request_t *kafka_request = bpf_map_lookup_elem(&kafka_request_storage_map, &actual_id); + struct kafka_request_t *kafka_request = + bpf_map_lookup_elem(&kafka_request_storage_map, &actual_id); if (kafka_request == NULL) { bpf_printk("uprobe/WriteMessages: Failed to get kafka_request"); return 0; @@ -177,7 +179,9 @@ int uprobe_WriteMessages(struct pt_regs *ctx) { start_span(&start_span_params); // Try to get a global topic from Writer - bool global_topic = get_go_string_from_user_ptr((void *)(writer + writer_topic_pos), kafka_request->global_topic, sizeof(kafka_request->global_topic)); + bool global_topic = get_go_string_from_user_ptr((void *)(writer + writer_topic_pos), + kafka_request->global_topic, + sizeof(kafka_request->global_topic)); void *msg_ptr = msgs_array; struct kafka_header_t header = {0}; @@ -203,7 +207,9 @@ int uprobe_WriteMessages(struct pt_regs *ctx) { // Copy the trace id and trace flags from the first message. This means the sampling decision is done on the first message, // and all the messages in the batch will have the same trace id and trace flags. kafka_request->msgs[i].sc.TraceFlags = kafka_request->msgs[0].sc.TraceFlags; - __builtin_memcpy(kafka_request->msgs[i].sc.TraceID, kafka_request->msgs[0].sc.TraceID, TRACE_ID_SIZE); + __builtin_memcpy(kafka_request->msgs[i].sc.TraceID, + kafka_request->msgs[0].sc.TraceID, + TRACE_ID_SIZE); } #ifndef NO_HEADER_PROPAGATION @@ -219,7 +225,6 @@ int uprobe_WriteMessages(struct pt_regs *ctx) { msg_ptr = msg_ptr + msg_size; } - bpf_map_update_elem(&kafka_events, &key, kafka_request, 0); // don't need to start tracking the span, as we don't have a context to propagate locally return 0; diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c index 4f043acbe2..e228bd16fa 100644 --- a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/auto/sdk/bpf/probe.bpf.c @@ -54,7 +54,7 @@ int uprobe_Tracer_start(struct pt_regs *ctx) { .psc = &otel_span.psc, .sc = &otel_span.sc, .get_parent_span_context_fn = NULL, - .get_parent_span_context_arg = NULL, // Default to new root. + .get_parent_span_context_arg = NULL, // Default to new root. }; start_span(¶ms); @@ -109,7 +109,8 @@ int uprobe_Span_ended(struct pt_regs *ctx) { bpf_map_delete_elem(&active_spans_by_span_ptr, &span_ptr); // Do not output un-sampled span data. - if (!sampled) return 0; + if (!sampled) + return 0; u64 len = (u64)get_argument(ctx, 3); if (len > MAX_SIZE) { @@ -149,7 +150,7 @@ int uprobe_Span_ended(struct pt_regs *ctx) { // Do not send the whole size.buf if it is not needed. u64 size = sizeof(event->size) + event->size; // Make the verifier happy, ensure no unbounded memory access. - if (size < sizeof(struct event_t)+1) { + if (size < sizeof(struct event_t) + 1) { return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, size); } bpf_printk("write too large: %d", event->size); diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c index cdb695c96f..d8ac0afbfe 100644 --- a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/trace/bpf/probe.bpf.c @@ -58,7 +58,8 @@ int uprobe_tracerProvider(struct pt_regs *ctx) { // Signal this uprobe should be unloaded. struct control_t ctrl = {1}; - return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t)); + return bpf_perf_event_output( + ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t)); } // This instrumentation attaches a uprobe to the following function: @@ -92,7 +93,7 @@ int uprobe_Tracer_start(struct pt_regs *ctx) { .psc = &otel_span.psc, .sc = &otel_span.sc, .get_parent_span_context_fn = NULL, - .get_parent_span_context_arg = NULL, // Default to new root. + .get_parent_span_context_arg = NULL, // Default to new root. }; start_span(¶ms); @@ -147,7 +148,8 @@ int uprobe_Span_ended(struct pt_regs *ctx) { bpf_map_delete_elem(&active_spans_by_span_ptr, &span_ptr); // Do not output un-sampled span data. - if (!sampled) return 0; + if (!sampled) + return 0; u64 len = (u64)get_argument(ctx, 3); if (len > MAX_SIZE) { @@ -187,7 +189,7 @@ int uprobe_Span_ended(struct pt_regs *ctx) { // Do not send the whole size.buf if it is not needed. u64 size = sizeof(event->kind) + sizeof(event->size) + event->size; // Make the verifier happy, ensure no unbounded memory access. - if (size < sizeof(struct event_t)+1) { + if (size < sizeof(struct event_t) + 1) { return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, size); } bpf_printk("write too large: %d", event->size); diff --git a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf/probe.bpf.c index cf02beb65c..d6df9fb705 100644 --- a/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/go.opentelemetry.io/otel/traceglobal/bpf/probe.bpf.c @@ -31,8 +31,8 @@ struct span_description_t { }; typedef struct otel_status { - u32 code; - struct span_description_t description; + u32 code; + struct span_description_t description; } __attribute__((packed)) otel_status_t; struct span_name_t { @@ -76,8 +76,7 @@ typedef struct go_tracer_with_scope_attributes { go_iface_t scope_attributes; } go_tracer_with_scope_attributes_t; - -typedef void* go_tracer_ptr; +typedef void *go_tracer_ptr; // tracerProvider contains a map of tracers MAP_BUCKET_DEFINITION(go_tracer_id_partial_t, go_tracer_ptr) @@ -85,60 +84,54 @@ MAP_BUCKET_DEFINITION(go_tracer_with_schema_t, go_tracer_ptr) MAP_BUCKET_DEFINITION(go_tracer_with_scope_attributes_t, go_tracer_ptr) struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __type(key, void*); - __type(value, struct otel_span_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, void *); + __type(value, struct otel_span_t); + __uint(max_entries, MAX_CONCURRENT); } active_spans_by_span_ptr SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct span_name_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, struct span_name_t); + __uint(max_entries, MAX_CONCURRENT); } span_name_by_context SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, tracer_id_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, tracer_id_t); + __uint(max_entries, MAX_CONCURRENT); } tracer_id_by_context SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct otel_span_t)); __uint(max_entries, 2); } otel_span_storage_map SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr))); __uint(max_entries, 1); } golang_mapbucket_storage_map SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(tracer_id_t)); __uint(max_entries, 1); } tracer_id_storage_map SEC(".maps"); - -struct -{ - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, tracer_id_t); - __uint(max_entries, MAX_TRACERS); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, tracer_id_t); + __uint(max_entries, MAX_TRACERS); } tracer_ptr_to_id_map SEC(".maps"); - // Injected in init volatile const u64 tracer_delegate_pos; volatile const u64 tracer_name_pos; @@ -151,199 +144,191 @@ volatile const bool tracer_id_contains_scope_attributes; // read_span_name reads the span name from the provided span_name_ptr and stores the result in // span_name.buf. -static __always_inline void read_span_name(struct span_name_t *span_name, const u64 span_name_len, void *span_name_ptr) { - const u64 span_name_size = MAX_SPAN_NAME_LEN < span_name_len ? MAX_SPAN_NAME_LEN : span_name_len; +static __always_inline void +read_span_name(struct span_name_t *span_name, const u64 span_name_len, void *span_name_ptr) { + const u64 span_name_size = + MAX_SPAN_NAME_LEN < span_name_len ? MAX_SPAN_NAME_LEN : span_name_len; bpf_probe_read(span_name->buf, span_name_size, span_name_ptr); } -static __always_inline long fill_partial_tracer_id_from_tracers_map(void *tracers_map, go_tracer_ptr tracer, tracer_id_t *tracer_id) { +static __always_inline long fill_partial_tracer_id_from_tracers_map(void *tracers_map, + go_tracer_ptr tracer, + tracer_id_t *tracer_id) { u64 tracers_count = 0; long res = 0; res = bpf_probe_read(&tracers_count, sizeof(tracers_count), tracers_map); - if (res < 0) - { + if (res < 0) { return -1; } - if (tracers_count == 0) - { + if (tracers_count == 0) { return -1; } unsigned char log_2_bucket_count; res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), tracers_map + 9); - if (res < 0) - { + if (res < 0) { return -1; } u64 bucket_count = 1 << log_2_bucket_count; void *buckets_array; - res = bpf_probe_read(&buckets_array, sizeof(buckets_array), (void*)(tracers_map + buckets_ptr_pos)); - if (res < 0) - { + res = bpf_probe_read( + &buckets_array, sizeof(buckets_array), (void *)(tracers_map + buckets_ptr_pos)); + if (res < 0) { return -1; } u32 map_id = 0; - MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr) *map_bucket = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); - if (!map_bucket) - { + MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr) *map_bucket = + bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); + if (!map_bucket) { return -1; } - for (u64 j = 0; j < MAX_BUCKETS; j++) - { - if (j >= bucket_count) - { + for (u64 j = 0; j < MAX_BUCKETS; j++) { + if (j >= bucket_count) { break; } - res = bpf_probe_read(map_bucket, sizeof(MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr)), buckets_array + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr)))); - if (res < 0) - { + res = bpf_probe_read( + map_bucket, + sizeof(MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr)), + buckets_array + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_id_partial_t, go_tracer_ptr)))); + if (res < 0) { continue; } - for (u64 i = 0; i < 8; i++) - { - if (map_bucket->tophash[i] == 0) - { + for (u64 i = 0; i < 8; i++) { + if (map_bucket->tophash[i] == 0) { continue; } - if (map_bucket->values[i] == NULL) - { + if (map_bucket->values[i] == NULL) { continue; } - if (map_bucket->values[i] != tracer) - { + if (map_bucket->values[i] != tracer) { continue; } - get_go_string_from_user_ptr(&map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); + get_go_string_from_user_ptr( + &map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); return 0; } } return 0; } -static __always_inline long fill_tracer_id_with_schema_from_tracers_map(void *tracers_map, go_tracer_ptr tracer, tracer_id_t *tracer_id) { +static __always_inline long fill_tracer_id_with_schema_from_tracers_map(void *tracers_map, + go_tracer_ptr tracer, + tracer_id_t *tracer_id) { u64 tracers_count = 0; long res = 0; res = bpf_probe_read(&tracers_count, sizeof(tracers_count), tracers_map); - if (res < 0) - { + if (res < 0) { return -1; } - if (tracers_count == 0) - { + if (tracers_count == 0) { return -1; } unsigned char log_2_bucket_count; res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), tracers_map + 9); - if (res < 0) - { + if (res < 0) { return -1; } u64 bucket_count = 1 << log_2_bucket_count; void *buckets_array; - res = bpf_probe_read(&buckets_array, sizeof(buckets_array), (void*)(tracers_map + buckets_ptr_pos)); - if (res < 0) - { + res = bpf_probe_read( + &buckets_array, sizeof(buckets_array), (void *)(tracers_map + buckets_ptr_pos)); + if (res < 0) { return -1; } u32 map_id = 0; - MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr) *map_bucket = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); - if (!map_bucket) - { + MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr) *map_bucket = + bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); + if (!map_bucket) { return -1; } - for (u64 j = 0; j < MAX_BUCKETS; j++) - { - if (j >= bucket_count) - { + for (u64 j = 0; j < MAX_BUCKETS; j++) { + if (j >= bucket_count) { break; } - res = bpf_probe_read(map_bucket, sizeof(MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr)), buckets_array + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr)))); - if (res < 0) - { + res = bpf_probe_read( + map_bucket, + sizeof(MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr)), + buckets_array + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_with_schema_t, go_tracer_ptr)))); + if (res < 0) { continue; } - for (u64 i = 0; i < 8; i++) - { - if (map_bucket->tophash[i] == 0) - { + for (u64 i = 0; i < 8; i++) { + if (map_bucket->tophash[i] == 0) { continue; } - if (map_bucket->values[i] == NULL) - { + if (map_bucket->values[i] == NULL) { continue; } - if (map_bucket->values[i] != tracer) - { + if (map_bucket->values[i] != tracer) { continue; } - get_go_string_from_user_ptr(&map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); - get_go_string_from_user_ptr(&map_bucket->keys[i].schema_url, tracer_id->schema_url, MAX_TRACER_SCHEMA_URL_LEN); + get_go_string_from_user_ptr( + &map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); + get_go_string_from_user_ptr( + &map_bucket->keys[i].schema_url, tracer_id->schema_url, MAX_TRACER_SCHEMA_URL_LEN); return 0; } } return 0; } -static __always_inline long fill_tracer_id_with_scope_attributes_from_tracers_map(void *tracers_map, go_tracer_ptr tracer, tracer_id_t *tracer_id) { +static __always_inline long fill_tracer_id_with_scope_attributes_from_tracers_map( + void *tracers_map, go_tracer_ptr tracer, tracer_id_t *tracer_id) { u64 tracers_count = 0; long res = 0; res = bpf_probe_read(&tracers_count, sizeof(tracers_count), tracers_map); - if (res < 0) - { + if (res < 0) { return -1; } - if (tracers_count == 0) - { + if (tracers_count == 0) { return -1; } unsigned char log_2_bucket_count; res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), tracers_map + 9); - if (res < 0) - { + if (res < 0) { return -1; } u64 bucket_count = 1 << log_2_bucket_count; void *buckets_array; - res = bpf_probe_read(&buckets_array, sizeof(buckets_array), (void*)(tracers_map + buckets_ptr_pos)); - if (res < 0) - { + res = bpf_probe_read( + &buckets_array, sizeof(buckets_array), (void *)(tracers_map + buckets_ptr_pos)); + if (res < 0) { return -1; } u32 map_id = 0; - MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr) *map_bucket = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); - if (!map_bucket) - { + MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr) *map_bucket = + bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); + if (!map_bucket) { return -1; } - for (u64 j = 0; j < MAX_BUCKETS; j++) - { - if (j >= bucket_count) - { + for (u64 j = 0; j < MAX_BUCKETS; j++) { + if (j >= bucket_count) { break; } - res = bpf_probe_read(map_bucket, sizeof(MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr)), buckets_array + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr)))); - if (res < 0) - { + res = bpf_probe_read( + map_bucket, + sizeof(MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr)), + buckets_array + + (j * sizeof(MAP_BUCKET_TYPE(go_tracer_with_scope_attributes_t, go_tracer_ptr)))); + if (res < 0) { continue; } - for (u64 i = 0; i < 8; i++) - { - if (map_bucket->tophash[i] == 0) - { + for (u64 i = 0; i < 8; i++) { + if (map_bucket->tophash[i] == 0) { continue; } - if (map_bucket->values[i] == NULL) - { + if (map_bucket->values[i] == NULL) { continue; } - if (map_bucket->values[i] != tracer) - { + if (map_bucket->values[i] != tracer) { continue; } - get_go_string_from_user_ptr(&map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); - get_go_string_from_user_ptr(&map_bucket->keys[i].schema_url, tracer_id->schema_url, MAX_TRACER_SCHEMA_URL_LEN); + get_go_string_from_user_ptr( + &map_bucket->keys[i].version, tracer_id->version, MAX_TRACER_VERSION_LEN); + get_go_string_from_user_ptr( + &map_bucket->keys[i].schema_url, tracer_id->schema_url, MAX_TRACER_SCHEMA_URL_LEN); return 0; } } @@ -358,19 +343,22 @@ static __always_inline long fill_tracer_id(tracer_id_t *tracer_id, go_tracer_ptr return 0; } - if (!get_go_string_from_user_ptr((void*)(tracer + tracer_name_pos), tracer_id->name, MAX_TRACER_NAME_LEN)) { + if (!get_go_string_from_user_ptr( + (void *)(tracer + tracer_name_pos), tracer_id->name, MAX_TRACER_NAME_LEN)) { return -1; } long res = 0; void *tracer_provider = NULL; - res = bpf_probe_read(&tracer_provider, sizeof(tracer_provider), (void*)(tracer + tracer_provider_pos)); + res = bpf_probe_read( + &tracer_provider, sizeof(tracer_provider), (void *)(tracer + tracer_provider_pos)); if (res < 0) { return res; } void *tracers_map = NULL; - res = bpf_probe_read(&tracers_map, sizeof(tracers_map), (void*)(tracer_provider + tracer_provider_tracers_pos)); + res = bpf_probe_read( + &tracers_map, sizeof(tracers_map), (void *)(tracer_provider + tracer_provider_tracers_pos)); if (res < 0) { return res; } @@ -381,7 +369,8 @@ static __always_inline long fill_tracer_id(tracer_id_t *tracer_id, go_tracer_ptr // version of otel-go is 1.32.0 or higher // we don't collect the scope attributes, but we need to take their presence into account, // when parsing the map bucket - res = fill_tracer_id_with_scope_attributes_from_tracers_map(tracers_map, tracer, tracer_id); + res = fill_tracer_id_with_scope_attributes_from_tracers_map( + tracers_map, tracer, tracer_id); } else { res = fill_tracer_id_with_schema_from_tracers_map(tracers_map, tracer, tracer_id); } @@ -423,7 +412,8 @@ int uprobe_newStart(struct pt_regs *ctx) { // Signal this uprobe should be unloaded. struct control_t ctrl = {1}; - return bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t)); + return bpf_perf_event_output( + ctx, &events, BPF_F_CURRENT_CPU, (void *)(&ctrl), sizeof(struct control_t)); } // This instrumentation attaches uprobe to the following function: @@ -433,7 +423,7 @@ SEC("uprobe/Start") int uprobe_Start(struct pt_regs *ctx) { void *tracer_ptr = get_argument(ctx, 1); void *delegate_ptr = NULL; - bpf_probe_read(&delegate_ptr, sizeof(delegate_ptr), (void*)(tracer_ptr + tracer_delegate_pos)); + bpf_probe_read(&delegate_ptr, sizeof(delegate_ptr), (void *)(tracer_ptr + tracer_delegate_pos)); if (delegate_ptr != NULL) { // Delegate is set, so we should not instrument this call return 0; @@ -476,7 +466,7 @@ int uprobe_Start_Returns(struct pt_regs *ctx) { get_Go_context(ctx, 1, 0, true, &go_context); void *key = (void *)GOROUTINE(ctx); - struct span_name_t *span_name = bpf_map_lookup_elem(&span_name_by_context, &key); + struct span_name_t *span_name = bpf_map_lookup_elem(&span_name_by_context, &key); if (span_name == NULL) { return 0; } @@ -533,7 +523,8 @@ int uprobe_Start_Returns(struct pt_regs *ctx) { SEC("uprobe/SetAttributes") int uprobe_SetAttributes(struct pt_regs *ctx) { void *non_recording_span_ptr = get_argument(ctx, 1); - struct otel_span_t *span = bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); + struct otel_span_t *span = + bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); if (span == NULL) { return 0; } @@ -553,7 +544,8 @@ int uprobe_SetAttributes(struct pt_regs *ctx) { SEC("uprobe/SetName") int uprobe_SetName(struct pt_regs *ctx) { void *non_recording_span_ptr = get_argument(ctx, 1); - struct otel_span_t *span = bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); + struct otel_span_t *span = + bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); if (span == NULL) { return 0; } @@ -574,7 +566,7 @@ int uprobe_SetName(struct pt_regs *ctx) { read_span_name(&span_name, span_name_len, span_name_ptr); span->span_name = span_name; bpf_map_update_elem(&active_spans_by_span_ptr, &non_recording_span_ptr, span, 0); - + return 0; } @@ -583,7 +575,8 @@ int uprobe_SetName(struct pt_regs *ctx) { SEC("uprobe/SetStatus") int uprobe_SetStatus(struct pt_regs *ctx) { void *non_recording_span_ptr = get_argument(ctx, 1); - struct otel_span_t *span = bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); + struct otel_span_t *span = + bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); if (span == NULL) { return 0; } @@ -599,7 +592,8 @@ int uprobe_SetStatus(struct pt_regs *ctx) { // Getting span description u64 description_len = (u64)get_argument(ctx, 4); - u64 description_size = MAX_STATUS_DESCRIPTION_LEN < description_len ? MAX_STATUS_DESCRIPTION_LEN : description_len; + u64 description_size = + MAX_STATUS_DESCRIPTION_LEN < description_len ? MAX_STATUS_DESCRIPTION_LEN : description_len; bpf_probe_read(description.buf, description_size, description_ptr); otel_status_t status = {0}; @@ -617,7 +611,8 @@ int uprobe_SetStatus(struct pt_regs *ctx) { SEC("uprobe/End") int uprobe_End(struct pt_regs *ctx) { void *non_recording_span_ptr = get_argument(ctx, 1); - struct otel_span_t *span = bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); + struct otel_span_t *span = + bpf_map_lookup_elem(&active_spans_by_span_ptr, &non_recording_span_ptr); if (span == NULL) { return 0; } diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf/probe.bpf.c index f47b86b01e..31c8887065 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf/probe.bpf.c @@ -14,8 +14,7 @@ char __license[] SEC("license") = "Dual MIT/GPL"; #define MAX_CONCURRENT 50 #define MAX_ERROR_LEN 128 -struct grpc_request_t -{ +struct grpc_request_t { BASE_SPAN_PROPERTIES char err_msg[MAX_ERROR_LEN]; char method[MAX_SIZE]; @@ -23,23 +22,20 @@ struct grpc_request_t u32 status_code; }; -struct hpack_header_field -{ +struct hpack_header_field { struct go_string name; struct go_string value; bool sensitive; }; -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, void *); __type(value, struct grpc_request_t); __uint(max_entries, MAX_CONCURRENT); } grpc_events SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, u32); __type(value, struct span_context); @@ -61,8 +57,7 @@ volatile const bool write_status_supported; // This instrumentation attaches uprobe to the following function: // func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error SEC("uprobe/ClientConn_Invoke") -int uprobe_ClientConn_Invoke(struct pt_regs *ctx) -{ +int uprobe_ClientConn_Invoke(struct pt_regs *ctx) { // positions u64 clientconn_pos = 1; u64 method_ptr_pos = 4; @@ -74,8 +69,7 @@ int uprobe_ClientConn_Invoke(struct pt_regs *ctx) // Get key void *key = (void *)GOROUTINE(ctx); void *grpcReq_ptr = bpf_map_lookup_elem(&grpc_events, &key); - if (grpcReq_ptr != NULL) - { + if (grpcReq_ptr != NULL) { bpf_printk("uprobe/ClientConn_Invoke already tracked with the current context"); return 0; } @@ -92,8 +86,9 @@ int uprobe_ClientConn_Invoke(struct pt_regs *ctx) // Read ClientConn.Target void *clientconn_ptr = get_argument(ctx, clientconn_pos); - if (!get_go_string_from_user_ptr((void*)(clientconn_ptr + clientconn_target_ptr_pos), grpcReq.target, sizeof(grpcReq.target))) - { + if (!get_go_string_from_user_ptr((void *)(clientconn_ptr + clientconn_target_ptr_pos), + grpcReq.target, + sizeof(grpcReq.target))) { bpf_printk("target write failed, aborting ebpf probe"); return 0; } @@ -125,7 +120,7 @@ int uprobe_ClientConn_Invoke_Returns(struct pt_regs *ctx) { return 0; } - if(!write_status_supported) { + if (!write_status_supported) { goto done; } // Getting the returned response (error) @@ -146,19 +141,21 @@ int uprobe_ClientConn_Invoke_Returns(struct pt_regs *ctx) { // Details []*anypb.Any // } void *resp_ptr = get_argument(ctx, 2); - if(resp_ptr == 0) { + if (resp_ptr == 0) { // err == nil goto done; } void *status_ptr = 0; // get `s` (Status pointer field) from Error struct - bpf_probe_read_user(&status_ptr, sizeof(status_ptr), (void *)(resp_ptr+error_status_pos)); + bpf_probe_read_user(&status_ptr, sizeof(status_ptr), (void *)(resp_ptr + error_status_pos)); // get `s` field from Status object pointer void *s_ptr = 0; bpf_probe_read_user(&s_ptr, sizeof(s_ptr), (void *)(status_ptr + status_s_pos)); // Get status code from Status.s pointer - bpf_probe_read_user(&grpc_span->status_code, sizeof(grpc_span->status_code), (void *)(s_ptr + status_code_pos)); - get_go_string_from_user_ptr((void *)(s_ptr + status_message_pos), grpc_span->err_msg, sizeof(grpc_span->err_msg)); + bpf_probe_read_user( + &grpc_span->status_code, sizeof(grpc_span->status_code), (void *)(s_ptr + status_code_pos)); + get_go_string_from_user_ptr( + (void *)(s_ptr + status_message_pos), grpc_span->err_msg, sizeof(grpc_span->err_msg)); done: grpc_span->end_time = bpf_ktime_get_ns(); @@ -170,14 +167,13 @@ int uprobe_ClientConn_Invoke_Returns(struct pt_regs *ctx) { // func (l *loopyWriter) headerHandler(h *headerFrame) error SEC("uprobe/loopyWriter_headerHandler") -int uprobe_LoopyWriter_HeaderHandler(struct pt_regs *ctx) -{ +int uprobe_LoopyWriter_HeaderHandler(struct pt_regs *ctx) { void *headerFrame_ptr = get_argument(ctx, 2); u32 stream_id = 0; - bpf_probe_read(&stream_id, sizeof(stream_id), (void *)(headerFrame_ptr + (headerFrame_streamid_pos))); + bpf_probe_read( + &stream_id, sizeof(stream_id), (void *)(headerFrame_ptr + (headerFrame_streamid_pos))); void *sc_ptr = bpf_map_lookup_elem(&streamid_to_span_contexts, &stream_id); - if (sc_ptr == NULL) - { + if (sc_ptr == NULL) { return 0; } @@ -211,8 +207,7 @@ int uprobe_LoopyWriter_HeaderHandler(struct pt_regs *ctx) SEC("uprobe/http2Client_NewStream") // func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error) -int uprobe_http2Client_NewStream(struct pt_regs *ctx) -{ +int uprobe_http2Client_NewStream(struct pt_regs *ctx) { struct go_iface go_context = {0}; get_Go_context(ctx, 2, 0, true, &go_context); void *httpclient_ptr = get_argument(ctx, 1); diff --git a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c index 16a59cab78..7ff2a42178 100644 --- a/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c @@ -16,8 +16,7 @@ char __license[] SEC("license") = "Dual MIT/GPL"; #define MAX_HEADERS 20 #define MAX_HEADER_STRING 50 -struct grpc_request_t -{ +struct grpc_request_t { BASE_SPAN_PROPERTIES char method[MAX_SIZE]; u32 status_code; @@ -25,32 +24,28 @@ struct grpc_request_t u8 has_status; }; -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, void *); __type(value, struct grpc_request_t); __uint(max_entries, MAX_CONCURRENT); } grpc_events SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, u32); __type(value, struct grpc_request_t); __uint(max_entries, MAX_CONCURRENT); } streamid_to_grpc_events SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct grpc_request_t)); __uint(max_entries, 1); } grpc_storage_map SEC(".maps"); -struct hpack_header_field -{ +struct hpack_header_field { struct go_string name; struct go_string value; bool sensitive; @@ -71,7 +66,8 @@ volatile const u64 peer_local_addr_pos; volatile const bool server_addr_supported; -static __always_inline long dummy_extract_span_context_from_headers(void *stream_id, struct span_context *parent_span_context) { +static __always_inline long +dummy_extract_span_context_from_headers(void *stream_id, struct span_context *parent_span_context) { return 0; } @@ -83,7 +79,8 @@ static __always_inline long dummy_extract_span_context_from_headers(void *stream // - go_context: the parsed Go context.Context // // Returns 0 on success, otherwise a negative error value in case of failure. -static __always_inline int handleStream(struct pt_regs *ctx, void *stream_ptr, struct go_iface *go_context) { +static __always_inline int +handleStream(struct pt_regs *ctx, void *stream_ptr, struct go_iface *go_context) { if (go_context == NULL) { bpf_printk("grpc:server:handleStream: NULL go_context"); return -1; @@ -103,7 +100,8 @@ static __always_inline int handleStream(struct pt_regs *ctx, void *stream_ptr, s // Get parent context if exists u32 stream_id = 0; - long rc = bpf_probe_read_user(&stream_id, sizeof(stream_id), (void *)(stream_ptr + stream_id_pos)); + long rc = + bpf_probe_read_user(&stream_id, sizeof(stream_id), (void *)(stream_ptr + stream_id_pos)); if (rc != 0) { bpf_printk("grpc:server:handleStream: failed to read stream ID"); return -2; @@ -135,7 +133,8 @@ static __always_inline int handleStream(struct pt_regs *ctx, void *stream_ptr, s // Set attributes void *method_ptr = stream_ptr + stream_method_ptr_pos; - bool parsed_method = get_go_string_from_user_ptr(method_ptr, grpcReq->method, sizeof(grpcReq->method)); + bool parsed_method = + get_go_string_from_user_ptr(method_ptr, grpcReq->method, sizeof(grpcReq->method)); if (!parsed_method) { bpf_printk("grpc:server:handleStream: failed to read gRPC method from stream"); bpf_map_delete_elem(&streamid_to_grpc_events, &stream_id); @@ -147,7 +146,8 @@ static __always_inline int handleStream(struct pt_regs *ctx, void *stream_ptr, s if (http2server != NULL) { void *local_addr_ptr = 0; void *local_addr_pos = http2server + http2server_peer_pos + peer_local_addr_pos; - bpf_probe_read_user(&local_addr_ptr, sizeof(local_addr_ptr), get_go_interface_instance(local_addr_pos)); + bpf_probe_read_user( + &local_addr_ptr, sizeof(local_addr_ptr), get_go_interface_instance(local_addr_pos)); get_tcp_net_addr_from_tcp_addr(ctx, &grpcReq->local_addr, (void *)(local_addr_ptr)); } else { bpf_printk("grpc:server:handleStream: failed to get http2server arg"); @@ -194,7 +194,8 @@ static __always_inline int writeStatus(struct pt_regs *ctx, void *status_ptr) { } // Get status code from Status.s pointer - rc = bpf_probe_read_user(&req_ptr->status_code, sizeof(req_ptr->status_code), (void *)(s_ptr + status_code_pos)); + rc = bpf_probe_read_user( + &req_ptr->status_code, sizeof(req_ptr->status_code), (void *)(s_ptr + status_code_pos)); if (rc != 0) { bpf_printk("grpc:server:handleStream: failed to read status code"); return -4; @@ -236,20 +237,24 @@ int uprobe_server_handleStream2(struct pt_regs *ctx) { } void *stream_ptr; - long rc = bpf_probe_read_user(&stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); + long rc = bpf_probe_read_user( + &stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); if (rc != 0) { bpf_printk("grpc:server:uprobe/server_handleStream2: failed to read stream_ptr"); return -2; } struct go_iface go_context = {0}; - rc = bpf_probe_read_user(&go_context.type, sizeof(go_context.type), (void *)(stream_ptr + stream_ctx_pos)); + rc = bpf_probe_read_user( + &go_context.type, sizeof(go_context.type), (void *)(stream_ptr + stream_ctx_pos)); if (rc != 0) { bpf_printk("grpc:server:uprobe/server_handleStream2: failed to read context type"); return -3; } - rc = bpf_probe_read_user(&go_context.data, sizeof(go_context.data), get_go_interface_instance(stream_ptr + stream_ctx_pos)); + rc = bpf_probe_read_user(&go_context.data, + sizeof(go_context.data), + get_go_interface_instance(stream_ptr + stream_ctx_pos)); if (rc != 0) { bpf_printk("grpc:server:uprobe/server_handleStream2: failed to read context data"); return -4; @@ -275,7 +280,8 @@ int uprobe_server_handleStream2_Returns(struct pt_regs *ctx) { } void *stream_ptr; - long rc = bpf_probe_read_user(&stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); + long rc = bpf_probe_read_user( + &stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); if (rc != 0) { bpf_printk("grpc:server:uprobe/server_handleStream2Return: failed to read stream_ptr"); return -2; @@ -299,28 +305,24 @@ int uprobe_server_handleStream2_Returns(struct pt_regs *ctx) { // for version 1.60 and above: // func (t *http2Server) operateHeaders(ctx context.Context, frame *http2.MetaHeadersFrame, handle func(*Stream)) error SEC("uprobe/http2Server_operateHeader") -int uprobe_http2Server_operateHeader(struct pt_regs *ctx) -{ +int uprobe_http2Server_operateHeader(struct pt_regs *ctx) { void *arg4 = get_argument(ctx, 4); void *arg2 = get_argument(ctx, 2); void *frame_ptr = is_new_frame_pos ? arg4 : arg2; struct go_slice header_fields = {}; bpf_probe_read(&header_fields, sizeof(header_fields), (void *)(frame_ptr + frame_fields_pos)); char key[W3C_KEY_LENGTH] = "traceparent"; - for (s32 i = 0; i < MAX_HEADERS; i++) - { - if (i >= header_fields.len) - { + for (s32 i = 0; i < MAX_HEADERS; i++) { + if (i >= header_fields.len) { break; } struct hpack_header_field hf = {}; - long res = bpf_probe_read(&hf, sizeof(hf), (void *)(header_fields.array + (i * sizeof(hf)))); - if (hf.name.len == W3C_KEY_LENGTH && hf.value.len == W3C_VAL_LENGTH) - { + long res = + bpf_probe_read(&hf, sizeof(hf), (void *)(header_fields.array + (i * sizeof(hf)))); + if (hf.name.len == W3C_KEY_LENGTH && hf.value.len == W3C_VAL_LENGTH) { char current_key[W3C_KEY_LENGTH]; bpf_probe_read(current_key, sizeof(current_key), hf.name.str); - if (bpf_memcmp(key, current_key, sizeof(key))) - { + if (bpf_memcmp(key, current_key, sizeof(key))) { char val[W3C_VAL_LENGTH]; bpf_probe_read(val, W3C_VAL_LENGTH, hf.value.str); @@ -328,7 +330,8 @@ int uprobe_http2Server_operateHeader(struct pt_regs *ctx) void *headers_frame = NULL; bpf_probe_read(&headers_frame, sizeof(headers_frame), frame_ptr); u32 stream_id = 0; - bpf_probe_read(&stream_id, sizeof(stream_id), (void *)(headers_frame + frame_stream_id_pod)); + bpf_probe_read( + &stream_id, sizeof(stream_id), (void *)(headers_frame + frame_stream_id_pod)); struct grpc_request_t grpcReq = {}; w3c_string_to_span_context(val, &grpcReq.psc); bpf_map_update_elem(&streamid_to_grpc_events, &stream_id, &grpcReq, 0); @@ -363,7 +366,8 @@ int uprobe_http2Server_WriteStatus2(struct pt_regs *ctx) { } void *stream_ptr; - long rc = bpf_probe_read_user(&stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); + long rc = bpf_probe_read_user( + &stream_ptr, sizeof(stream_ptr), (void *)(server_stream_ptr + server_stream_stream_pos)); if (rc != 0) { bpf_printk("grpc:server:uprobe/http2Server_WriteStatus2: failed to read stream_ptr"); return -2; diff --git a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c index b003cb6002..f68bd26022 100644 --- a/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/client/bpf/probe.bpf.c @@ -43,14 +43,13 @@ struct http_request_t { }; struct { - __uint(type, BPF_MAP_TYPE_HASH); - __type(key, void*); - __type(value, struct http_request_t); - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, void *); + __type(value, struct http_request_t); + __uint(max_entries, MAX_CONCURRENT); } http_events SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct http_request_t)); @@ -58,10 +57,10 @@ struct } http_client_uprobe_storage_map SEC(".maps"); struct { - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __type(key, void*); // the headers ptr - __type(value, void*); // request key, goroutine or context ptr - __uint(max_entries, MAX_CONCURRENT); + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, void *); // the headers ptr + __type(value, void *); // request key, goroutine or context ptr + __uint(max_entries, MAX_CONCURRENT); } http_headers SEC(".maps"); // Injected in init @@ -99,16 +98,14 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) { void *key = (void *)GOROUTINE(ctx); void *httpReq_ptr = bpf_map_lookup_elem(&http_events, &key); - if (httpReq_ptr != NULL) - { + if (httpReq_ptr != NULL) { bpf_printk("uprobe/Transport_RoundTrip already tracked with the current context"); return 0; } u32 map_id = 0; struct http_request_t *httpReq = bpf_map_lookup_elem(&http_client_uprobe_storage_map, &map_id); - if (httpReq == NULL) - { + if (httpReq == NULL) { bpf_printk("uprobe/Transport_roundTrip: httpReq is NULL"); return 0; } @@ -126,77 +123,92 @@ int uprobe_Transport_roundTrip(struct pt_regs *ctx) { }; start_span(&start_span_params); - if (!get_go_string_from_user_ptr((void *)(req_ptr+method_ptr_pos), httpReq->method, sizeof(httpReq->method))) { + if (!get_go_string_from_user_ptr( + (void *)(req_ptr + method_ptr_pos), httpReq->method, sizeof(httpReq->method))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get method from request"); return 0; } // get path from Request.URL void *url_ptr = 0; - bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *)(req_ptr+url_ptr_pos)); - if (!get_go_string_from_user_ptr((void *)(url_ptr+path_ptr_pos), httpReq->path, sizeof(httpReq->path))) { + bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *)(req_ptr + url_ptr_pos)); + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + path_ptr_pos), httpReq->path, sizeof(httpReq->path))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get path from Request.URL"); } // get scheme from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+scheme_pos), httpReq->scheme, sizeof(httpReq->scheme))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + scheme_pos), httpReq->scheme, sizeof(httpReq->scheme))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get scheme from Request.URL"); } // get opaque from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+opaque_pos), httpReq->opaque, sizeof(httpReq->opaque))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + opaque_pos), httpReq->opaque, sizeof(httpReq->opaque))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get opaque from Request.URL"); } // get RawPath from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+raw_path_pos), httpReq->raw_path, sizeof(httpReq->raw_path))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + raw_path_pos), httpReq->raw_path, sizeof(httpReq->raw_path))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get RawPath from Request.URL"); } // get username from Request.URL.User void *user_ptr = 0; - bpf_probe_read(&user_ptr, sizeof(user_ptr), (void *)(url_ptr+user_ptr_pos)); - if (!get_go_string_from_user_ptr((void *)(user_ptr+username_pos), httpReq->username, sizeof(httpReq->username))) { + bpf_probe_read(&user_ptr, sizeof(user_ptr), (void *)(url_ptr + user_ptr_pos)); + if (!get_go_string_from_user_ptr( + (void *)(user_ptr + username_pos), httpReq->username, sizeof(httpReq->username))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get RawQuery from Request.URL"); } // get RawQuery from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+raw_query_pos), httpReq->raw_query, sizeof(httpReq->raw_query))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + raw_query_pos), httpReq->raw_query, sizeof(httpReq->raw_query))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get RawQuery from Request.URL"); } // get Fragment from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+fragment_pos), httpReq->fragment, sizeof(httpReq->fragment))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + fragment_pos), httpReq->fragment, sizeof(httpReq->fragment))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get Fragment from Request.URL"); } // get RawFragment from Request.URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+raw_fragment_pos), httpReq->raw_fragment, sizeof(httpReq->raw_fragment))) { + if (!get_go_string_from_user_ptr((void *)(url_ptr + raw_fragment_pos), + httpReq->raw_fragment, + sizeof(httpReq->raw_fragment))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get RawFragment from Request.URL"); } // get ForceQuery from Request.URL - bpf_probe_read(&httpReq->force_query, sizeof(httpReq->force_query), (void *)(url_ptr+force_query_pos)); + bpf_probe_read( + &httpReq->force_query, sizeof(httpReq->force_query), (void *)(url_ptr + force_query_pos)); // get OmitHost from Request.URL - bpf_probe_read(&httpReq->omit_host, sizeof(httpReq->omit_host), (void *)(url_ptr+omit_host_pos)); + bpf_probe_read( + &httpReq->omit_host, sizeof(httpReq->omit_host), (void *)(url_ptr + omit_host_pos)); // get host from Request - if (!get_go_string_from_user_ptr((void *)(req_ptr+request_host_pos), httpReq->host, sizeof(httpReq->host))) { + if (!get_go_string_from_user_ptr( + (void *)(req_ptr + request_host_pos), httpReq->host, sizeof(httpReq->host))) { // If host is not present in Request, get it from URL - if (!get_go_string_from_user_ptr((void *)(url_ptr+url_host_pos), httpReq->host, sizeof(httpReq->host))) { + if (!get_go_string_from_user_ptr( + (void *)(url_ptr + url_host_pos), httpReq->host, sizeof(httpReq->host))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get host from Request and URL"); } } // get proto from Request - if (!get_go_string_from_user_ptr((void *)(req_ptr+request_proto_pos), httpReq->proto, sizeof(httpReq->proto))) { + if (!get_go_string_from_user_ptr( + (void *)(req_ptr + request_proto_pos), httpReq->proto, sizeof(httpReq->proto))) { bpf_printk("uprobe_Transport_roundTrip: Failed to get proto from Request"); } // get headers from Request void *headers_ptr = 0; - bpf_probe_read(&headers_ptr, sizeof(headers_ptr), (void *)(req_ptr+headers_ptr_pos)); + bpf_probe_read(&headers_ptr, sizeof(headers_ptr), (void *)(req_ptr + headers_ptr_pos)); if (headers_ptr) { bpf_map_update_elem(&http_headers, &headers_ptr, &key, 0); } @@ -222,7 +234,9 @@ int uprobe_Transport_roundTrip_Returns(struct pt_regs *ctx) { // Getting the returned response void *resp_ptr = get_argument(ctx, 1); // Get status code from response - bpf_probe_read(&http_req_span->status_code, sizeof(http_req_span->status_code), (void *)(resp_ptr + status_code_pos)); + bpf_probe_read(&http_req_span->status_code, + sizeof(http_req_span->status_code), + (void *)(resp_ptr + status_code_pos)); http_req_span->end_time = end_time; @@ -253,25 +267,32 @@ int uprobe_writeSubset(struct pt_regs *ctx) { span_context_to_w3c_string(&http_req_span->sc, tp); void *buf_ptr = 0; - bpf_probe_read(&buf_ptr, sizeof(buf_ptr), (void *)(io_writer_ptr + io_writer_buf_ptr_pos)); // grab buf ptr + bpf_probe_read(&buf_ptr, + sizeof(buf_ptr), + (void *)(io_writer_ptr + io_writer_buf_ptr_pos)); // grab buf ptr if (!buf_ptr) { bpf_printk("uprobe_writeSubset: Failed to get buf from io writer"); goto done; } s64 size = 0; - if (bpf_probe_read(&size, sizeof(s64), (void *)(io_writer_ptr + io_writer_buf_ptr_pos + offsetof(struct go_slice, cap)))) { // grab capacity + if (bpf_probe_read(&size, + sizeof(s64), + (void *)(io_writer_ptr + io_writer_buf_ptr_pos + + offsetof(struct go_slice, cap)))) { // grab capacity bpf_printk("uprobe_writeSubset: Failed to get size from io writer"); goto done; } s64 len = 0; - if (bpf_probe_read(&len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos))) { // grab len + if (bpf_probe_read( + &len, sizeof(s64), (void *)(io_writer_ptr + io_writer_n_pos))) { // grab len bpf_printk("uprobe_writeSubset: Failed to get len from io writer"); goto done; } - if (len < (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") + if (len < + (size - W3C_VAL_LENGTH - W3C_KEY_LENGTH - 4)) { // 4 = strlen(":_") + strlen("\r\n") char tp_str[W3C_KEY_LENGTH + 2 + W3C_VAL_LENGTH + 2] = "Traceparent: "; char end[2] = "\r\n"; __builtin_memcpy(&tp_str[W3C_KEY_LENGTH + 2], tp, sizeof(tp)); @@ -281,7 +302,8 @@ int uprobe_writeSubset(struct pt_regs *ctx) { goto done; } len += W3C_KEY_LENGTH + 2 + W3C_VAL_LENGTH + 2; - if (bpf_probe_write_user((void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len))) { + if (bpf_probe_write_user( + (void *)(io_writer_ptr + io_writer_n_pos), &len, sizeof(len))) { bpf_printk("uprobe_writeSubset: Failed to change io writer n"); goto done; } diff --git a/internal/pkg/instrumentation/bpf/net/http/server/bpf/probe.bpf.c b/internal/pkg/instrumentation/bpf/net/http/server/bpf/probe.bpf.c index ef27142fe5..ed530cea6a 100644 --- a/internal/pkg/instrumentation/bpf/net/http/server/bpf/probe.bpf.c +++ b/internal/pkg/instrumentation/bpf/net/http/server/bpf/probe.bpf.c @@ -19,8 +19,7 @@ char __license[] SEC("license") = "Dual MIT/GPL"; #define HOST_MAX_LEN 256 #define PROTO_MAX_LEN 8 -struct http_server_span_t -{ +struct http_server_span_t { BASE_SPAN_PROPERTIES u64 status_code; char method[METHOD_MAX_LEN]; @@ -31,8 +30,7 @@ struct http_server_span_t char proto[PROTO_MAX_LEN]; }; -struct uprobe_data_t -{ +struct uprobe_data_t { struct http_server_span_t span; // bpf2go doesn't support pointers fields // saving the response pointer in the entry probe @@ -42,32 +40,28 @@ struct uprobe_data_t MAP_BUCKET_DEFINITION(go_string_t, go_slice_t) -struct -{ +struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, void *); __type(value, struct uprobe_data_t); __uint(max_entries, MAX_CONCURRENT); } http_server_uprobes SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); __type(key, void *); __type(value, struct span_context); __uint(max_entries, MAX_CONCURRENT); } http_server_context_headers SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(MAP_BUCKET_TYPE(go_string_t, go_slice_t))); __uint(max_entries, 1); } golang_mapbucket_storage_map SEC(".maps"); -struct -{ +struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(struct uprobe_data_t)); @@ -101,87 +95,82 @@ volatile const bool swiss_maps_used; // Extracts the span context from the request headers by looking for the 'traceparent' header. // Fills the parent_span_context with the extracted span context. // Returns 0 on success, negative value on error. -static __always_inline long extract_context_from_req_headers_go_map(void *headers_ptr_ptr, struct span_context *parent_span_context) -{ +static __always_inline long +extract_context_from_req_headers_go_map(void *headers_ptr_ptr, + struct span_context *parent_span_context) { void *headers_ptr; long res; res = bpf_probe_read(&headers_ptr, sizeof(headers_ptr), headers_ptr_ptr); - if (res < 0) - { + if (res < 0) { return res; } u64 headers_count = 0; res = bpf_probe_read(&headers_count, sizeof(headers_count), headers_ptr); - if (res < 0) - { + if (res < 0) { return res; } - if (headers_count == 0) - { + if (headers_count == 0) { return -1; } unsigned char log_2_bucket_count; res = bpf_probe_read(&log_2_bucket_count, sizeof(log_2_bucket_count), headers_ptr + 9); - if (res < 0) - { + if (res < 0) { return -1; } u64 bucket_count = 1 << log_2_bucket_count; void *header_buckets; - res = bpf_probe_read(&header_buckets, sizeof(header_buckets), (void*)(headers_ptr + buckets_ptr_pos)); - if (res < 0) - { + res = bpf_probe_read( + &header_buckets, sizeof(header_buckets), (void *)(headers_ptr + buckets_ptr_pos)); + if (res < 0) { return -1; } u32 map_id = 0; - MAP_BUCKET_TYPE(go_string_t, go_slice_t) *map_value = bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); - if (!map_value) - { + MAP_BUCKET_TYPE(go_string_t, go_slice_t) *map_value = + bpf_map_lookup_elem(&golang_mapbucket_storage_map, &map_id); + if (!map_value) { return -1; } - for (u64 j = 0; j < MAX_BUCKETS; j++) - { - if (j >= bucket_count) - { + for (u64 j = 0; j < MAX_BUCKETS; j++) { + if (j >= bucket_count) { break; } - res = bpf_probe_read(map_value, sizeof(MAP_BUCKET_TYPE(go_string_t, go_slice_t)), header_buckets + (j * sizeof(MAP_BUCKET_TYPE(go_string_t, go_slice_t)))); - if (res < 0) - { + res = + bpf_probe_read(map_value, + sizeof(MAP_BUCKET_TYPE(go_string_t, go_slice_t)), + header_buckets + (j * sizeof(MAP_BUCKET_TYPE(go_string_t, go_slice_t)))); + if (res < 0) { continue; } - for (u64 i = 0; i < 8; i++) - { - if (map_value->tophash[i] == 0) - { + for (u64 i = 0; i < 8; i++) { + if (map_value->tophash[i] == 0) { continue; } - if (map_value->keys[i].len != W3C_KEY_LENGTH) - { + if (map_value->keys[i].len != W3C_KEY_LENGTH) { continue; } char current_header_key[W3C_KEY_LENGTH]; bpf_probe_read(current_header_key, sizeof(current_header_key), map_value->keys[i].str); - if (!bpf_memcmp(current_header_key, "traceparent", W3C_KEY_LENGTH) && !bpf_memcmp(current_header_key, "Traceparent", W3C_KEY_LENGTH)) - { + if (!bpf_memcmp(current_header_key, "traceparent", W3C_KEY_LENGTH) && + !bpf_memcmp(current_header_key, "Traceparent", W3C_KEY_LENGTH)) { continue; } void *traceparent_header_value_ptr = map_value->values[i].array; struct go_string traceparent_header_value_go_str; - res = bpf_probe_read(&traceparent_header_value_go_str, sizeof(traceparent_header_value_go_str), traceparent_header_value_ptr); - if (res < 0) - { + res = bpf_probe_read(&traceparent_header_value_go_str, + sizeof(traceparent_header_value_go_str), + traceparent_header_value_ptr); + if (res < 0) { return -1; } - if (traceparent_header_value_go_str.len != W3C_VAL_LENGTH) - { + if (traceparent_header_value_go_str.len != W3C_VAL_LENGTH) { continue; } char traceparent_header_value[W3C_VAL_LENGTH]; - res = bpf_probe_read(&traceparent_header_value, sizeof(traceparent_header_value), traceparent_header_value_go_str.str); - if (res < 0) - { + res = bpf_probe_read(&traceparent_header_value, + sizeof(traceparent_header_value), + traceparent_header_value_go_str.str); + if (res < 0) { return res; } w3c_string_to_span_context(traceparent_header_value, parent_span_context); @@ -191,8 +180,10 @@ static __always_inline long extract_context_from_req_headers_go_map(void *header return -1; } -static __always_inline long extract_context_from_req_headers_pre_parsed(void *key, struct span_context *parent_span_context) { - struct span_context *parsed_header_context = bpf_map_lookup_elem(&http_server_context_headers, &key); +static __always_inline long +extract_context_from_req_headers_pre_parsed(void *key, struct span_context *parent_span_context) { + struct span_context *parsed_header_context = + bpf_map_lookup_elem(&http_server_context_headers, &key); if (!parsed_header_context) { return -1; } @@ -201,14 +192,16 @@ static __always_inline long extract_context_from_req_headers_pre_parsed(void *ke return 0; } -static __always_inline long extract_context_from_req_headers(void *key, struct span_context *parent_span_context) { +static __always_inline long +extract_context_from_req_headers(void *key, struct span_context *parent_span_context) { if (swiss_maps_used) { return extract_context_from_req_headers_pre_parsed(key, parent_span_context); } return extract_context_from_req_headers_go_map(key, parent_span_context); } -static __always_inline void read_go_string(void *base, int offset, char *output, int maxLen, const char *errorMsg) { +static __always_inline void +read_go_string(void *base, int offset, char *output, int maxLen, const char *errorMsg) { void *ptr = (void *)(base + offset); if (!get_go_string_from_user_ptr(ptr, output, maxLen)) { bpf_printk("Failed to get %s", errorMsg); @@ -218,22 +211,20 @@ static __always_inline void read_go_string(void *base, int offset, char *output, // This instrumentation attaches uprobe to the following function: // func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) SEC("uprobe/serverHandler_ServeHTTP") -int uprobe_serverHandler_ServeHTTP(struct pt_regs *ctx) -{ +int uprobe_serverHandler_ServeHTTP(struct pt_regs *ctx) { struct go_iface go_context = {0}; get_Go_context(ctx, 4, ctx_ptr_pos, false, &go_context); void *key = (void *)GOROUTINE(ctx); void *httpReq_ptr = bpf_map_lookup_elem(&http_server_uprobes, &key); - if (httpReq_ptr != NULL) - { + if (httpReq_ptr != NULL) { bpf_printk("uprobe/HandlerFunc_ServeHTTP already tracked with the current request"); return 0; } u32 map_id = 0; - struct uprobe_data_t *uprobe_data = bpf_map_lookup_elem(&http_server_uprobe_storage_map, &map_id); - if (uprobe_data == NULL) - { + struct uprobe_data_t *uprobe_data = + bpf_map_lookup_elem(&http_server_uprobe_storage_map, &map_id); + if (uprobe_data == NULL) { bpf_printk("uprobe/HandlerFunc_ServeHTTP: http_server_span is NULL"); return 0; } @@ -263,7 +254,7 @@ int uprobe_serverHandler_ServeHTTP(struct pt_regs *ctx) if (swiss_maps_used) { start_span_params.get_parent_span_context_arg = key; } else { - start_span_params.get_parent_span_context_arg = (void*)(req_ptr + headers_ptr_pos); + start_span_params.get_parent_span_context_arg = (void *)(req_ptr + headers_ptr_pos); } start_span(&start_span_params); @@ -298,25 +289,55 @@ int uprobe_serverHandler_ServeHTTP_Returns(struct pt_regs *ctx) { void *url_ptr = 0; bpf_probe_read(&url_ptr, sizeof(url_ptr), (void *)(req_ptr + url_ptr_pos)); // Collect fields from response - read_go_string(req_ptr, method_ptr_pos, http_server_span->method, sizeof(http_server_span->method), "method from request"); + read_go_string(req_ptr, + method_ptr_pos, + http_server_span->method, + sizeof(http_server_span->method), + "method from request"); if (pattern_path_supported) { if (pattern_path_public_supported) { - read_go_string(req_ptr, req_pattern_pos, http_server_span->path_pattern, sizeof(http_server_span->path_pattern), "pattern from Request"); + read_go_string(req_ptr, + req_pattern_pos, + http_server_span->path_pattern, + sizeof(http_server_span->path_pattern), + "pattern from Request"); } else { void *pat_ptr = NULL; bpf_probe_read(&pat_ptr, sizeof(pat_ptr), (void *)(req_ptr + req_pat_pos)); if (pat_ptr != NULL) { - read_go_string(pat_ptr, pat_str_pos, http_server_span->path_pattern, sizeof(http_server_span->path), "patterned path from Request"); + read_go_string(pat_ptr, + pat_str_pos, + http_server_span->path_pattern, + sizeof(http_server_span->path), + "patterned path from Request"); } } } - read_go_string(url_ptr, path_ptr_pos, http_server_span->path, sizeof(http_server_span->path), "path from Request.URL"); - read_go_string(req_ptr, remote_addr_pos, http_server_span->remote_addr, sizeof(http_server_span->remote_addr), "remote addr from Request.RemoteAddr"); - read_go_string(req_ptr, host_pos, http_server_span->host, sizeof(http_server_span->host), "host from Request.Host"); - read_go_string(req_ptr, proto_pos, http_server_span->proto, sizeof(http_server_span->proto), "proto from Request.Proto"); + read_go_string(url_ptr, + path_ptr_pos, + http_server_span->path, + sizeof(http_server_span->path), + "path from Request.URL"); + read_go_string(req_ptr, + remote_addr_pos, + http_server_span->remote_addr, + sizeof(http_server_span->remote_addr), + "remote addr from Request.RemoteAddr"); + read_go_string(req_ptr, + host_pos, + http_server_span->host, + sizeof(http_server_span->host), + "host from Request.Host"); + read_go_string(req_ptr, + proto_pos, + http_server_span->proto, + sizeof(http_server_span->proto), + "proto from Request.Proto"); // status code - bpf_probe_read(&http_server_span->status_code, sizeof(http_server_span->status_code), (void *)(resp_ptr + status_code_pos)); + bpf_probe_read(&http_server_span->status_code, + sizeof(http_server_span->status_code), + (void *)(resp_ptr + status_code_pos)); output_span_event(ctx, http_server_span, sizeof(*http_server_span), &http_server_span->sc); @@ -341,7 +362,7 @@ int uprobe_textproto_Reader_readContinuedLineSlice_Returns(struct pt_regs *ctx) if (!bpf_memicmp((const char *)temp, "traceparent: ", W3C_KEY_LENGTH + 2)) { struct span_context parent_span_context = {}; - w3c_string_to_span_context((char *)(temp + W3C_KEY_LENGTH + 2), &parent_span_context); + w3c_string_to_span_context((char *)(temp + W3C_KEY_LENGTH + 2), &parent_span_context); bpf_map_update_elem(&http_server_context_headers, &key, &parent_span_context, BPF_ANY); } } From c5492c2c5cebded1316ac6bca675a46cd779ff4f Mon Sep 17 00:00:00 2001 From: Rafael Roquetto Date: Mon, 18 Aug 2025 18:22:30 -0600 Subject: [PATCH 2/3] Add pre-commit hooks for clang-format --- Makefile | 8 ++++++++ hooks/pre-commit | 15 +++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 hooks/pre-commit diff --git a/Makefile b/Makefile index 7251749ce9..6c89ea66a5 100644 --- a/Makefile +++ b/Makefile @@ -308,3 +308,11 @@ markdown-lint: clang-format: find ./internal -type f -name "*.c" | xargs -P 0 -n 1 clang-format -i find ./internal -type f -name "*.h" | xargs -P 0 -n 1 clang-format -i + +.PHONY: install-hooks +install-hooks: + @if [ ! -f .git/hooks/pre-commit ]; then \ + echo "Installing pre-commit hook..."; \ + cp hooks/pre-commit .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit; \ + echo "Pre-commit hook installed."; \ + fi diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100644 index 0000000000..a32b8cd5e4 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,15 @@ +#!/bin/bash + +# Format all staged C/C++ files +files=$(git diff --cached --name-only --diff-filter=ACM | grep -E '\.(c|cpp|h)$') + +if [ -n "$files" ]; then + echo "Running clang-format on:" + echo "$files" + + # Run clang-format on the staged files + for file in $files; do + clang-format -i "$file" + git add "$file" # Re-add formatted files to staging + done +fi From 10acb3ab43b0e0e7b5e9bca7b01bce2a1e4378b6 Mon Sep 17 00:00:00 2001 From: Rafael Roquetto Date: Mon, 18 Aug 2025 18:24:37 -0600 Subject: [PATCH 3/3] Add github workflow --- .github/workflows/clang-format-check.yml | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/clang-format-check.yml diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml new file mode 100644 index 0000000000..ab6356fead --- /dev/null +++ b/.github/workflows/clang-format-check.yml @@ -0,0 +1,39 @@ +name: Clang Format Check + +on: + push: + branches: [ 'main' ] + pull_request: + branches: [ 'main' ] + +permissions: + contents: read + +jobs: + clang-format: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install Clang-Format + run: sudo apt-get install clang-format-19 + + - name: Check Clang Format + run: | + # Check for modified C/C++ files + files=$(git diff --name-status origin/main...HEAD | awk '$1 != "D" && /\.(c|cpp|h)$/ { print ($3 != "" ? $3 : $2) }' ) + if [ -z "$files" ]; then + echo "No C/C++ files modified." + exit 0 + fi + + # Run clang-format and check for changes + clang-format-19 -i $files + if ! git diff --exit-code $files; then + echo "Error: Some files are not formatted correctly. Please run clang-format on the modified files or use the provided pre-commit hook." + exit 1 + fi