providers: add support for USDT tracing using libbpf/usdt

Provide lightweight USDT tracing as an alternative to LTTng.
This piggybacks on the existing tracing code added for LTTng
for a minimal set of changes.

The cmake build configuration is updated to allow either one
of -DTRACING=LTTNG or -DTRACING=USDT for this functionality
(by default tracing is disabled, as before).

> $ sudo bpftrace -l usdt:build/lib/lib*.so:*
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_recv
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:post_send
> usdt:build/lib/libefa-rdmav59.so:rdma_core_efa:process_completion
> usdt:build/lib/libefa.so:rdma_core_efa:post_recv
> usdt:build/lib/libefa.so:rdma_core_efa:post_send
> usdt:build/lib/libefa.so:rdma_core_efa:process_completion
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:poll_cq
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_recv
> usdt:build/lib/libhns-rdmav59.so:rdma_core_hns:post_send
> usdt:build/lib/libhns.so:rdma_core_hns:poll_cq
> usdt:build/lib/libhns.so:rdma_core_hns:post_recv
> usdt:build/lib/libhns.so:rdma_core_hns:post_send
> usdt:build/lib/libmlx5-rdmav59.so:rdma_core_mlx5:post_send
> usdt:build/lib/libmlx5.so:rdma_core_mlx5:post_send
> usdt:build/lib/librxe-rdmav59.so:rdma_core_rxe:post_send

The USDT header used here is from the libbpf/usdt project at
https://github.com/libbpf/usdt.git

Further background discussion for this commit is included in
linux-rdma#1621

Signed-off-by: Nathan Scott <nathans@redhat.com>

Author: natoscott

CMakeLists.txt

@@ -24,7 +24,7 @@
 #      Use the historical search path for providers, in the standard system library.
 #  -DNO_COMPAT_SYMS=1 (default disabled)
 #      Do not generate backwards compatibility symbols in the shared
-#      libraries. This may is necessary if using a dynmic linker that does
+#      libraries. This may be necessary if using a dynamic linker that does
 #      not support symbol versions, such as uclibc.
 #  -DIOCTL_MODE=write (default both)
 #      Disable new kABI ioctl() support and support only the legacy write
@@ -52,8 +52,9 @@
 #      Disable man pages. Allows rdma-core to be built and installed
 #      (without man pages) when neither pandoc/rst2man nor the pandoc-prebuilt
 #      directory are available.
-#   -DENABLE_LTTNG (default, no tracing support)
-#      Enable LTTng tracing.
+#   -DTRACING=LTTNG/USDT/None (default None)
+#      Enable LTTng (Linux Trace Toolkit) or USDT (Userspace Statically Defined)
+#      trace points for dynamic tracing.
 
 if (${CMAKE_VERSION} VERSION_LESS "3.18.1")
 	# Centos 7 support
@@ -171,6 +172,9 @@ endif()
 set(DISTRO_FLAVOUR "None" CACHE
   STRING "Flavour of distribution to install for. This primarily impacts the init.d scripts installed.")
 
+set(TRACING "None" CACHE
+  STRING "Enable LTTNG (Linux Trace Toolkit) or USDT (Userspace Statically Defined Tracing).")
+
 #-------------------------
 # Load CMake components
 set(BUILDLIB "${PROJECT_SOURCE_DIR}/buildlib")
@@ -567,10 +571,14 @@ if (DRM_INCLUDE_DIRS)
   endif()
 endif()
 
-# LTTng Tracer support
-if (DEFINED ENABLE_LTTNG)
+# Dynamic Tracing support
+if (TRACING STREQUAL "LTTNG")
   include(FindLTTngUST REQUIRED)
   add_definitions(-DLTTNG_ENABLED)
+elseif (TRACING STREQUAL "USDT")
+  add_definitions(-DUSDT_ENABLED)
+elseif (NOT TRACING STREQUAL "None")
+  message(FATAL_ERROR "-DTRACING=${TRACING} is not a valid choice")
 endif()
 
 #-------------------------

Documentation/contributing.md

@@ -66,7 +66,7 @@ that are designed to weed out bugs.
 Serious errors will result in a red X in the PR and will need to be corrected.
 Less serious errors, including checkpatch related, will show up with a green
 check but it is necessary to check the details to see that everything is
-appropriate. checkpatch is an informative too, not all of its feedback is
+appropriate. checkpatch is an informative tool, not all of its feedback is
 appropriate to fix.
 
 A build similar to AZP can be run locally using docker and the

providers/efa/efa_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_efa
@@ -109,7 +113,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined (USDT_ENABLED)
+
+#ifndef __EFA_TRACE_H__
+#define __EFA_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __EFA_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __EFA_TRACE_H__
 #define __EFA_TRACE_H__
@@ -118,4 +133,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __EFA_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/efa/verbs.c

@@ -2171,7 +2171,7 @@ static void efa_set_common_ctrl_flags(struct efa_io_tx_meta_desc *desc,
 	EFA_SET(&desc->ctrl2, EFA_IO_TX_META_DESC_COMP_REQ, 1);
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint32_t efa_get_wqe_length(struct efa_io_tx_wqe *tx_wqe)
 {
 	enum efa_io_send_op_type op_type;

providers/hns/hns_roce_u_hw_v2.c

@@ -683,7 +683,7 @@ static int parse_cqe_for_cq(struct hns_roce_context *ctx, struct hns_roce_cq *cq
 	return 0;
 }
 
-#ifdef LTTNG_ENABLED
+#if defined(LTTNG_ENABLED) || defined(USDT_ENABLED)
 static uint8_t read_wc_sl(struct hns_roce_qp *hr_qp,
 				   struct hns_roce_v2_cqe *cqe,
 				   struct ibv_wc *wc)

providers/hns/hns_roce_u_trace.h

@@ -3,7 +3,11 @@
  * Copyright (c) 2025 Hisilicon Limited.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_hns
@@ -121,7 +125,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __HNS_TRACE_H__
+#define __HNS_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __HNS_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __HNS_TRACE_H__
 #define __HNS_TRACE_H__
@@ -130,4 +145,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __HNS_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/mlx5/mlx5_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_mlx5
@@ -47,7 +51,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __MLX5_TRACE_H__
+#define __MLX5_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __MLX5_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __MLX5_TRACE_H__
 #define __MLX5_TRACE_H__
@@ -56,4 +71,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __MLX5_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

providers/rxe/rxe_trace.h

@@ -3,7 +3,11 @@
  * Copyright 2023 Bytedance.com, Inc. or its affiliates. All rights reserved.
  */
 
-#if defined(LTTNG_ENABLED)
+#if defined(LTTNG_ENABLED) && defined(USDT_ENABLED)
+
+#error "Only one of LTTNG_ENABLED and USDT_ENABLED allowed"
+
+#elif defined(LTTNG_ENABLED)
 
 #undef LTTNG_UST_TRACEPOINT_PROVIDER
 #define LTTNG_UST_TRACEPOINT_PROVIDER rdma_core_rxe
@@ -47,7 +51,18 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #include <lttng/tracepoint-event.h>
 
-#else
+#elif defined(USDT_ENABLED)
+
+#ifndef __RXE_TRACE_H__
+#define __RXE_TRACE_H__
+
+#include <util/usdt.h>
+
+#define rdma_tracepoint(arg...) USDT(arg)
+
+#endif /* __RXE_TRACE_H__*/
+
+#else /* !defined(LTTNG_ENABLED) && !defined(USDT_ENABLED) */
 
 #ifndef __RXE_TRACE_H__
 #define __RXE_TRACE_H__
@@ -56,4 +71,4 @@ LTTNG_UST_TRACEPOINT_EVENT(
 
 #endif /* __RXE_TRACE_H__*/
 
-#endif /* defined(LTTNG_ENABLED) */
+#endif /* defined(LTTNG_ENABLED) || defined(USDT_ENABLED) */

util/CMakeLists.txt

@@ -6,6 +6,7 @@ publish_internal_headers(util
   node_name_map.h
   rdma_nl.h
   symver.h
+  usdt.h
   util.h
   )