invoke waf for curl requests

Author: cataphract

appsec/src/extension/configuration.h

@@ -70,7 +70,9 @@ extern bool runtime_config_first_init;
     CONFIG(STRING, DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON, "")                                                                          \
     CONFIG(BOOL, DD_APM_TRACING_ENABLED, "true")                                                                                      \
     CONFIG(BOOL, DD_API_SECURITY_ENABLED, "true", .ini_change = zai_config_system_ini_change)                                         \
-    CONFIG(DOUBLE, DD_API_SECURITY_SAMPLE_DELAY, "30.0", .ini_change = zai_config_system_ini_change)
+    CONFIG(DOUBLE, DD_API_SECURITY_SAMPLE_DELAY, "30.0", .ini_change = zai_config_system_ini_change)                                  \
+    CONFIG(INT, DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS, "1")                                                            \
+    CONFIG(DOUBLE, DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE, "0.5")
 
 #ifdef __linux__
 #define DD_CONFIGURATION \

appsec/src/extension/curl.c

@@ -0,0 +1,46 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog
+// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
+
+// clang-format off
+#include <php.h>  // (must come before php_streams.h)
+// clang-format on
+#include <main/php_streams.h>
+
+#include "compatibility.h"
+#include "php_compat.h"
+#include "php_objects.h"
+
+static PHP_FUNCTION(datadog_appsec_fflush_stdiocast)
+{
+    zval *stream_zv;
+
+    ZEND_PARSE_PARAMETERS_START(1, 1)
+    Z_PARAM_RESOURCE(stream_zv)
+    ZEND_PARSE_PARAMETERS_END();
+
+    php_stream *stream = NULL;
+    php_stream_from_zval(stream, stream_zv);
+
+    if (stream->stdiocast != NULL) {
+        int result = fflush(stream->stdiocast);
+        RETURN_BOOL(result == 0);
+    }
+
+    RETURN_TRUE;
+}
+
+// clang-format off
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(fflush_stdiocast_arginfo, 0, 1, _IS_BOOL, 0)
+    ZEND_ARG_INFO(0, stream)
+ZEND_END_ARG_INFO()
+
+static const zend_function_entry functions[] = {
+    ZEND_RAW_FENTRY(DD_APPSEC_NS "fflush_stdiocast", PHP_FN(datadog_appsec_fflush_stdiocast), fflush_stdiocast_arginfo, 0, NULL, NULL)
+    PHP_FE_END
+};
+// clang-format on
+
+void dd_curl_register_functions(void) { dd_phpobj_reg_funcs(functions); }

appsec/src/extension/curl.h

@@ -0,0 +1,9 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog
+// (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
+#pragma once
+
+void dd_curl_register_functions(void);
+

appsec/src/extension/ddappsec.c

@@ -28,6 +28,7 @@
 #include "commands_ctx.h"
 #include "compatibility.h"
 #include "configuration.h"
+#include "curl.h"
 #include "ddappsec.h"
 #include "dddefs.h"
 #include "ddtrace.h"
@@ -515,7 +516,7 @@ static PHP_FUNCTION(datadog_appsec_push_addresses)
     if (!DDAPPSEC_G(active)) {
         mlog(dd_log_debug, "Trying to access to push_addresses "
                            "function while appsec is disabled");
-        return;
+        RETURN_FALSE;
     }
 
     zend_array *addresses = NULL;
@@ -577,13 +578,13 @@ static PHP_FUNCTION(datadog_appsec_push_addresses)
 
     if (opts.rasp_rule && ZSTR_LEN(opts.rasp_rule) > 0 &&
         !get_global_DD_APPSEC_RASP_ENABLED()) {
-        return;
+        RETURN_FALSE;
     }
 
     dd_conn *conn = dd_helper_mgr_cur_conn();
     if (conn == NULL) {
         mlog_g(dd_log_debug, "No connection; skipping push_addresses");
-        return;
+        RETURN_FALSE;
     }
 
     struct block_params block_params = {0};
@@ -603,6 +604,8 @@ static PHP_FUNCTION(datadog_appsec_push_addresses)
 
     dd_req_lifecycle_abort(REQUEST_STAGE_MID_REQUEST, res, &block_params);
     dd_request_abort_destroy_block_params(&block_params);
+
+    RETURN_TRUE;
 }
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(
@@ -617,7 +620,7 @@ ZEND_ARG_INFO_WITH_DEFAULT_VALUE(0, "subctx_last_call", false)
 ZEND_END_ARG_INFO()
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(
-    push_addresses_arginfo, 0, 0, IS_VOID, 1)
+    push_addresses_arginfo, 0, 0, _IS_BOOL, 1)
 ZEND_ARG_INFO(0, addresses)
 ZEND_ARG_INFO_WITH_DEFAULT_VALUE(0, rasp_rule_or_opts, NULL)
 ZEND_END_ARG_INFO()
@@ -642,6 +645,7 @@ static const zend_function_entry testing_functions[] = {
 static void _register_testing_objects(void)
 {
     dd_phpobj_reg_funcs(functions);
+    dd_curl_register_functions();
 
     if (!get_global_DD_APPSEC_TESTING()) {
         return;

appsec/src/extension/entity_body.c

@@ -72,9 +72,7 @@ void dd_entity_body_startup(void)
 
     dd_xml_parser_startup();
 
-    if (get_global_DD_APPSEC_TESTING()) {
-        dd_phpobj_reg_funcs(ext_functions);
-    }
+    dd_phpobj_reg_funcs(ext_functions);
 }
 
 void dd_entity_body_shutdown(void) { dd_xml_parser_shutdown(); }

appsec/src/extension/json_truncated_parser.cpp

@@ -90,8 +90,8 @@ class TruncatedJsonInputStream {
 public:
     using Ch = char;
 
-    // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
     TruncatedJsonInputStream(
+        // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
         const char *data, std::size_t length, std::size_t size_limit = SIZE_MAX)
         : data_{data}, length_{length}, size_limit_{size_limit}
     {}
@@ -189,9 +189,9 @@ class ToZvalHandler {
         return AddValue(val);
     }
 
-    // NOLINTNEXTLINE(readability-convert-member-functions-to-static,readability-named-parameter)
-    auto RawNumber(
-        const TruncatedJsonInputStream::Ch *, rapidjson::SizeType, bool) -> bool
+    // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
+    auto RawNumber(const TruncatedJsonInputStream::Ch * /* unused */,
+        rapidjson::SizeType /* unused */, bool /* unused */) -> bool
     {
         assert("RawNumber should not be called (requires flags we are not "
                "using)" == nullptr);

appsec/src/extension/request_abort.c

@@ -364,10 +364,6 @@ zend_array *nonnull dd_request_abort_redirect_spec(
 // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
 void _request_abort_static_page(struct block_params *nonnull block_params)
 {
-    SG(sapi_headers).http_response_code = block_params->response_code
-                                              ? block_params->response_code
-                                              : DEFAULT_BLOCKING_RESPONSE_CODE;
-
     dd_response_type response_type = block_params->response_type;
     if (response_type == response_type_auto) {
         zval *server =
@@ -402,6 +398,10 @@ void _request_abort_static_page(struct block_params *nonnull block_params)
         return;
     }
 
+    SG(sapi_headers).http_response_code = block_params->response_code
+                                              ? block_params->response_code
+                                              : DEFAULT_BLOCKING_RESPONSE_CODE;
+
     _set_content_type(content_type);
     _set_output_zstr(body);
     zend_string_release(body);

appsec/src/extension/request_lifecycle.c

@@ -39,10 +39,11 @@ static void _reset_globals(void);
 const zend_array *nonnull _get_server_equiv(
     const zend_array *nonnull superglob_equiv);
 static uint64_t _calc_sampling_key(zend_object *root_span, int status_code);
-static void _register_testing_objects(void);
+static void _register_functions(void);
 
 static bool _enabled_user_req;
 static zend_string *_server_zstr;
+static zend_string *_dd_downstream_request_metric;
 
 static THREAD_LOCAL_ON_ZTS zend_object *nullable _cur_req_span;
 static THREAD_LOCAL_ON_ZTS zend_array *nullable _superglob_equiv;
@@ -54,6 +55,9 @@ static THREAD_LOCAL_ON_ZTS bool _request_blocked;
 #define MAX_LENGTH_OF_REM_CFG_PATH 31
 static THREAD_LOCAL_ON_ZTS char
     _last_rem_cfg_path[MAX_LENGTH_OF_REM_CFG_PATH + 1];
+static THREAD_LOCAL_ON_ZTS uint64_t _downstream_body_count_this_req;
+static THREAD_LOCAL_ON_ZTS uint64_t _downstream_body_count_total;
+
 #define CLIENT_IP_LOOKUP_FAILED ((zend_string *)-1)
 
 bool dd_req_is_user_req(void) { return _enabled_user_req; }
@@ -87,8 +91,10 @@ void dd_req_lifecycle_startup(void)
     }
 
     _server_zstr = zend_string_init_interned(LSTRARG("_SERVER"), 1);
+    _dd_downstream_request_metric =
+        zend_string_init_interned(LSTRARG("_dd.appsec.downstream_request"), 1);
 
-    _register_testing_objects();
+    _register_functions();
 }
 
 void dd_req_lifecycle_rinit(bool force)
@@ -193,6 +199,8 @@ static zend_array *nullable _do_request_begin(
 
     dd_tags_rinit();
 
+    _downstream_body_count_this_req = 0;
+
     zend_string *nullable rbe = NULL;
     if (rbe_zv) {
         rbe = _get_entity_as_string(rbe_zv);
@@ -272,6 +280,21 @@ void dd_req_lifecycle_rshutdown(bool ignore_verdict, bool force)
         return;
     }
 
+    // RFC-1062: Set downstream request metric if any bodies were analyzed
+    if (_cur_req_span && _downstream_body_count_this_req > 0) {
+        zval *metrics_zv = dd_trace_span_get_metrics(_cur_req_span);
+        if (metrics_zv) {
+            zval zv;
+            ZVAL_DOUBLE(&zv, 1.0);
+            zend_hash_update(
+                Z_ARRVAL_P(metrics_zv), _dd_downstream_request_metric, &zv);
+            mlog_g(dd_log_debug,
+                "Set _dd.appsec.downstream_request metric (analyzed %" PRIu64
+                " bodies)",
+                _downstream_body_count_this_req);
+        }
+    }
+
     if (_enabled_user_req) {
         if (_cur_req_span) {
             mlog_g(dd_log_info,
@@ -1082,20 +1105,74 @@ PHP_FUNCTION(datadog_appsec_testing_dump_req_lifecycle_state)
     }
 }
 
+static PHP_FUNCTION(datadog_appsec_should_send_downstream_bodies)
+{
+    if (zend_parse_parameters_none() == FAILURE) {
+        RETURN_FALSE;
+    }
+
+    if (!DDAPPSEC_G(active)) {
+        RETURN_FALSE;
+    }
+
+    _downstream_body_count_total++;
+
+    if (_downstream_body_count_this_req >=
+        (uint64_t)get_DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS()) {
+        mlog_g(dd_log_debug,
+            "Downstream body count limit reached for this request (did for "
+            "%" PRIu64 " requests)",
+            _downstream_body_count_this_req);
+        RETURN_FALSE;
+    }
+
+    double sample_rate =
+        get_DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE();
+    if (sample_rate >= 1.0) {
+        mlog_g(dd_log_debug, "Downstream body analysis sample rate is 1.0; "
+                             "sending all downstream bodies up to the limit");
+        _downstream_body_count_this_req++;
+        RETURN_TRUE;
+    }
+
+    static const uint64_t KNUTH_FACTOR = 11400714819323198549ULL;
+    uint64_t threshold = (uint64_t)(sample_rate * (double)UINT64_MAX);
+    uint64_t sample_value = _downstream_body_count_total * KNUTH_FACTOR;
+
+    if (sample_value < threshold) {
+        _downstream_body_count_this_req++;
+        mlog_g(dd_log_debug,
+            "We're sending the bodies of this request for analysis");
+        RETURN_TRUE;
+    }
+
+    mlog_g(dd_log_debug,
+        "We're NOT sending the bodies of this request for analysis");
+    RETURN_FALSE;
+}
+
 // clang-format off
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(dump_arginfo, 0, 0, IS_ARRAY, 0)
 ZEND_END_ARG_INFO()
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(dump_downstream_bodies_arginfo, 0, 0, _IS_BOOL, 0)
+ZEND_END_ARG_INFO()
 
 static const zend_function_entry functions[] = {
+    ZEND_RAW_FENTRY(DD_APPSEC_NS "should_send_downstream_bodies", PHP_FN(datadog_appsec_should_send_downstream_bodies), dump_downstream_bodies_arginfo, 0, NULL, NULL)
+    PHP_FE_END
+};
+
+static const zend_function_entry test_functions[] = {
     ZEND_RAW_FENTRY(DD_TESTING_NS "dump_req_lifecycle_state", PHP_FN(datadog_appsec_testing_dump_req_lifecycle_state), dump_arginfo, 0, NULL, NULL)
     PHP_FE_END
 };
 // clang-format on
 
-static void _register_testing_objects(void)
+static void _register_functions(void)
 {
-    if (!get_global_DD_APPSEC_TESTING()) {
-        return;
-    }
     dd_phpobj_reg_funcs(functions);
+
+    if (get_global_DD_APPSEC_TESTING()) {
+        dd_phpobj_reg_funcs(test_functions);
+    }
 }

appsec/src/helper/parameter_base.hpp

@@ -41,6 +41,7 @@ class parameter_base {
     [[nodiscard]] parameter_type type() const noexcept
     {
         switch (ddwaf_object_get_type(&obj_)) {
+        default:
         case DDWAF_OBJ_INVALID:
             return parameter_type::invalid;
         case DDWAF_OBJ_NULL: