set a keepalive_connections_max_age and add some testing to confirm it works

Author: jtmkrueger

config/schema.cue

@@ -414,6 +414,7 @@ import "path"
   router: {
     api_backends: {
       keepalive_idle_timeout: uint | *120
+      keepalive_connections_max_age: uint | *1800
     }
     trusted_proxies: [...string] | *[]
     global_rate_limits: {

src/api-umbrella/utils/active_config_store/set_envoy_config.lua

@@ -123,6 +123,13 @@ local function build_cluster_resource(cluster_name, options)
           -- important to improving performance by keeping pre-established
           -- connections around.
           idle_timeout = file_config["router"]["api_backends"]["keepalive_idle_timeout"] .. "s",
+
+          -- Maximum lifetime for any connection to an API backend, regardless
+          -- of activity. This ensures connections are periodically
+          -- re-established so that DNS or routing changes on the backend side
+          -- are picked up in a timely fashion, rather than holding open
+          -- connections indefinitely.
+          max_connection_duration = file_config["router"]["api_backends"]["keepalive_connections_max_age"] .. "s",
         },
       },
     },

test/proxy/keep_alive/test_server_side.rb

@@ -94,6 +94,110 @@ def test_keeps_idle_connections_open_verify_layers
     end
   end
 
+  def test_max_connection_age_closes_connections
+    envoy_upstream_keepalive_connections_max_age = 5
+    envoy_upstream_keepalive_idle_timeout = 120
+    override_config({
+      :router => {
+        :api_backends => {
+          # Set a long idle timeout so it doesn't interfere with the max age
+          # behavior being tested.
+          :keepalive_idle_timeout => envoy_upstream_keepalive_idle_timeout,
+          :keepalive_connections_max_age => envoy_upstream_keepalive_connections_max_age,
+        },
+      },
+    }) do
+      # Open a bunch of concurrent connections to establish a connection pool.
+      max_concurrency = 100
+      hydra = Typhoeus::Hydra.new(max_concurrency: max_concurrency)
+      requests = Array.new(300) do
+        request = Typhoeus::Request.new("http://127.0.0.1:9080/#{unique_test_class_id}/keepalive-default/delay/500", http_options)
+        hydra.queue(request)
+        request
+      end
+      hydra.run
+      assert_equal(300, requests.length)
+      requests.each do |req|
+        assert_response_code(200, req.response)
+      end
+
+      # Immediately after the requests, verify that Envoy has idle connections
+      # to the API backend (the connection pool is populated).
+      stats = connection_stats
+      max_concurrency_delta_buffer = (max_concurrency * 0.3).round
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_active_connections_per_envoy), max_concurrency_delta_buffer)
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend), max_concurrency_delta_buffer)
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend), max_concurrency_delta_buffer)
+
+      # Record the baseline count of locally-destroyed connections.
+      baseline_destroy_local = stats.fetch(:envoy_to_api_backend_destroy_local_connections_per_envoy)
+
+      # Wait for the max connection age to expire and verify Envoy closes the
+      # connections, even though the idle timeout (120s) hasn't been reached.
+      # Also measure the timing to verify it corresponds with the configured
+      # max age setting.
+      begin_time = Time.now
+      timing = nil
+      begin
+        stats = nil
+        # This should generally happen within the configured timeout seconds,
+        # but we'll add a significant timeout since we sometimes see this take
+        # longer in CI (but the exact timing of this behavior isn't really
+        # that important).
+        Timeout.timeout(300) do
+          loop do
+            stats = connection_stats
+            elapsed_time = Time.now - begin_time
+
+            # Once Envoy's active connections to the API backend drop to zero,
+            # note the elapsed time.
+            if !timing && stats.fetch(:envoy_to_api_backend_active_connections_per_envoy) == 0
+              timing = elapsed_time
+              break
+            end
+
+            sleep 0.1
+          end
+        end
+      rescue Timeout::Error
+        flunk("Envoy did not close connections after max connection age expired. Last connection stats: #{stats.inspect}")
+      end
+
+      # Verify Envoy closed the connections and the API backend agrees.
+      stats = connection_stats
+      assert_equal(0, stats.fetch(:envoy_to_api_backend_active_connections_per_envoy))
+      assert_operator(stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend), :<=, max_concurrency_delta_buffer)
+      assert_operator(stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend), :<=, max_concurrency_delta_buffer)
+
+      # Verify that Envoy initiated the connection closures (destroy_local
+      # should have increased).
+      assert_operator(stats.fetch(:envoy_to_api_backend_destroy_local_connections_per_envoy), :>, baseline_destroy_local)
+
+      # Check the timing of when connections were closed. This verifies that
+      # the observed behavior corresponds with the configured max age setting.
+      assert_in_delta(envoy_upstream_keepalive_connections_max_age, timing, 2)
+
+      # Make another batch of requests after the max age has expired to ensure
+      # that new connections work successfully.
+      requests = Array.new(300) do
+        request = Typhoeus::Request.new("http://127.0.0.1:9080/#{unique_test_class_id}/keepalive-default/delay/500", http_options)
+        hydra.queue(request)
+        request
+      end
+      hydra.run
+      assert_equal(300, requests.length)
+      requests.each do |req|
+        assert_response_code(200, req.response)
+      end
+
+      # Verify new connections were established.
+      stats = connection_stats
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_active_connections_per_envoy), max_concurrency_delta_buffer)
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_active_connections_per_api_backend), max_concurrency_delta_buffer)
+      assert_in_delta(max_concurrency, stats.fetch(:envoy_to_api_backend_idle_connections_per_api_backend), max_concurrency_delta_buffer)
+    end
+  end
+
   def test_concurrent_backend_connections_can_exceed_keepalive_count
     max_values = {
       client_to_nginx_router_active_connections_per_nginx_router: 0,