From 52224612763d912d2bd1b0498c96ad73f5ee7dd9 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Wed, 10 Sep 2025 14:29:49 -0400 Subject: [PATCH 1/3] chore(rust): add more memory usage statstics to benchmarks --- .../benchmarks/rust/Cargo.toml | 34 +++--- .../benchmarks/rust/README.md | 12 +- .../benchmarks/rust/src/alloc.rs | 112 ++++++++++++++++++ .../benchmarks/rust/src/benchmark.rs | 6 +- .../benchmarks/rust/src/main.rs | 16 +-- .../benchmarks/rust/src/results.rs | 2 + .../benchmarks/rust/src/tests.rs | 33 ++++-- 7 files changed, 175 insertions(+), 40 deletions(-) create mode 100644 esdk-performance-testing/benchmarks/rust/src/alloc.rs diff --git a/esdk-performance-testing/benchmarks/rust/Cargo.toml b/esdk-performance-testing/benchmarks/rust/Cargo.toml index 8f392990d..8c493549b 100644 --- a/esdk-performance-testing/benchmarks/rust/Cargo.toml +++ b/esdk-performance-testing/benchmarks/rust/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "esdk-benchmark" version = "0.1.0" -edition = "2021" +edition = "2024" [[bin]] name = "esdk_benchmark" @@ -9,10 +9,11 @@ path = "src/main.rs" [dependencies] # AWS Encryption SDK +# aws-esdk = { path = "/Users/ajewell/esdk_rust/esdk" } aws-esdk = { path = "../../../AwsEncryptionSDK/runtimes/rust" } # Async runtime -tokio = { version = "1.0", features = ["full"] } +tokio = { version = "1.47", features = ["full"] } # Serialization serde = { version = "1.0", features = ["derive"] } @@ -20,41 +21,42 @@ serde_json = "1.0" serde_yaml = "0.9" # CLI and progress -clap = { version = "4.0", features = ["derive"] } -indicatif = "0.17" +clap = { version = "4.5", features = ["derive"] } +indicatif = "0.18" # Statistics and benchmarking hdrhistogram = "7.5" -rand = "0.8" +rand = "0.9" # System info -sysinfo = "0.30" +sysinfo = "0.37" # Error handling anyhow = "1.0" -thiserror = "1.0" +thiserror = "2.0" # Logging log = "0.4" -env_logger = "0.10" +env_logger = "0.11" # Time chrono = { version = "0.4", features = ["serde"] } # Memory profiling -memory-stats = "1.0" +memory-stats = "1.2" stats_alloc = "0.1" # Async utilities futures = "0.3" aws-config = "1.8.6" -aws-sdk-dynamodb = "1.73.0" -aws-sdk-kms = "1.67.0" -aws-sdk-sso = "1.69.0" -aws-sdk-ssooidc = "1.69.0" -aws-sdk-sts = "1.69.0" -aws-smithy-types = "1.2" +aws-sdk-dynamodb = "1.92.0" +aws-sdk-kms = "1.86.0" +aws-sdk-sso = "1.83.0" +aws-sdk-ssooidc = "1.84.0" +aws-sdk-sts = "1.85.0" +aws-smithy-types = "1.3" +cpu-time = "1.0.0" [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.7", features = ["html_reports"] } diff --git a/esdk-performance-testing/benchmarks/rust/README.md b/esdk-performance-testing/benchmarks/rust/README.md index 5bb240fba..cc4ca3507 100644 --- a/esdk-performance-testing/benchmarks/rust/README.md +++ b/esdk-performance-testing/benchmarks/rust/README.md @@ -11,18 +11,22 @@ Rust implementation of the AWS Encryption SDK performance benchmark suite. ## Prerequisites -- Rust 1.85.0+ ([rustup.rs](https://rustup.rs/)) +- Rust 1.86.0+ ([rustup.rs](https://rustup.rs/)) - Build tools (Xcode CLI Tools on macOS, build-essential on Linux) ## Quick Start ```bash +# If necessary, build the ESDK and return here +cd ../../../AwsEncryptionSDK/ +make polymorph_rust transpile_rust +cd ../esdk-performance-testing/benchmarks/rust/ + # Build and run -cargo build --release -./target/release/esdk_benchmark --config ../../config/test-scenarios.yaml +cargo run --release -- --config ../../config/test-scenarios.yaml # Quick test (requires quick_config in YAML) -./target/release/esdk_benchmark --quick +cargo run --release -- --quick ``` ## Configuration diff --git a/esdk-performance-testing/benchmarks/rust/src/alloc.rs b/esdk-performance-testing/benchmarks/rust/src/alloc.rs new file mode 100644 index 000000000..0d7e7e6dd --- /dev/null +++ b/esdk-performance-testing/benchmarks/rust/src/alloc.rs @@ -0,0 +1,112 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] +use serde::Serialize; +use std::sync::atomic::AtomicIsize; +use std::sync::atomic::Ordering; + +pub struct ResourceTracker { + pub count: isize, + pub total: isize, + pub net_total: isize, + pub net_count: isize, +} + +#[derive(Debug, Default, Clone, Serialize)] +pub struct ResourceResults { + pub count_k: isize, + pub total_m: isize, + pub max_bytes_m: isize, + pub net_count: isize, + pub net_total: isize, +} + +impl ResourceTracker { + pub fn new() -> Self { + Self { + count: get_counter(), + total: get_total(), + net_total: get_net_total(), + net_count: get_net_counter(), + } + } + pub fn report_leak(&self) { + println!( + "{} outstanding allocations totalling {} bytes.", + get_net_counter(), + get_net_total() + ); + } + pub fn get_results(&self) -> ResourceResults { + ResourceResults { + count_k: (get_counter() - self.count) / 1000, + total_m: (get_total() - self.total) / 1_000_000, + max_bytes_m: (get_max_total() - self.net_total) / 1_000_000, + net_count: get_net_counter() - self.net_count, + net_total: (get_net_total() - self.net_total) / 1_000_000, + } + } +} + +static COUNTER: AtomicIsize = AtomicIsize::new(0); +static TOTAL: AtomicIsize = AtomicIsize::new(0); + +static NET_COUNTER: AtomicIsize = AtomicIsize::new(0); +static NET_TOTAL: AtomicIsize = AtomicIsize::new(0); +static MAX_NET_TOTAL: AtomicIsize = AtomicIsize::new(0); + +fn add_to_counter(inc: isize) { + COUNTER.fetch_add(1, Ordering::SeqCst); + TOTAL.fetch_add(inc, Ordering::SeqCst); + NET_COUNTER.fetch_add(1, Ordering::SeqCst); + NET_TOTAL.fetch_add(inc, Ordering::SeqCst); + MAX_NET_TOTAL.fetch_max(NET_TOTAL.load(Ordering::SeqCst), Ordering::SeqCst); +} + +fn subtract_from_counter(inc: isize) { + NET_COUNTER.fetch_sub(1, Ordering::SeqCst); + NET_TOTAL.fetch_sub(inc, Ordering::SeqCst); +} + +fn get_counter() -> isize { + COUNTER.load(Ordering::SeqCst) +} + +fn get_total() -> isize { + TOTAL.load(Ordering::SeqCst) +} + +fn get_net_counter() -> isize { + NET_COUNTER.load(Ordering::SeqCst) +} + +fn get_net_total() -> isize { + NET_TOTAL.load(Ordering::SeqCst) +} + +fn clear_max() { + MAX_NET_TOTAL.store(0, Ordering::SeqCst) +} + +fn get_max_total() -> isize { + MAX_NET_TOTAL.load(Ordering::SeqCst) +} + +use std::alloc::{GlobalAlloc, Layout, System}; + +struct MyAllocator; + +unsafe impl GlobalAlloc for MyAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + add_to_counter(layout.size() as isize); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + subtract_from_counter(layout.size() as isize); + unsafe { System.dealloc(ptr, layout) } + } +} + +#[global_allocator] +static GLOBAL: MyAllocator = MyAllocator; diff --git a/esdk-performance-testing/benchmarks/rust/src/benchmark.rs b/esdk-performance-testing/benchmarks/rust/src/benchmark.rs index bdb57274a..de974cb92 100644 --- a/esdk-performance-testing/benchmarks/rust/src/benchmark.rs +++ b/esdk-performance-testing/benchmarks/rust/src/benchmark.rs @@ -4,15 +4,15 @@ use anyhow::{Context, Result}; use aws_esdk::client as esdk_client; use aws_esdk::material_providers::client as mpl_client; -use aws_esdk::material_providers::types::material_providers_config::MaterialProvidersConfig; use aws_esdk::material_providers::types::AesWrappingAlg; use aws_esdk::material_providers::types::EsdkCommitmentPolicy; +use aws_esdk::material_providers::types::material_providers_config::MaterialProvidersConfig; use aws_esdk::types::aws_encryption_sdk_config::AwsEncryptionSdkConfig; use log::info; use rand::Rng; use sysinfo::System; -use crate::config::{load_config, TestConfig}; +use crate::config::{TestConfig, load_config}; use crate::results::BenchmarkResult; // Constants for memory testing @@ -72,7 +72,7 @@ impl EsdkBenchmark { // Create default AES-256 keyring let mut key = [0u8; 32]; // 256-bit key - rand::thread_rng().fill(&mut key); + rand::rng().fill(&mut key); let raw_keyring = mpl_client .create_raw_aes_keyring() diff --git a/esdk-performance-testing/benchmarks/rust/src/main.rs b/esdk-performance-testing/benchmarks/rust/src/main.rs index d7b58f18d..c06292ca8 100644 --- a/esdk-performance-testing/benchmarks/rust/src/main.rs +++ b/esdk-performance-testing/benchmarks/rust/src/main.rs @@ -1,6 +1,9 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(clippy::collapsible_if)] + +mod alloc; mod benchmark; mod config; mod results; @@ -18,10 +21,8 @@ fn main() -> Result<()> { .thread_stack_size(8 * 1024 * 1024) // 8MB stack size .enable_all() .build()?; - - rt.block_on(async { - run_benchmark().await - }) + + rt.block_on(async { run_benchmark().await }) } async fn run_benchmark() -> Result<()> { @@ -63,9 +64,10 @@ async fn run_benchmark() -> Result<()> { // Adjust config for quick test if quick { - let quick_config = bench.config.quick_config.as_ref() - .ok_or_else(|| anyhow::anyhow!("Quick mode requested but no quick_config found in config file"))?; - + let quick_config = bench.config.quick_config.as_ref().ok_or_else(|| { + anyhow::anyhow!("Quick mode requested but no quick_config found in config file") + })?; + bench.config.iterations.measurement = quick_config.iterations.measurement; bench.config.iterations.warmup = quick_config.iterations.warmup; bench.config.data_sizes.small = quick_config.data_sizes.small.clone(); diff --git a/esdk-performance-testing/benchmarks/rust/src/results.rs b/esdk-performance-testing/benchmarks/rust/src/results.rs index b97bd2556..5a0e559bc 100644 --- a/esdk-performance-testing/benchmarks/rust/src/results.rs +++ b/esdk-performance-testing/benchmarks/rust/src/results.rs @@ -1,6 +1,7 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +use crate::alloc; use anyhow::Result; use chrono::Utc; use serde::Serialize; @@ -28,6 +29,7 @@ pub struct BenchmarkResult { pub rust_version: String, pub cpu_count: usize, pub total_memory_gb: f64, + pub alloc: alloc::ResourceResults, } #[derive(Debug, Serialize)] diff --git a/esdk-performance-testing/benchmarks/rust/src/tests.rs b/esdk-performance-testing/benchmarks/rust/src/tests.rs index 20e57504d..fa9e3dfe0 100644 --- a/esdk-performance-testing/benchmarks/rust/src/tests.rs +++ b/esdk-performance-testing/benchmarks/rust/src/tests.rs @@ -1,6 +1,7 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +use crate::alloc; use anyhow::Result; use aws_esdk::client as esdk_client; use chrono::Utc; @@ -13,7 +14,7 @@ use std::collections::HashMap; use std::time::Instant; use crate::benchmark::{EsdkBenchmark, MEMORY_TEST_ITERATIONS}; -use crate::results::{average, percentile, BenchmarkResult}; +use crate::results::{BenchmarkResult, average, percentile}; impl EsdkBenchmark { // === Helper Functions === @@ -81,7 +82,7 @@ impl EsdkBenchmark { // Generate test data on heap to avoid stack overflow let data = { let mut data = vec![0u8; data_size]; - rand::thread_rng().fill(&mut data[..]); + rand::rng().fill(&mut data[..]); data.into_boxed_slice() }; @@ -109,6 +110,7 @@ impl EsdkBenchmark { pb.set_message("Throughput test"); let start_time = Instant::now(); + let alloc = alloc::ResourceTracker::new(); for i in 0..iterations { let iteration_start = Instant::now(); let (encrypt_ms, decrypt_ms) = self @@ -148,6 +150,7 @@ impl EsdkBenchmark { rust_version: std::env::var("RUSTC_VERSION").unwrap_or_else(|_| "unknown".to_string()), cpu_count: self.cpu_count, total_memory_gb: self.total_memory_gb, + alloc: alloc.get_results(), }; info!( @@ -171,19 +174,20 @@ impl EsdkBenchmark { // Generate test data on heap to avoid stack overflow let data = { let mut data = vec![0u8; data_size]; - rand::thread_rng().fill(&mut data[..]); + rand::rng().fill(&mut data[..]); data.into_boxed_slice() }; let mut peak_memory_delta_mb = 0.0; let mut avg_memory_samples = Vec::new(); + let alloc = alloc::ResourceTracker::new(); // Run iterations with memory sampling for i in 0..MEMORY_TEST_ITERATIONS { // Force garbage collection and get baseline memory std::hint::black_box(&data); // Prevent optimization tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - + let baseline_memory = if let Some(stats) = memory_stats() { stats.physical_mem as f64 / (1024.0 * 1024.0) } else { @@ -194,7 +198,7 @@ impl EsdkBenchmark { let mut iteration_peak = baseline_memory; let operation_start = Instant::now(); - + // Run ESDK operation with memory sampling let data_clone = data.clone(); let esdk_client = self.esdk_client.clone(); @@ -215,7 +219,7 @@ impl EsdkBenchmark { if delta > 0.0 { iteration_samples.push(delta); } - + // Track peak regardless of sign if current_physical > iteration_peak { iteration_peak = current_physical; @@ -256,7 +260,11 @@ impl EsdkBenchmark { info!( "=== Iteration {} === Peak Delta: {:.2} MB, Avg Delta: {:.2} MB ({:?}, {} samples)", - i + 1, iter_peak_delta_mb, iter_avg_delta_mb, operation_duration, sample_count + i + 1, + iter_peak_delta_mb, + iter_avg_delta_mb, + operation_duration, + sample_count ); } @@ -279,7 +287,10 @@ impl EsdkBenchmark { "- Peak Memory Delta: {:.2} MB (operation overhead)", peak_memory_delta_mb ); - info!("- Average Memory Delta: {:.2} MB (operation overhead)", overall_avg_delta_mb); + info!( + "- Average Memory Delta: {:.2} MB (operation overhead)", + overall_avg_delta_mb + ); let result = BenchmarkResult { test_name: "memory".to_string(), @@ -300,6 +311,7 @@ impl EsdkBenchmark { rust_version: std::env::var("RUSTC_VERSION").unwrap_or_else(|_| "unknown".to_string()), cpu_count: self.cpu_count, total_memory_gb: self.total_memory_gb, + alloc: alloc.get_results(), }; Ok(result) @@ -372,7 +384,7 @@ impl EsdkBenchmark { // Generate data per worker on heap to avoid stack overflow let worker_data = { let mut data = vec![0u8; data_size]; - rand::thread_rng().fill(&mut data[..]); + rand::rng().fill(&mut data[..]); data }; @@ -421,6 +433,7 @@ impl EsdkBenchmark { } let start_time = Instant::now(); + let alloc = alloc::ResourceTracker::new(); let results = join_all(tasks).await; let total_duration = start_time.elapsed().as_secs_f64(); @@ -455,6 +468,7 @@ impl EsdkBenchmark { rust_version: std::env::var("RUSTC_VERSION").unwrap_or_else(|_| "unknown".to_string()), cpu_count: self.cpu_count, total_memory_gb: self.total_memory_gb, + alloc: alloc.get_results(), }; info!( @@ -557,7 +571,6 @@ impl EsdkBenchmark { } else { info!("Skipping concurrency tests (not in test_types)"); } - info!( "Benchmark suite completed. Total results: {}", self.results.len() From 254c7d064cdec891ea319794179388cdb14cd849 Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Thu, 11 Sep 2025 07:03:19 -0400 Subject: [PATCH 2/3] clean up alloc.rs --- .../benchmarks/rust/src/alloc.rs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/esdk-performance-testing/benchmarks/rust/src/alloc.rs b/esdk-performance-testing/benchmarks/rust/src/alloc.rs index 0d7e7e6dd..df9bb464b 100644 --- a/esdk-performance-testing/benchmarks/rust/src/alloc.rs +++ b/esdk-performance-testing/benchmarks/rust/src/alloc.rs @@ -1,6 +1,5 @@ // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -#![allow(dead_code)] use serde::Serialize; use std::sync::atomic::AtomicIsize; use std::sync::atomic::Ordering; @@ -23,6 +22,7 @@ pub struct ResourceResults { impl ResourceTracker { pub fn new() -> Self { + clear_max(); Self { count: get_counter(), total: get_total(), @@ -30,13 +30,6 @@ impl ResourceTracker { net_count: get_net_counter(), } } - pub fn report_leak(&self) { - println!( - "{} outstanding allocations totalling {} bytes.", - get_net_counter(), - get_net_total() - ); - } pub fn get_results(&self) -> ResourceResults { ResourceResults { count_k: (get_counter() - self.count) / 1000, @@ -48,11 +41,22 @@ impl ResourceTracker { } } +// total number of allocations made over the life of the program static COUNTER: AtomicIsize = AtomicIsize::new(0); + +// total number of bytes allocated over the life of the program static TOTAL: AtomicIsize = AtomicIsize::new(0); +// number allocations not yet deallocated static NET_COUNTER: AtomicIsize = AtomicIsize::new(0); + +// number bytes not yet deallocated static NET_TOTAL: AtomicIsize = AtomicIsize::new(0); + +// the peak value reached by NET_TOTAL +// This is reset whenever a ResourceTracker is created +// so it gives the right answer for a single operation +// but it does not handle nested ResourceTrackers correctly. static MAX_NET_TOTAL: AtomicIsize = AtomicIsize::new(0); fn add_to_counter(inc: isize) { From 719de571e2204656e7ff0d291fb12cf580dccf2a Mon Sep 17 00:00:00 2001 From: Andy Jewell Date: Thu, 11 Sep 2025 12:25:01 -0400 Subject: [PATCH 3/3] mpl --- .github/workflows/library_rust_tests.yml | 7 ------- mpl | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/library_rust_tests.yml b/.github/workflows/library_rust_tests.yml index 6ac43916c..34a3b5bf4 100644 --- a/.github/workflows/library_rust_tests.yml +++ b/.github/workflows/library_rust_tests.yml @@ -94,13 +94,6 @@ jobs: CORES=$(node -e 'console.log(os.cpus().length)') make transpile_rust CORES=$CORES - - name: Test Linter for Rust examples and implementation_from_dafny.rs - working-directory: ${{ matrix.library }}/runtimes/rust - shell: bash - run: | - cargo clippy - cargo clippy --example main - - name: Test Rust working-directory: ${{ matrix.library }} shell: bash diff --git a/mpl b/mpl index 2c575232d..7b8d6ac58 160000 --- a/mpl +++ b/mpl @@ -1 +1 @@ -Subproject commit 2c575232dd97fedcbb84232c8fb75e82e8483734 +Subproject commit 7b8d6ac580a3b386e764cf975c063f3e3a2722c2