From 54b18b252b7b9693336253f5b7a9192d6a426822 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Tue, 2 Sep 2025 15:55:14 -0700 Subject: [PATCH 1/4] PerfTests: python --- .../benchmarks/python/README.md | 32 + .../benchmarks/python/benchmark.py | 105 ++ .../benchmarks/python/config.py | 17 + .../benchmarks/python/esdk_benchmark.py | 89 ++ .../benchmarks/python/requirements.txt | 31 + .../benchmarks/python/results.py | 94 ++ .../benchmarks/python/tests.py | 450 ++++++ .../results/raw-data/python_results.json | 1308 +++++++++++++++++ 8 files changed, 2126 insertions(+) create mode 100644 esdk-performance-testing/benchmarks/python/README.md create mode 100644 esdk-performance-testing/benchmarks/python/benchmark.py create mode 100644 esdk-performance-testing/benchmarks/python/config.py create mode 100644 esdk-performance-testing/benchmarks/python/esdk_benchmark.py create mode 100644 esdk-performance-testing/benchmarks/python/requirements.txt create mode 100644 esdk-performance-testing/benchmarks/python/results.py create mode 100644 esdk-performance-testing/benchmarks/python/tests.py create mode 100644 esdk-performance-testing/results/raw-data/python_results.json diff --git a/esdk-performance-testing/benchmarks/python/README.md b/esdk-performance-testing/benchmarks/python/README.md new file mode 100644 index 000000000..5d4c4ba1c --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/README.md @@ -0,0 +1,32 @@ +# AWS Encryption SDK Python Benchmark + +Performance testing suite for the AWS Encryption SDK Python implementation. + +## Quick Start + +```bash +# Install dependencies +pip install -r requirements.txt + +# Run benchmark +python esdk_benchmark.py + +# Quick test (reduced iterations) +python esdk_benchmark.py --quick +``` + +## Options + +- `--config` - Path to test configuration file (default: `../../config/test-scenarios.yaml`) +- `--output` - Path to output results file (default: `../../results/raw-data/python_results.json`) +- `--quick` - Run with reduced iterations for faster testing + +## Test Types + +- **Throughput** - Measures encryption/decryption operations per second +- **Memory** - Tracks memory usage and allocations during operations +- **Concurrency** - Tests performance under concurrent load + +## Output + +Results are saved as JSON with performance metrics including latency, throughput, memory usage, and system information. diff --git a/esdk-performance-testing/benchmarks/python/benchmark.py b/esdk-performance-testing/benchmarks/python/benchmark.py new file mode 100644 index 000000000..c92fd81e9 --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/benchmark.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Core benchmark module for ESDK Python benchmark +""" + +import logging +import multiprocessing +import sys + +import psutil +from config import load_config + +# ESDK imports +try: + from aws_encryption_sdk import EncryptionSDKClient, CommitmentPolicy +except ImportError as e: + print(f"Warning: Could not import ESDK modules: {e}") + print("Please install the AWS Encryption SDK: pip install aws-encryption-sdk") + sys.exit(1) + + +class ESDKBenchmark: + """Main benchmark class for ESDK Python performance testing""" + + def __init__(self, config_path: str = "../../config/test-scenarios.yaml"): + self.config = load_config(config_path) + self.results = [] + + self._setup_logging() + self._setup_esdk() + self._setup_system_info() + + def _setup_system_info(self): + """Initialize system information""" + self.cpu_count = multiprocessing.cpu_count() + self.total_memory_gb = psutil.virtual_memory().total / (1024**3) + + self.logger.info( + f"Initialized ESDK Benchmark - CPU cores: {self.cpu_count}, " + f"Memory: {self.total_memory_gb:.1f}GB" + ) + + def _setup_logging(self): + """Setup logging configuration""" + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + handlers=[logging.StreamHandler(sys.stdout)], + ) + # Suppress AWS SDK logging + logging.getLogger("aws_encryption_sdk").setLevel(logging.WARNING) + logging.getLogger("botocore").setLevel(logging.WARNING) + logging.getLogger("boto3").setLevel(logging.WARNING) + + self.logger = logging.getLogger(__name__) + + def _setup_esdk(self): + """Initialize ESDK client and raw AES keyring""" + try: + self.keyring = self._create_keyring() + self.esdk_client = self._create_client() + self.logger.info("ESDK client initialized successfully") + except Exception as e: + self.logger.error(f"Failed to initialize ESDK: {e}") + raise + + def _create_keyring(self): + """Create raw AES keyring""" + import secrets + from aws_cryptographic_material_providers.mpl import ( + AwsCryptographicMaterialProviders, + ) + from aws_cryptographic_material_providers.mpl.config import ( + MaterialProvidersConfig, + ) + from aws_cryptographic_material_providers.mpl.models import ( + AesWrappingAlg, + CreateRawAesKeyringInput, + ) + + static_key = secrets.token_bytes(32) + mat_prov = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) + + keyring_input = CreateRawAesKeyringInput( + key_namespace="esdk-performance-test", + key_name="test-aes-256-key", + wrapping_key=static_key, + wrapping_alg=AesWrappingAlg.ALG_AES256_GCM_IV12_TAG16, + ) + + return mat_prov.create_raw_aes_keyring(input=keyring_input) + + def _create_client(self): + """Create ESDK client""" + return EncryptionSDKClient( + commitment_policy=CommitmentPolicy.REQUIRE_ENCRYPT_REQUIRE_DECRYPT + ) + + def should_run_test_type(self, test_type: str, is_quick_mode: bool = False) -> bool: + """Determine if a test type should be run based on configuration""" + if is_quick_mode: + quick_config = self.config.get("quick_config") + if quick_config and "test_types" in quick_config: + return test_type in quick_config["test_types"] + return True diff --git a/esdk-performance-testing/benchmarks/python/config.py b/esdk-performance-testing/benchmarks/python/config.py new file mode 100644 index 000000000..6c04d6075 --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/config.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +""" +Configuration module for ESDK Python benchmark +""" + +import yaml + + +def load_config(config_path: str): + """Load test configuration from YAML file""" + try: + with open(config_path, "r") as f: + return yaml.safe_load(f) + except FileNotFoundError: + raise FileNotFoundError(f"Config file not found: {config_path}") + except Exception as e: + raise RuntimeError(f"Failed to parse config file: {e}") diff --git a/esdk-performance-testing/benchmarks/python/esdk_benchmark.py b/esdk-performance-testing/benchmarks/python/esdk_benchmark.py new file mode 100644 index 000000000..ddfa9ad64 --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/esdk_benchmark.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +ESDK Performance Benchmark Suite - Python Implementation + +This module provides comprehensive performance testing for the AWS Encryption SDK (ESDK) +Python runtime, measuring throughput, latency, memory usage, and scalability. +""" + +import sys +import argparse +from benchmark import ESDKBenchmark +from tests import run_all_benchmarks + + +def main(): + """Main entry point for the benchmark suite""" + args = _parse_arguments() + + try: + benchmark = ESDKBenchmark(config_path=args.config) + + if args.quick: + _adjust_config_for_quick_mode(benchmark) + + results = run_all_benchmarks(benchmark, is_quick_mode=args.quick) + + _save_and_summarize_results(results, args.output) + + except Exception as e: + print(f"Benchmark failed: {e}") + sys.exit(1) + + +def _parse_arguments(): + """Parse command line arguments""" + parser = argparse.ArgumentParser(description="ESDK Python Performance Benchmark") + parser.add_argument( + "--config", + default="../../config/test-scenarios.yaml", + help="Path to test configuration file", + ) + parser.add_argument( + "--output", + default="../../results/raw-data/python_results.json", + help="Path to output results file", + ) + parser.add_argument( + "--quick", action="store_true", help="Run quick test with reduced iterations" + ) + return parser.parse_args() + + +def _adjust_config_for_quick_mode(benchmark): + """Adjust benchmark configuration for quick mode""" + quick_config = benchmark.config.get("quick_config") + if not quick_config: + raise RuntimeError( + "Quick mode requested but no quick_config found in config file" + ) + + benchmark.config["iterations"]["measurement"] = quick_config["iterations"][ + "measurement" + ] + benchmark.config["iterations"]["warmup"] = quick_config["iterations"]["warmup"] + benchmark.config["data_sizes"]["small"] = quick_config["data_sizes"]["small"] + benchmark.config["data_sizes"]["medium"] = [] + benchmark.config["data_sizes"]["large"] = [] + benchmark.config["concurrency_levels"] = quick_config["concurrency_levels"] + + +def _save_and_summarize_results(results, output_path): + """Save results and print summary""" + from results import save_results + + save_results(results, output_path) + + print("\n=== ESDK Python Benchmark Summary ===") + print(f"Total tests completed: {len(results)}") + print(f"Results saved to: {output_path}") + + if results: + throughput_results = [r for r in results if r.test_name == "throughput"] + if throughput_results: + max_throughput = max(r.ops_per_second for r in throughput_results) + print("Maximum throughput: {:.2f} ops/sec".format(max_throughput)) + + +if __name__ == "__main__": + main() diff --git a/esdk-performance-testing/benchmarks/python/requirements.txt b/esdk-performance-testing/benchmarks/python/requirements.txt new file mode 100644 index 000000000..d4542cd60 --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/requirements.txt @@ -0,0 +1,31 @@ +# ESDK Performance Testing - Python Dependencies + +# Core dependencies +pyyaml>=6.0 +psutil>=5.9.0 +numpy>=1.24.0 +pandas>=2.0.0 +matplotlib>=3.7.0 +seaborn>=0.12.0 + +# Performance measurement +memory-profiler>=0.61.0 +py-spy>=0.3.14 + +# Statistics and analysis +scipy>=1.10.0 +statsmodels>=0.14.0 + +# Progress and logging +tqdm>=4.65.0 +colorlog>=6.7.0 + +# AWS and ESDK dependencies (public packages) +aws-encryption-sdk>=3.1.0 +boto3>=1.26.0 +botocore>=1.29.0 +cryptography>=41.0.0 + +# Testing and validation +pytest>=7.4.0 +pytest-benchmark>=4.0.0 diff --git a/esdk-performance-testing/benchmarks/python/results.py b/esdk-performance-testing/benchmarks/python/results.py new file mode 100644 index 000000000..e927919c4 --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/results.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Results module for ESDK Python benchmark +""" + +import json +import multiprocessing +import sys +import time +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import List, Optional + +import psutil + + +@dataclass +class BenchmarkResult: + """Container for benchmark results""" + + test_name: str + language: str = "python" + data_size: int = 0 + algorithm_suite: str = "" + frame_length: Optional[int] = None + concurrency: int = 1 + encrypt_latency_ms: float = 0.0 + decrypt_latency_ms: float = 0.0 + end_to_end_latency_ms: float = 0.0 + ops_per_second: float = 0.0 + bytes_per_second: float = 0.0 + peak_memory_mb: float = 0.0 + avg_memory_mb: float = 0.0 + cumulative_allocations_mb: float = 0.0 + memory_efficiency_ratio: float = 0.0 + p50_latency: float = 0.0 + p95_latency: float = 0.0 + p99_latency: float = 0.0 + timestamp: str = "" + python_version: str = "" + cpu_count: int = 0 + total_memory_gb: float = 0.0 + + def __post_init__(self): + self.timestamp = self.timestamp or time.strftime("%Y-%m-%d %H:%M:%S") + self.python_version = self.python_version or self._get_python_version() + self.cpu_count = self.cpu_count or multiprocessing.cpu_count() + self.total_memory_gb = self.total_memory_gb or self._get_total_memory() + + def _get_python_version(self): + """Get Python version string""" + return ( + f"{sys.version_info.major}.{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + + def _get_total_memory(self): + """Get total system memory in GB""" + return psutil.virtual_memory().total / (1024**3) + + +def save_results(results: List[BenchmarkResult], output_path: str): + """Save benchmark results to JSON file""" + output_file = Path(output_path) + output_file.parent.mkdir(parents=True, exist_ok=True) + + metadata = _create_metadata(results) + results_data = { + "metadata": metadata, + "results": [asdict(result) for result in results], + } + + with open(output_file, "w") as f: + json.dump(results_data, f, indent=2) + + +def _create_metadata(results: List[BenchmarkResult]): + """Create metadata for results file""" + metadata = { + "language": "python", + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "total_tests": len(results), + } + + if results: + metadata.update( + { + "python_version": results[0].python_version, + "cpu_count": results[0].cpu_count, + "total_memory_gb": results[0].total_memory_gb, + } + ) + + return metadata diff --git a/esdk-performance-testing/benchmarks/python/tests.py b/esdk-performance-testing/benchmarks/python/tests.py new file mode 100644 index 000000000..8510db3ed --- /dev/null +++ b/esdk-performance-testing/benchmarks/python/tests.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python3 +""" +Test implementations for ESDK Python benchmark +""" + +import os +import statistics +import threading +import time +import tracemalloc +from concurrent.futures import ThreadPoolExecutor, as_completed + +import psutil +from memory_profiler import memory_usage +from tqdm import tqdm + +from results import BenchmarkResult + + +def run_encrypt_decrypt_cycle(esdk_client, keyring, data: bytes) -> tuple[float, float]: + """Run a single encrypt-decrypt cycle and return timing""" + + # Encryption context + encryption_context = {"purpose": "performance-test", "size": str(len(data))} + + # Encrypt + encrypt_start = time.time() + encrypted_result, _ = esdk_client.encrypt( + source=data, keyring=keyring, encryption_context=encryption_context + ) + encrypt_time = (time.time() - encrypt_start) * 1000 + + # Decrypt + decrypt_start = time.time() + decrypted_result, _ = esdk_client.decrypt(source=encrypted_result, keyring=keyring) + decrypt_time = (time.time() - decrypt_start) * 1000 + + # Verify data integrity + if decrypted_result != data: + raise ValueError("Decrypted data does not match original") + + return encrypt_time, decrypt_time + + +def run_throughput_test( + benchmark, + data_size: int, + iterations: int, + algorithm_suite: str = "default", + frame_length: int = None, +) -> BenchmarkResult: + """Run throughput benchmark test""" + data = os.urandom(data_size) + + # Warmup + warmup_iterations = benchmark.config.get("iterations", {}).get("warmup", 2) + for _ in range(warmup_iterations): + run_encrypt_decrypt_cycle(benchmark.esdk_client, benchmark.keyring, data) + + # Collect timing data + timing_data = _collect_timing_data(benchmark, data, iterations) + + # Calculate statistics + return _create_throughput_result( + timing_data, data_size, algorithm_suite, frame_length + ) + + +def _collect_timing_data(benchmark, data, iterations): + """Collect timing data for throughput test""" + encrypt_times = [] + decrypt_times = [] + end_to_end_times = [] + + with tqdm(total=iterations, desc="Throughput test", leave=False) as pbar: + for _ in range(iterations): + start_time = time.time() + encrypt_time, decrypt_time = run_encrypt_decrypt_cycle( + benchmark.esdk_client, benchmark.keyring, data + ) + end_to_end_time = (time.time() - start_time) * 1000 + + encrypt_times.append(encrypt_time) + decrypt_times.append(decrypt_time) + end_to_end_times.append(end_to_end_time) + pbar.update(1) + + return { + "encrypt_times": encrypt_times, + "decrypt_times": decrypt_times, + "end_to_end_times": end_to_end_times, + } + + +def _create_throughput_result(timing_data, data_size, algorithm_suite, frame_length): + """Create throughput benchmark result from timing data""" + avg_encrypt = statistics.mean(timing_data["encrypt_times"]) + avg_decrypt = statistics.mean(timing_data["decrypt_times"]) + avg_end_to_end = statistics.mean(timing_data["end_to_end_times"]) + + ops_per_second = 1000.0 / avg_end_to_end + bytes_per_second = ops_per_second * data_size + + # Calculate percentiles + sorted_times = sorted(timing_data["end_to_end_times"]) + n = len(sorted_times) + p50 = sorted_times[int(0.50 * n)] if n > 0 else 0 + p95 = sorted_times[int(0.95 * n)] if n > 0 else 0 + p99 = sorted_times[int(0.99 * n)] if n > 0 else 0 + + return BenchmarkResult( + test_name="throughput", + data_size=data_size, + algorithm_suite=algorithm_suite, + frame_length=frame_length, + encrypt_latency_ms=avg_encrypt, + decrypt_latency_ms=avg_decrypt, + end_to_end_latency_ms=avg_end_to_end, + ops_per_second=ops_per_second, + bytes_per_second=bytes_per_second, + p50_latency=p50, + p95_latency=p95, + p99_latency=p99, + ) + + +def run_memory_test( + benchmark, data_size: int, algorithm_suite: str = "default" +) -> BenchmarkResult: + """Run memory usage benchmark test""" + data = os.urandom(data_size) + iterations = 5 + + tracemalloc.start() + process = psutil.Process() + + memory_samples = [] + total_allocations = 0 + + def memory_test_function(): + nonlocal total_allocations + iteration_peaks, iteration_avgs, iteration_allocs = [], [], [] + + for i in range(iterations): + peak, avg, allocs = _run_memory_iteration(benchmark, data, process, i) + iteration_peaks.append(peak) + iteration_avgs.append(avg) + iteration_allocs.append(allocs) + total_allocations += allocs * 1024 * 1024 # Convert back to bytes + + _log_memory_summary( + benchmark, iteration_peaks, iteration_avgs, iteration_allocs + ) + return True + + mem_usage = memory_usage(memory_test_function, interval=0.01, timeout=60) + tracemalloc.stop() + + if not mem_usage: + raise RuntimeError("Failed to collect memory usage data") + + return _create_memory_result( + data_size, + algorithm_suite, + mem_usage + memory_samples, + total_allocations, + data_size, + ) + + +def _run_memory_iteration(benchmark, data, process, iteration_num): + """Run a single memory test iteration""" + iteration_samples = [] + snapshot_before = tracemalloc.take_snapshot() + + # Sample memory during operation + stop_sampling = threading.Event() + sampler = threading.Thread( + target=lambda: _sample_memory_continuously( + process, iteration_samples, stop_sampling + ) + ) + sampler.daemon = True + sampler.start() + + start_time = time.time() + run_encrypt_decrypt_cycle(benchmark.esdk_client, benchmark.keyring, data) + end_time = time.time() + + stop_sampling.set() + sampler.join(timeout=1.0) + + snapshot_after = tracemalloc.take_snapshot() + top_stats = snapshot_after.compare_to(snapshot_before, "lineno") + iteration_allocations = sum( + stat.size_diff for stat in top_stats if stat.size_diff > 0 + ) + + if iteration_samples: + peak = max(iteration_samples) + avg = sum(iteration_samples) / len(iteration_samples) + allocs_mb = iteration_allocations / 1024 / 1024 + + duration = end_time - start_time + benchmark.logger.info( + f"=== Iteration {iteration_num + 1} === Peak: {peak:.2f} MB, " + f"Allocs: {allocs_mb:.2f} MB, Avg: {avg:.2f} MB " + f"({duration:.3f}s, {len(iteration_samples)} samples)" + ) + return peak, avg, allocs_mb + + return 0, 0, 0 + + +def _sample_memory_continuously(process, samples, stop_event): + """Continuously sample memory usage""" + while not stop_event.is_set(): + current_memory = process.memory_info().rss / 1024 / 1024 + samples.append(current_memory) + time.sleep(0.001) + + +def _log_memory_summary(benchmark, peaks, avgs, allocs): + """Log memory test summary""" + if peaks and avgs and allocs: + abs_peak = max(peaks) + overall_avg = sum(avgs) / len(avgs) + max_allocs = max(allocs) + + benchmark.logger.info("") + benchmark.logger.info("Memory Summary:") + benchmark.logger.info(f"- Absolute Peak Heap: {abs_peak:.2f} MB") + benchmark.logger.info(f"- Average Heap: {overall_avg:.2f} MB") + benchmark.logger.info(f"- Total Allocations: {max_allocs:.2f} MB") + + +def _create_memory_result( + data_size, algorithm_suite, all_samples, total_allocations, original_data_size +): + """Create memory benchmark result""" + peak_memory_mb = max(all_samples) + avg_memory_mb = sum(all_samples) / len(all_samples) + cumulative_allocations_mb = total_allocations / 1024 / 1024 + memory_efficiency = ( + original_data_size / (peak_memory_mb * 1024 * 1024) if peak_memory_mb > 0 else 0 + ) + + return BenchmarkResult( + test_name="memory", + data_size=data_size, + algorithm_suite=algorithm_suite, + peak_memory_mb=peak_memory_mb, + avg_memory_mb=avg_memory_mb, + cumulative_allocations_mb=cumulative_allocations_mb, + memory_efficiency_ratio=memory_efficiency, + ) + + +def run_concurrent_test(benchmark, data_size: int, concurrency: int) -> BenchmarkResult: + """Run concurrent benchmark test""" + data = os.urandom(data_size) + operations_per_worker = 5 + total_operations = concurrency * operations_per_worker + + start_time = time.time() + all_times, errors = _execute_concurrent_workers( + benchmark, data, concurrency, operations_per_worker + ) + total_duration = time.time() - start_time + + if errors: + raise RuntimeError( + f"Concurrent test failed with {len(errors)} errors: {errors[0]}" + ) + + if not all_times: + raise RuntimeError("No timing data collected from concurrent test") + + return _create_concurrent_result( + all_times, total_operations, total_duration, data_size, concurrency + ) + + +def _execute_concurrent_workers(benchmark, data, concurrency, operations_per_worker): + """Execute concurrent workers and collect results""" + all_times = [] + errors = [] + + def worker_function(): + worker_times = [] + try: + for _ in range(operations_per_worker): + start_time = time.time() + run_encrypt_decrypt_cycle( + benchmark.esdk_client, benchmark.keyring, data + ) + operation_time = (time.time() - start_time) * 1000 + worker_times.append(operation_time) + except Exception as e: + errors.append(e) + return worker_times + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(worker_function) for _ in range(concurrency)] + + for future in as_completed(futures): + try: + worker_times = future.result() + all_times.extend(worker_times) + except Exception as e: + errors.append(e) + + return all_times, errors + + +def _create_concurrent_result( + all_times, total_operations, total_duration, data_size, concurrency +): + """Create concurrent benchmark result""" + avg_latency = statistics.mean(all_times) + ops_per_second = total_operations / total_duration + bytes_per_second = ops_per_second * data_size + + return BenchmarkResult( + test_name="concurrent", + data_size=data_size, + concurrency=concurrency, + end_to_end_latency_ms=avg_latency, + ops_per_second=ops_per_second, + bytes_per_second=bytes_per_second, + ) + + +def run_all_benchmarks(benchmark, is_quick_mode: bool = False) -> list[BenchmarkResult]: + """Run all configured benchmark tests""" + benchmark.logger.info("Starting comprehensive ESDK benchmark suite") + results = [] + + test_params = _get_test_parameters(benchmark.config) + total_tests = _calculate_total_tests(test_params) + + with tqdm(total=total_tests, desc="Running benchmarks") as pbar: + if benchmark.should_run_test_type("throughput", is_quick_mode): + _run_throughput_tests(benchmark, test_params, results, pbar) + else: + benchmark.logger.info("Skipping throughput tests (not in test_types)") + + if benchmark.should_run_test_type("memory", is_quick_mode): + _run_memory_tests(benchmark, test_params, results, pbar) + else: + benchmark.logger.info("Skipping memory tests (not in test_types)") + + if benchmark.should_run_test_type("concurrency", is_quick_mode): + _run_concurrent_tests(benchmark, test_params, results, pbar) + else: + benchmark.logger.info("Skipping concurrency tests (not in test_types)") + + benchmark.results = results + benchmark.logger.info(f"Benchmark suite completed. Total results: {len(results)}") + return results + + +def _get_test_parameters(config): + """Extract test parameters from config""" + data_sizes = [] + for category in ["small", "medium", "large"]: + if category in config.get("data_sizes", {}): + data_sizes.extend(config["data_sizes"][category]) + + return { + "data_sizes": data_sizes, + "algorithm_suites": config.get("algorithm_suites", ["default"]), + "frame_lengths": config.get("frame_lengths", [None]), + "concurrency_levels": config.get("concurrency_levels", [1, 2, 4]), + "iterations": config.get("iterations", {}).get("measurement", 10), + } + + +def _calculate_total_tests(params): + """Calculate total number of tests to run""" + return ( + len(params["data_sizes"]) + * len(params["algorithm_suites"]) + * (len(params["frame_lengths"]) + len(params["concurrency_levels"]) + 1) + ) + + +def _run_throughput_tests(benchmark, params, results, pbar): + """Run all throughput tests""" + for data_size in params["data_sizes"]: + for algorithm_suite in params["algorithm_suites"]: + for frame_length in params["frame_lengths"]: + try: + benchmark.logger.info( + f"Running throughput test - Size: {data_size} bytes, " + f"Iterations: {params['iterations']}" + ) + result = run_throughput_test( + benchmark, + data_size, + params["iterations"], + algorithm_suite, + frame_length, + ) + results.append(result) + benchmark.logger.info( + f"Throughput test completed: " + f"{result.ops_per_second:.2f} ops/sec" + ) + except Exception as e: + benchmark.logger.error(f"Throughput test failed: {e}") + pbar.update(1) + + +def _run_memory_tests(benchmark, params, results, pbar): + """Run all memory tests""" + for data_size in params["data_sizes"]: + for algorithm_suite in params["algorithm_suites"]: + try: + benchmark.logger.info(f"Running memory test - Size: {data_size} bytes") + result = run_memory_test(benchmark, data_size, algorithm_suite) + results.append(result) + benchmark.logger.info( + f"Memory test completed: {result.peak_memory_mb:.2f} MB peak" + ) + except Exception as e: + benchmark.logger.error(f"Memory test failed: {e}") + pbar.update(1) + + +def _run_concurrent_tests(benchmark, params, results, pbar): + """Run all concurrent tests""" + for data_size in params["data_sizes"]: + for concurrency in params["concurrency_levels"]: + if concurrency > 1: + try: + benchmark.logger.info( + f"Running concurrent test - Size: {data_size} bytes, " + f"Concurrency: {concurrency}" + ) + result = run_concurrent_test(benchmark, data_size, concurrency) + results.append(result) + benchmark.logger.info( + f"Concurrent test completed: " + f"{result.ops_per_second:.2f} ops/sec " + f"@ {concurrency} threads" + ) + except Exception as e: + benchmark.logger.error(f"Concurrent test failed: {e}") + pbar.update(1) diff --git a/esdk-performance-testing/results/raw-data/python_results.json b/esdk-performance-testing/results/raw-data/python_results.json new file mode 100644 index 000000000..0cf50f4f5 --- /dev/null +++ b/esdk-performance-testing/results/raw-data/python_results.json @@ -0,0 +1,1308 @@ +{ + "metadata": { + "language": "python", + "timestamp": "2025-09-02 17:01:18", + "total_tests": 54, + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + "results": [ + { + "test_name": "throughput", + "language": "python", + "data_size": 1024, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 1.9099712371826172, + "decrypt_latency_ms": 4.344916343688965, + "end_to_end_latency_ms": 6.2569379806518555, + "ops_per_second": 159.82258464000608, + "bytes_per_second": 163658.32667136623, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 4.068851470947266, + "p95_latency": 25.39801597595215, + "p99_latency": 25.39801597595215, + "timestamp": "2025-09-02 16:57:30", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 5120, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 1.7065763473510742, + "decrypt_latency_ms": 2.078890800476074, + "end_to_end_latency_ms": 3.7878036499023438, + "ops_per_second": 264.00523692028804, + "bytes_per_second": 1351706.8130318748, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 3.8411617279052734, + "p95_latency": 4.286050796508789, + "p99_latency": 4.286050796508789, + "timestamp": "2025-09-02 16:57:31", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 10240, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 1.7570734024047852, + "decrypt_latency_ms": 1.9857168197631836, + "end_to_end_latency_ms": 3.7441492080688477, + "ops_per_second": 267.0833731318573, + "bytes_per_second": 2734933.7408702187, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 3.62396240234375, + "p95_latency": 4.662036895751953, + "p99_latency": 4.662036895751953, + "timestamp": "2025-09-02 16:57:31", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 102400, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 2.0096540451049805, + "decrypt_latency_ms": 2.328824996948242, + "end_to_end_latency_ms": 4.342937469482422, + "ops_per_second": 230.2588989657217, + "bytes_per_second": 23578511.254089903, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 4.28318977355957, + "p95_latency": 5.136013031005859, + "p99_latency": 5.136013031005859, + "timestamp": "2025-09-02 16:57:31", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 512000, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 3.1792640686035156, + "decrypt_latency_ms": 3.606557846069336, + "end_to_end_latency_ms": 6.798648834228516, + "ops_per_second": 147.08805004979732, + "bytes_per_second": 75309081.62549622, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 6.873130798339844, + "p95_latency": 7.0400238037109375, + "p99_latency": 7.0400238037109375, + "timestamp": "2025-09-02 16:57:31", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 4.8476457595825195, + "decrypt_latency_ms": 5.4534912109375, + "end_to_end_latency_ms": 10.32414436340332, + "ops_per_second": 96.86032709352325, + "bytes_per_second": 101565414.34241824, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 10.346174240112305, + "p95_latency": 10.860681533813477, + "p99_latency": 10.860681533813477, + "timestamp": "2025-09-02 16:57:31", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 35.567426681518555, + "decrypt_latency_ms": 38.695549964904785, + "end_to_end_latency_ms": 74.97415542602539, + "ops_per_second": 13.337929508077861, + "bytes_per_second": 139858327.7186225, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 74.99313354492188, + "p95_latency": 77.53705978393555, + "p99_latency": 77.53705978393555, + "timestamp": "2025-09-02 16:57:32", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 167.90552139282227, + "decrypt_latency_ms": 182.42778778076172, + "end_to_end_latency_ms": 353.8800239562988, + "ops_per_second": 2.8258164697182555, + "bytes_per_second": 148154166.52756447, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 354.4912338256836, + "p95_latency": 356.77409172058105, + "p99_latency": 356.77409172058105, + "timestamp": "2025-09-02 16:57:38", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "throughput", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 332.6861381530762, + "decrypt_latency_ms": 362.6670837402344, + "end_to_end_latency_ms": 703.7388801574707, + "ops_per_second": 1.4209816001302031, + "bytes_per_second": 149000720.23381278, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 703.5648822784424, + "p95_latency": 713.519811630249, + "p99_latency": 713.519811630249, + "timestamp": "2025-09-02 16:57:49", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 1024, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.234375, + "avg_memory_mb": 334.84517045454544, + "cumulative_allocations_mb": 0.3372917175292969, + "memory_efficiency_ratio": 2.913073875553484e-06, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:49", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 5120, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.25, + "avg_memory_mb": 335.2421875, + "cumulative_allocations_mb": 0.34973716735839844, + "memory_efficiency_ratio": 1.456469052945563e-05, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:50", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 10240, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.390625, + "avg_memory_mb": 335.3390625, + "cumulative_allocations_mb": 0.3968391418457031, + "memory_efficiency_ratio": 2.9117167481947357e-05, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:51", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 102400, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.390625, + "avg_memory_mb": 335.3792613636364, + "cumulative_allocations_mb": 1.2789506912231445, + "memory_efficiency_ratio": 0.0002911716748194736, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:52", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 512000, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.390625, + "avg_memory_mb": 335.38169642857144, + "cumulative_allocations_mb": 5.2001848220825195, + "memory_efficiency_ratio": 0.0014558583740973679, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:52", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 335.421875, + "avg_memory_mb": 335.3988486842105, + "cumulative_allocations_mb": 10.33751392364502, + "memory_efficiency_ratio": 0.0029813201658359344, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:53", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 355.46875, + "avg_memory_mb": 348.4203125, + "cumulative_allocations_mb": 100.69142055511475, + "memory_efficiency_ratio": 0.028131868131868132, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:57:55", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 448.71875, + "avg_memory_mb": 385.78617294520546, + "cumulative_allocations_mb": 502.26827335357666, + "memory_efficiency_ratio": 0.11142837244933491, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:02", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "memory", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "default", + "frame_length": null, + "concurrency": 1, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 0.0, + "ops_per_second": 0.0, + "bytes_per_second": 0.0, + "peak_memory_mb": 549.1875, + "avg_memory_mb": 411.4255622739602, + "cumulative_allocations_mb": 1004.2456932067871, + "memory_efficiency_ratio": 0.18208717423466483, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1024, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.577303886413574, + "ops_per_second": 277.5240351213832, + "bytes_per_second": 284184.6119642964, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1024, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.268360137939453, + "ops_per_second": 290.45722575976345, + "bytes_per_second": 297428.19917799777, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1024, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.7978172302246094, + "ops_per_second": 296.1817636155, + "bytes_per_second": 303290.125942272, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1024, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.565165400505066, + "ops_per_second": 297.53642233773326, + "bytes_per_second": 304677.29647383885, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 5120, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.389596939086914, + "ops_per_second": 293.807238874451, + "bytes_per_second": 1504293.0630371892, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:16", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 5120, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.360271453857422, + "ops_per_second": 296.4465742193574, + "bytes_per_second": 1517806.4600031099, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 5120, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.3717453479766846, + "ops_per_second": 295.63063650428364, + "bytes_per_second": 1513628.8589019324, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 5120, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.21522855758667, + "ops_per_second": 296.52385619211145, + "bytes_per_second": 1518202.1437036106, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10240, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.4344911575317383, + "ops_per_second": 289.896117719428, + "bytes_per_second": 2968536.2454469427, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10240, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.277920722961426, + "ops_per_second": 290.53669890000276, + "bytes_per_second": 2975095.796736028, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10240, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 3.4348905086517334, + "ops_per_second": 289.4419984818163, + "bytes_per_second": 2963886.0644537993, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10240, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.671064019203186, + "ops_per_second": 277.0389561923445, + "bytes_per_second": 2836878.911409607, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:17", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 102400, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.224300384521484, + "ops_per_second": 235.97177995566707, + "bytes_per_second": 24163510.26746031, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 102400, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.057180881500244, + "ops_per_second": 245.76024000023438, + "bytes_per_second": 25165848.576024, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 102400, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 5.054503679275513, + "ops_per_second": 246.21395698318776, + "bytes_per_second": 25212309.195078425, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 102400, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 4.062226414680481, + "ops_per_second": 245.5199946146394, + "bytes_per_second": 25141247.448539075, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 512000, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 11.237502098083496, + "ops_per_second": 142.19859574655632, + "bytes_per_second": 72805681.02223684, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 512000, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 17.5534725189209, + "ops_per_second": 144.90474257434295, + "bytes_per_second": 74191228.19806358, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 512000, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 39.74910378456116, + "ops_per_second": 130.24484349005417, + "bytes_per_second": 66685359.86690773, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:19", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 512000, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 31.3551664352417, + "ops_per_second": 146.77259726063588, + "bytes_per_second": 75147569.79744557, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:19", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 17.444610595703125, + "ops_per_second": 96.39396857425865, + "bytes_per_second": 101076401.99172184, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:19", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 31.107473373413086, + "ops_per_second": 95.48046366359803, + "bytes_per_second": 100118522.66652097, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:20", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 58.18980932235718, + "ops_per_second": 90.38834648078848, + "bytes_per_second": 94779050.79943927, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:20", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 1048576, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 56.39534294605255, + "ops_per_second": 95.7876199363803, + "bytes_per_second": 100440599.3624099, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:21", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 150.94387531280518, + "ops_per_second": 13.030402067812828, + "bytes_per_second": 136633668.78658903, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:22", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 296.8547224998474, + "ops_per_second": 13.011930305905558, + "bytes_per_second": 136439978.32445228, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:23", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 571.5517699718475, + "ops_per_second": 13.032108583090837, + "bytes_per_second": 136651562.89623058, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:26", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 10485760, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 1118.0467426776886, + "ops_per_second": 12.357680604362868, + "bytes_per_second": 129579672.97400399, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:33", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 700.9313821792603, + "ops_per_second": 2.8261352066857444, + "bytes_per_second": 148170877.52428555, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:37", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 1404.3150186538696, + "ops_per_second": 2.796435143641608, + "bytes_per_second": 146613738.85895714, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:44", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 2820.2352464199066, + "ops_per_second": 2.745385669438221, + "bytes_per_second": 143937276.1858426, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:58:59", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 52428800, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 5537.272220849991, + "ops_per_second": 2.7413794859986993, + "bytes_per_second": 143727236.7955286, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:59:28", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 2, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 1410.4030847549438, + "ops_per_second": 1.4100913399973307, + "bytes_per_second": 147858793.6929041, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:59:36", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 4, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 2806.2445402145386, + "ops_per_second": 1.4072359363354092, + "bytes_per_second": 147559382.9178838, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 16:59:50", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 8, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 5545.182865858078, + "ops_per_second": 1.403170790617305, + "bytes_per_second": 147133121.49423313, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 17:00:19", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + }, + { + "test_name": "concurrent", + "language": "python", + "data_size": 104857600, + "algorithm_suite": "", + "frame_length": null, + "concurrency": 16, + "encrypt_latency_ms": 0.0, + "decrypt_latency_ms": 0.0, + "end_to_end_latency_ms": 11181.330859661102, + "ops_per_second": 1.3749360047730161, + "bytes_per_second": 144172489.61408702, + "peak_memory_mb": 0.0, + "avg_memory_mb": 0.0, + "cumulative_allocations_mb": 0.0, + "memory_efficiency_ratio": 0.0, + "p50_latency": 0.0, + "p95_latency": 0.0, + "p99_latency": 0.0, + "timestamp": "2025-09-02 17:01:18", + "python_version": "3.12.7", + "cpu_count": 12, + "total_memory_gb": 36.0 + } + ] +} \ No newline at end of file From f736ae605f52df42b58f7793a0d7d98a48946ceb Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Wed, 3 Sep 2025 09:29:45 -0700 Subject: [PATCH 2/4] fix: update esdk dependency Co-authored-by: Lucas McDonald --- esdk-performance-testing/benchmarks/python/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esdk-performance-testing/benchmarks/python/requirements.txt b/esdk-performance-testing/benchmarks/python/requirements.txt index d4542cd60..f78f9ccd0 100644 --- a/esdk-performance-testing/benchmarks/python/requirements.txt +++ b/esdk-performance-testing/benchmarks/python/requirements.txt @@ -21,7 +21,8 @@ tqdm>=4.65.0 colorlog>=6.7.0 # AWS and ESDK dependencies (public packages) -aws-encryption-sdk>=3.1.0 +aws-encryption-sdk>=4.0.1 +aws-cryptographic-material-providers>=1.11.0 boto3>=1.26.0 botocore>=1.29.0 cryptography>=41.0.0 From 9dee36129da82a2ad82b9631e99ab3f138cba03c Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Wed, 3 Sep 2025 13:41:19 -0700 Subject: [PATCH 3/4] fix: PR comments --- .../benchmarks/python/README.md | 13 +- .../benchmarks/python/benchmark.py | 25 +- .../benchmarks/python/requirements.txt | 19 +- .../benchmarks/python/results.py | 4 - .../benchmarks/python/tests.py | 84 +- .../results/raw-data/python_results.json | 724 ++++++------------ 6 files changed, 307 insertions(+), 562 deletions(-) diff --git a/esdk-performance-testing/benchmarks/python/README.md b/esdk-performance-testing/benchmarks/python/README.md index 5d4c4ba1c..6541077a7 100644 --- a/esdk-performance-testing/benchmarks/python/README.md +++ b/esdk-performance-testing/benchmarks/python/README.md @@ -21,6 +21,13 @@ python esdk_benchmark.py --quick - `--output` - Path to output results file (default: `../../results/raw-data/python_results.json`) - `--quick` - Run with reduced iterations for faster testing +## Configuration + +Edit `../../config/test-scenarios.yaml` for test parameters: + +- Data sizes (small/medium/large) +- Iterations and concurrency levels + ## Test Types - **Throughput** - Measures encryption/decryption operations per second @@ -29,4 +36,8 @@ python esdk_benchmark.py --quick ## Output -Results are saved as JSON with performance metrics including latency, throughput, memory usage, and system information. +Results saved as JSON to `../../results/raw-data/python_results.json` with: + +- Performance metrics (ops/sec, latency percentiles) +- Memory usage (peak, average, allocations, input data to memory ratio) +- System information (CPU, memory, Python version) diff --git a/esdk-performance-testing/benchmarks/python/benchmark.py b/esdk-performance-testing/benchmarks/python/benchmark.py index c92fd81e9..70986c235 100644 --- a/esdk-performance-testing/benchmarks/python/benchmark.py +++ b/esdk-performance-testing/benchmarks/python/benchmark.py @@ -5,19 +5,16 @@ import logging import multiprocessing +import secrets import sys import psutil +from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders +from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig +from aws_cryptographic_material_providers.mpl.models import AesWrappingAlg, CreateRawAesKeyringInput +from aws_encryption_sdk import EncryptionSDKClient, CommitmentPolicy from config import load_config -# ESDK imports -try: - from aws_encryption_sdk import EncryptionSDKClient, CommitmentPolicy -except ImportError as e: - print(f"Warning: Could not import ESDK modules: {e}") - print("Please install the AWS Encryption SDK: pip install aws-encryption-sdk") - sys.exit(1) - class ESDKBenchmark: """Main benchmark class for ESDK Python performance testing""" @@ -66,18 +63,6 @@ def _setup_esdk(self): def _create_keyring(self): """Create raw AES keyring""" - import secrets - from aws_cryptographic_material_providers.mpl import ( - AwsCryptographicMaterialProviders, - ) - from aws_cryptographic_material_providers.mpl.config import ( - MaterialProvidersConfig, - ) - from aws_cryptographic_material_providers.mpl.models import ( - AesWrappingAlg, - CreateRawAesKeyringInput, - ) - static_key = secrets.token_bytes(32) mat_prov = AwsCryptographicMaterialProviders(config=MaterialProvidersConfig()) diff --git a/esdk-performance-testing/benchmarks/python/requirements.txt b/esdk-performance-testing/benchmarks/python/requirements.txt index f78f9ccd0..9c310faa8 100644 --- a/esdk-performance-testing/benchmarks/python/requirements.txt +++ b/esdk-performance-testing/benchmarks/python/requirements.txt @@ -3,30 +3,13 @@ # Core dependencies pyyaml>=6.0 psutil>=5.9.0 -numpy>=1.24.0 -pandas>=2.0.0 -matplotlib>=3.7.0 -seaborn>=0.12.0 # Performance measurement memory-profiler>=0.61.0 -py-spy>=0.3.14 - -# Statistics and analysis -scipy>=1.10.0 -statsmodels>=0.14.0 # Progress and logging tqdm>=4.65.0 -colorlog>=6.7.0 -# AWS and ESDK dependencies (public packages) +# AWS and ESDK dependencies aws-encryption-sdk>=4.0.1 aws-cryptographic-material-providers>=1.11.0 -boto3>=1.26.0 -botocore>=1.29.0 -cryptography>=41.0.0 - -# Testing and validation -pytest>=7.4.0 -pytest-benchmark>=4.0.0 diff --git a/esdk-performance-testing/benchmarks/python/results.py b/esdk-performance-testing/benchmarks/python/results.py index e927919c4..ff670d5b9 100644 --- a/esdk-performance-testing/benchmarks/python/results.py +++ b/esdk-performance-testing/benchmarks/python/results.py @@ -21,8 +21,6 @@ class BenchmarkResult: test_name: str language: str = "python" data_size: int = 0 - algorithm_suite: str = "" - frame_length: Optional[int] = None concurrency: int = 1 encrypt_latency_ms: float = 0.0 decrypt_latency_ms: float = 0.0 @@ -30,8 +28,6 @@ class BenchmarkResult: ops_per_second: float = 0.0 bytes_per_second: float = 0.0 peak_memory_mb: float = 0.0 - avg_memory_mb: float = 0.0 - cumulative_allocations_mb: float = 0.0 memory_efficiency_ratio: float = 0.0 p50_latency: float = 0.0 p95_latency: float = 0.0 diff --git a/esdk-performance-testing/benchmarks/python/tests.py b/esdk-performance-testing/benchmarks/python/tests.py index 8510db3ed..2a4f54eab 100644 --- a/esdk-performance-testing/benchmarks/python/tests.py +++ b/esdk-performance-testing/benchmarks/python/tests.py @@ -3,6 +3,7 @@ Test implementations for ESDK Python benchmark """ +import gc import os import statistics import threading @@ -46,8 +47,6 @@ def run_throughput_test( benchmark, data_size: int, iterations: int, - algorithm_suite: str = "default", - frame_length: int = None, ) -> BenchmarkResult: """Run throughput benchmark test""" data = os.urandom(data_size) @@ -62,7 +61,7 @@ def run_throughput_test( # Calculate statistics return _create_throughput_result( - timing_data, data_size, algorithm_suite, frame_length + timing_data, data_size ) @@ -92,7 +91,7 @@ def _collect_timing_data(benchmark, data, iterations): } -def _create_throughput_result(timing_data, data_size, algorithm_suite, frame_length): +def _create_throughput_result(timing_data, data_size): """Create throughput benchmark result from timing data""" avg_encrypt = statistics.mean(timing_data["encrypt_times"]) avg_decrypt = statistics.mean(timing_data["decrypt_times"]) @@ -111,8 +110,6 @@ def _create_throughput_result(timing_data, data_size, algorithm_suite, frame_len return BenchmarkResult( test_name="throughput", data_size=data_size, - algorithm_suite=algorithm_suite, - frame_length=frame_length, encrypt_latency_ms=avg_encrypt, decrypt_latency_ms=avg_decrypt, end_to_end_latency_ms=avg_end_to_end, @@ -125,7 +122,7 @@ def _create_throughput_result(timing_data, data_size, algorithm_suite, frame_len def run_memory_test( - benchmark, data_size: int, algorithm_suite: str = "default" + benchmark, data_size: int ) -> BenchmarkResult: """Run memory usage benchmark test""" data = os.urandom(data_size) @@ -161,7 +158,6 @@ def memory_test_function(): return _create_memory_result( data_size, - algorithm_suite, mem_usage + memory_samples, total_allocations, data_size, @@ -235,7 +231,7 @@ def _log_memory_summary(benchmark, peaks, avgs, allocs): def _create_memory_result( - data_size, algorithm_suite, all_samples, total_allocations, original_data_size + data_size, all_samples, total_allocations, original_data_size ): """Create memory benchmark result""" peak_memory_mb = max(all_samples) @@ -248,10 +244,7 @@ def _create_memory_result( return BenchmarkResult( test_name="memory", data_size=data_size, - algorithm_suite=algorithm_suite, peak_memory_mb=peak_memory_mb, - avg_memory_mb=avg_memory_mb, - cumulative_allocations_mb=cumulative_allocations_mb, memory_efficiency_ratio=memory_efficiency, ) @@ -369,8 +362,6 @@ def _get_test_parameters(config): return { "data_sizes": data_sizes, - "algorithm_suites": config.get("algorithm_suites", ["default"]), - "frame_lengths": config.get("frame_lengths", [None]), "concurrency_levels": config.get("concurrency_levels", [1, 2, 4]), "iterations": config.get("iterations", {}).get("measurement", 10), } @@ -380,52 +371,47 @@ def _calculate_total_tests(params): """Calculate total number of tests to run""" return ( len(params["data_sizes"]) - * len(params["algorithm_suites"]) - * (len(params["frame_lengths"]) + len(params["concurrency_levels"]) + 1) + * (1 + len(params["concurrency_levels"]) + 1) ) def _run_throughput_tests(benchmark, params, results, pbar): """Run all throughput tests""" for data_size in params["data_sizes"]: - for algorithm_suite in params["algorithm_suites"]: - for frame_length in params["frame_lengths"]: - try: - benchmark.logger.info( - f"Running throughput test - Size: {data_size} bytes, " - f"Iterations: {params['iterations']}" - ) - result = run_throughput_test( - benchmark, - data_size, - params["iterations"], - algorithm_suite, - frame_length, - ) - results.append(result) - benchmark.logger.info( - f"Throughput test completed: " - f"{result.ops_per_second:.2f} ops/sec" - ) - except Exception as e: - benchmark.logger.error(f"Throughput test failed: {e}") - pbar.update(1) + try: + benchmark.logger.info( + f"Running throughput test - Size: {data_size} bytes, " + f"Iterations: {params['iterations']}" + ) + result = run_throughput_test( + benchmark, + data_size, + params["iterations"], + ) + results.append(result) + benchmark.logger.info( + f"Throughput test completed: " + f"{result.ops_per_second:.2f} ops/sec" + ) + except Exception as e: + benchmark.logger.error(f"Throughput test failed: {e}") + pbar.update(1) def _run_memory_tests(benchmark, params, results, pbar): """Run all memory tests""" for data_size in params["data_sizes"]: - for algorithm_suite in params["algorithm_suites"]: - try: - benchmark.logger.info(f"Running memory test - Size: {data_size} bytes") - result = run_memory_test(benchmark, data_size, algorithm_suite) - results.append(result) - benchmark.logger.info( - f"Memory test completed: {result.peak_memory_mb:.2f} MB peak" - ) - except Exception as e: - benchmark.logger.error(f"Memory test failed: {e}") - pbar.update(1) + try: + benchmark.logger.info(f"Running memory test - Size: {data_size} bytes") + gc.collect() + result = run_memory_test(benchmark, data_size) + results.append(result) + benchmark.logger.info( + f"Memory test completed: {result.peak_memory_mb:.2f} MB peak" + ) + except Exception as e: + benchmark.logger.error(f"Memory test failed: {e}") + pbar.update(1) def _run_concurrent_tests(benchmark, params, results, pbar): diff --git a/esdk-performance-testing/results/raw-data/python_results.json b/esdk-performance-testing/results/raw-data/python_results.json index 0cf50f4f5..816f3e181 100644 --- a/esdk-performance-testing/results/raw-data/python_results.json +++ b/esdk-performance-testing/results/raw-data/python_results.json @@ -1,7 +1,7 @@ { "metadata": { "language": "python", - "timestamp": "2025-09-02 17:01:18", + "timestamp": "2025-09-03 13:34:50", "total_tests": 54, "python_version": "3.12.7", "cpu_count": 12, @@ -12,22 +12,18 @@ "test_name": "throughput", "language": "python", "data_size": 1024, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 1.9099712371826172, - "decrypt_latency_ms": 4.344916343688965, - "end_to_end_latency_ms": 6.2569379806518555, - "ops_per_second": 159.82258464000608, - "bytes_per_second": 163658.32667136623, + "encrypt_latency_ms": 1.8790721893310547, + "decrypt_latency_ms": 1.9063234329223633, + "end_to_end_latency_ms": 3.7869691848754883, + "ops_per_second": 264.063410918111, + "bytes_per_second": 270400.93278014567, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 4.068851470947266, - "p95_latency": 25.39801597595215, - "p99_latency": 25.39801597595215, - "timestamp": "2025-09-02 16:57:30", + "p50_latency": 3.789663314819336, + "p95_latency": 4.518985748291016, + "p99_latency": 4.518985748291016, + "timestamp": "2025-09-03 13:30:59", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -36,22 +32,18 @@ "test_name": "throughput", "language": "python", "data_size": 5120, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 1.7065763473510742, - "decrypt_latency_ms": 2.078890800476074, - "end_to_end_latency_ms": 3.7878036499023438, - "ops_per_second": 264.00523692028804, - "bytes_per_second": 1351706.8130318748, + "encrypt_latency_ms": 1.6942501068115234, + "decrypt_latency_ms": 1.7327547073364258, + "end_to_end_latency_ms": 3.428959846496582, + "ops_per_second": 291.63362791247454, + "bytes_per_second": 1493164.1749118697, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 3.8411617279052734, - "p95_latency": 4.286050796508789, - "p99_latency": 4.286050796508789, - "timestamp": "2025-09-02 16:57:31", + "p50_latency": 3.4220218658447266, + "p95_latency": 3.831624984741211, + "p99_latency": 3.831624984741211, + "timestamp": "2025-09-03 13:30:59", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -60,22 +52,18 @@ "test_name": "throughput", "language": "python", "data_size": 10240, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 1.7570734024047852, - "decrypt_latency_ms": 1.9857168197631836, - "end_to_end_latency_ms": 3.7441492080688477, - "ops_per_second": 267.0833731318573, - "bytes_per_second": 2734933.7408702187, + "encrypt_latency_ms": 1.6153335571289062, + "decrypt_latency_ms": 1.754903793334961, + "end_to_end_latency_ms": 3.3715248107910156, + "ops_per_second": 296.60170282578565, + "bytes_per_second": 3037201.436936045, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 3.62396240234375, - "p95_latency": 4.662036895751953, - "p99_latency": 4.662036895751953, - "timestamp": "2025-09-02 16:57:31", + "p50_latency": 3.4189224243164062, + "p95_latency": 3.6020278930664062, + "p99_latency": 3.6020278930664062, + "timestamp": "2025-09-03 13:30:59", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -84,22 +72,18 @@ "test_name": "throughput", "language": "python", "data_size": 102400, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 2.0096540451049805, - "decrypt_latency_ms": 2.328824996948242, - "end_to_end_latency_ms": 4.342937469482422, - "ops_per_second": 230.2588989657217, - "bytes_per_second": 23578511.254089903, + "encrypt_latency_ms": 1.9028186798095703, + "decrypt_latency_ms": 2.0013809204101562, + "end_to_end_latency_ms": 3.908061981201172, + "ops_per_second": 255.8813050586886, + "bytes_per_second": 26202245.638009712, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 4.28318977355957, - "p95_latency": 5.136013031005859, - "p99_latency": 5.136013031005859, - "timestamp": "2025-09-02 16:57:31", + "p50_latency": 3.880023956298828, + "p95_latency": 4.33802604675293, + "p99_latency": 4.33802604675293, + "timestamp": "2025-09-03 13:31:00", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -108,22 +92,18 @@ "test_name": "throughput", "language": "python", "data_size": 512000, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 3.1792640686035156, - "decrypt_latency_ms": 3.606557846069336, - "end_to_end_latency_ms": 6.798648834228516, - "ops_per_second": 147.08805004979732, - "bytes_per_second": 75309081.62549622, + "encrypt_latency_ms": 2.991819381713867, + "decrypt_latency_ms": 3.244638442993164, + "end_to_end_latency_ms": 6.2480926513671875, + "ops_per_second": 160.04884303071006, + "bytes_per_second": 81945007.63172355, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 6.873130798339844, - "p95_latency": 7.0400238037109375, - "p99_latency": 7.0400238037109375, - "timestamp": "2025-09-02 16:57:31", + "p50_latency": 6.187915802001953, + "p95_latency": 6.437063217163086, + "p99_latency": 6.437063217163086, + "timestamp": "2025-09-03 13:31:00", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -132,22 +112,18 @@ "test_name": "throughput", "language": "python", "data_size": 1048576, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 4.8476457595825195, - "decrypt_latency_ms": 5.4534912109375, - "end_to_end_latency_ms": 10.32414436340332, - "ops_per_second": 96.86032709352325, - "bytes_per_second": 101565414.34241824, + "encrypt_latency_ms": 4.959416389465332, + "decrypt_latency_ms": 5.180764198303223, + "end_to_end_latency_ms": 10.1637601852417, + "ops_per_second": 98.38878345949674, + "bytes_per_second": 103168117.00482525, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 10.346174240112305, - "p95_latency": 10.860681533813477, - "p99_latency": 10.860681533813477, - "timestamp": "2025-09-02 16:57:31", + "p50_latency": 10.123014450073242, + "p95_latency": 10.914087295532227, + "p99_latency": 10.914087295532227, + "timestamp": "2025-09-03 13:31:00", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -156,22 +132,18 @@ "test_name": "throughput", "language": "python", "data_size": 10485760, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 35.567426681518555, - "decrypt_latency_ms": 38.695549964904785, - "end_to_end_latency_ms": 74.97415542602539, - "ops_per_second": 13.337929508077861, - "bytes_per_second": 139858327.7186225, + "encrypt_latency_ms": 35.22911071777344, + "decrypt_latency_ms": 37.80970573425293, + "end_to_end_latency_ms": 73.72639179229736, + "ops_per_second": 13.563663915863518, + "bytes_per_second": 142225324.54240504, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 74.99313354492188, - "p95_latency": 77.53705978393555, - "p99_latency": 77.53705978393555, - "timestamp": "2025-09-02 16:57:32", + "p50_latency": 72.88718223571777, + "p95_latency": 86.45200729370117, + "p99_latency": 86.45200729370117, + "timestamp": "2025-09-03 13:31:01", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -180,22 +152,18 @@ "test_name": "throughput", "language": "python", "data_size": 52428800, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 167.90552139282227, - "decrypt_latency_ms": 182.42778778076172, - "end_to_end_latency_ms": 353.8800239562988, - "ops_per_second": 2.8258164697182555, - "bytes_per_second": 148154166.52756447, + "encrypt_latency_ms": 188.44032287597656, + "decrypt_latency_ms": 202.92325019836426, + "end_to_end_latency_ms": 396.1131811141968, + "ops_per_second": 2.524530986793158, + "bytes_per_second": 132358130.20038112, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 354.4912338256836, - "p95_latency": 356.77409172058105, - "p99_latency": 356.77409172058105, - "timestamp": "2025-09-02 16:57:38", + "p50_latency": 416.7790412902832, + "p95_latency": 421.2830066680908, + "p99_latency": 421.2830066680908, + "timestamp": "2025-09-03 13:31:07", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -204,22 +172,18 @@ "test_name": "throughput", "language": "python", "data_size": 104857600, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, - "encrypt_latency_ms": 332.6861381530762, - "decrypt_latency_ms": 362.6670837402344, - "end_to_end_latency_ms": 703.7388801574707, - "ops_per_second": 1.4209816001302031, - "bytes_per_second": 149000720.23381278, + "encrypt_latency_ms": 406.12549781799316, + "decrypt_latency_ms": 434.38572883605957, + "end_to_end_latency_ms": 849.9415636062622, + "ops_per_second": 1.17655147461791, + "bytes_per_second": 123370363.90489496, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, - "p50_latency": 703.5648822784424, - "p95_latency": 713.519811630249, - "p99_latency": 713.519811630249, - "timestamp": "2025-09-02 16:57:49", + "p50_latency": 850.6782054901123, + "p95_latency": 883.1961154937744, + "p99_latency": 883.1961154937744, + "timestamp": "2025-09-03 13:31:20", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -228,22 +192,18 @@ "test_name": "memory", "language": "python", "data_size": 1024, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.234375, - "avg_memory_mb": 334.84517045454544, - "cumulative_allocations_mb": 0.3372917175292969, - "memory_efficiency_ratio": 2.913073875553484e-06, + "peak_memory_mb": 327.234375, + "memory_efficiency_ratio": 2.9842906937879007e-6, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:49", + "timestamp": "2025-09-03 13:31:21", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -252,22 +212,18 @@ "test_name": "memory", "language": "python", "data_size": 5120, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.25, - "avg_memory_mb": 335.2421875, - "cumulative_allocations_mb": 0.34973716735839844, - "memory_efficiency_ratio": 1.456469052945563e-05, + "peak_memory_mb": 327.328125, + "memory_efficiency_ratio": 1.4917179817652394e-5, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:50", + "timestamp": "2025-09-03 13:31:21", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -276,22 +232,18 @@ "test_name": "memory", "language": "python", "data_size": 10240, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.390625, - "avg_memory_mb": 335.3390625, - "cumulative_allocations_mb": 0.3968391418457031, - "memory_efficiency_ratio": 2.9117167481947357e-05, + "peak_memory_mb": 327.421875, + "memory_efficiency_ratio": 2.982581722739203e-5, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:51", + "timestamp": "2025-09-03 13:31:22", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -300,22 +252,18 @@ "test_name": "memory", "language": "python", "data_size": 102400, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.390625, - "avg_memory_mb": 335.3792613636364, - "cumulative_allocations_mb": 1.2789506912231445, - "memory_efficiency_ratio": 0.0002911716748194736, + "peak_memory_mb": 327.53125, + "memory_efficiency_ratio": 0.00029815857265528097, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:52", + "timestamp": "2025-09-03 13:31:22", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -324,22 +272,18 @@ "test_name": "memory", "language": "python", "data_size": 512000, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.390625, - "avg_memory_mb": 335.38169642857144, - "cumulative_allocations_mb": 5.2001848220825195, - "memory_efficiency_ratio": 0.0014558583740973679, + "peak_memory_mb": 327.796875, + "memory_efficiency_ratio": 0.0014895848229181563, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:52", + "timestamp": "2025-09-03 13:31:23", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -348,22 +292,18 @@ "test_name": "memory", "language": "python", "data_size": 1048576, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 335.421875, - "avg_memory_mb": 335.3988486842105, - "cumulative_allocations_mb": 10.33751392364502, - "memory_efficiency_ratio": 0.0029813201658359344, + "peak_memory_mb": 327.859375, + "memory_efficiency_ratio": 0.003050088166611066, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:53", + "timestamp": "2025-09-03 13:31:24", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -372,22 +312,18 @@ "test_name": "memory", "language": "python", "data_size": 10485760, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 355.46875, - "avg_memory_mb": 348.4203125, - "cumulative_allocations_mb": 100.69142055511475, - "memory_efficiency_ratio": 0.028131868131868132, + "peak_memory_mb": 355.265625, + "memory_efficiency_ratio": 0.028147952676254563, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:57:55", + "timestamp": "2025-09-03 13:31:26", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -396,22 +332,18 @@ "test_name": "memory", "language": "python", "data_size": 52428800, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 448.71875, - "avg_memory_mb": 385.78617294520546, - "cumulative_allocations_mb": 502.26827335357666, - "memory_efficiency_ratio": 0.11142837244933491, + "peak_memory_mb": 402.578125, + "memory_efficiency_ratio": 0.12419949543954978, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:02", + "timestamp": "2025-09-03 13:31:34", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -420,22 +352,18 @@ "test_name": "memory", "language": "python", "data_size": 104857600, - "algorithm_suite": "default", - "frame_length": null, "concurrency": 1, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, "end_to_end_latency_ms": 0.0, "ops_per_second": 0.0, "bytes_per_second": 0.0, - "peak_memory_mb": 549.1875, - "avg_memory_mb": 411.4255622739602, - "cumulative_allocations_mb": 1004.2456932067871, - "memory_efficiency_ratio": 0.18208717423466483, + "peak_memory_mb": 537.140625, + "memory_efficiency_ratio": 0.18617098641533583, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -444,22 +372,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1024, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.577303886413574, - "ops_per_second": 277.5240351213832, - "bytes_per_second": 284184.6119642964, + "end_to_end_latency_ms": 10.280156135559082, + "ops_per_second": 165.03326788616127, + "bytes_per_second": 168994.06631542914, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -468,22 +392,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1024, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.268360137939453, - "ops_per_second": 290.45722575976345, - "bytes_per_second": 297428.19917799777, + "end_to_end_latency_ms": 7.921397686004639, + "ops_per_second": 227.24425361307888, + "bytes_per_second": 232698.11569979278, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -492,22 +412,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1024, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.7978172302246094, - "ops_per_second": 296.1817636155, - "bytes_per_second": 303290.125942272, + "end_to_end_latency_ms": 10.525625944137573, + "ops_per_second": 253.8041541231109, + "bytes_per_second": 259895.45382206555, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -516,22 +432,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1024, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.565165400505066, - "ops_per_second": 297.53642233773326, - "bytes_per_second": 304677.29647383885, + "end_to_end_latency_ms": 5.144643783569336, + "ops_per_second": 264.4419593073245, + "bytes_per_second": 270788.5663307003, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -540,22 +452,18 @@ "test_name": "concurrent", "language": "python", "data_size": 5120, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.389596939086914, - "ops_per_second": 293.807238874451, - "bytes_per_second": 1504293.0630371892, + "end_to_end_latency_ms": 4.716682434082031, + "ops_per_second": 210.84639088710935, + "bytes_per_second": 1079533.521342, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:16", + "timestamp": "2025-09-03 13:31:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -564,22 +472,18 @@ "test_name": "concurrent", "language": "python", "data_size": 5120, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.360271453857422, - "ops_per_second": 296.4465742193574, - "bytes_per_second": 1517806.4600031099, + "end_to_end_latency_ms": 7.594752311706543, + "ops_per_second": 204.57424911962386, + "bytes_per_second": 1047420.1554924741, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -588,22 +492,18 @@ "test_name": "concurrent", "language": "python", "data_size": 5120, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.3717453479766846, - "ops_per_second": 295.63063650428364, - "bytes_per_second": 1513628.8589019324, + "end_to_end_latency_ms": 7.602018117904663, + "ops_per_second": 273.7135287169548, + "bytes_per_second": 1401413.2670308086, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -612,22 +512,18 @@ "test_name": "concurrent", "language": "python", "data_size": 5120, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.21522855758667, - "ops_per_second": 296.52385619211145, - "bytes_per_second": 1518202.1437036106, + "end_to_end_latency_ms": 4.655987024307251, + "ops_per_second": 287.723520070245, + "bytes_per_second": 1473144.4227596545, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -636,22 +532,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10240, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.4344911575317383, - "ops_per_second": 289.896117719428, - "bytes_per_second": 2968536.2454469427, + "end_to_end_latency_ms": 3.811049461364746, + "ops_per_second": 261.1516238294481, + "bytes_per_second": 2674192.6280135484, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -660,22 +552,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10240, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.277920722961426, - "ops_per_second": 290.53669890000276, - "bytes_per_second": 2975095.796736028, + "end_to_end_latency_ms": 3.8301825523376465, + "ops_per_second": 259.8983161123418, + "bytes_per_second": 2661358.7569903797, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -684,22 +572,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10240, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 3.4348905086517334, - "ops_per_second": 289.4419984818163, - "bytes_per_second": 2963886.0644537993, + "end_to_end_latency_ms": 4.701852798461914, + "ops_per_second": 265.3749398931997, + "bytes_per_second": 2717439.3845063653, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:51", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -708,22 +592,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10240, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.671064019203186, - "ops_per_second": 277.0389561923445, - "bytes_per_second": 2836878.911409607, + "end_to_end_latency_ms": 7.730191946029663, + "ops_per_second": 263.062045435124, + "bytes_per_second": 2693755.3452556697, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:17", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -732,22 +612,18 @@ "test_name": "concurrent", "language": "python", "data_size": 102400, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.224300384521484, - "ops_per_second": 235.97177995566707, - "bytes_per_second": 24163510.26746031, + "end_to_end_latency_ms": 4.592490196228027, + "ops_per_second": 216.8831894100005, + "bytes_per_second": 22208838.595584054, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -756,22 +632,18 @@ "test_name": "concurrent", "language": "python", "data_size": 102400, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.057180881500244, - "ops_per_second": 245.76024000023438, - "bytes_per_second": 25165848.576024, + "end_to_end_latency_ms": 5.870294570922852, + "ops_per_second": 210.65948112413895, + "bytes_per_second": 21571530.86711183, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -780,22 +652,18 @@ "test_name": "concurrent", "language": "python", "data_size": 102400, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 5.054503679275513, - "ops_per_second": 246.21395698318776, - "bytes_per_second": 25212309.195078425, + "end_to_end_latency_ms": 7.066059112548828, + "ops_per_second": 218.67485551769892, + "bytes_per_second": 22392305.20501237, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -804,22 +672,18 @@ "test_name": "concurrent", "language": "python", "data_size": 102400, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 4.062226414680481, - "ops_per_second": 245.5199946146394, - "bytes_per_second": 25141247.448539075, + "end_to_end_latency_ms": 4.879406094551086, + "ops_per_second": 216.84450522038316, + "bytes_per_second": 22204877.334567234, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -828,22 +692,18 @@ "test_name": "concurrent", "language": "python", "data_size": 512000, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 11.237502098083496, - "ops_per_second": 142.19859574655632, - "bytes_per_second": 72805681.02223684, + "end_to_end_latency_ms": 14.16919231414795, + "ops_per_second": 126.89708587472165, + "bytes_per_second": 64971307.96785749, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:52", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -852,22 +712,18 @@ "test_name": "concurrent", "language": "python", "data_size": 512000, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 17.5534725189209, - "ops_per_second": 144.90474257434295, - "bytes_per_second": 74191228.19806358, + "end_to_end_latency_ms": 14.004456996917725, + "ops_per_second": 127.47288282401568, + "bytes_per_second": 65266116.00589603, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:18", + "timestamp": "2025-09-03 13:31:53", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -876,22 +732,18 @@ "test_name": "concurrent", "language": "python", "data_size": 512000, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 39.74910378456116, - "ops_per_second": 130.24484349005417, - "bytes_per_second": 66685359.86690773, + "end_to_end_latency_ms": 31.768375635147095, + "ops_per_second": 127.41015258340751, + "bytes_per_second": 65233998.12270465, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:19", + "timestamp": "2025-09-03 13:31:53", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -900,22 +752,18 @@ "test_name": "concurrent", "language": "python", "data_size": 512000, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 31.3551664352417, - "ops_per_second": 146.77259726063588, - "bytes_per_second": 75147569.79744557, + "end_to_end_latency_ms": 44.03595328330994, + "ops_per_second": 135.69715263690864, + "bytes_per_second": 69476942.15009722, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:19", + "timestamp": "2025-09-03 13:31:53", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -924,22 +772,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1048576, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 17.444610595703125, - "ops_per_second": 96.39396857425865, - "bytes_per_second": 101076401.99172184, + "end_to_end_latency_ms": 19.460415840148926, + "ops_per_second": 94.34773415392367, + "bytes_per_second": 98930769.68818466, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:19", + "timestamp": "2025-09-03 13:31:54", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -948,22 +792,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1048576, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 31.107473373413086, - "ops_per_second": 95.48046366359803, - "bytes_per_second": 100118522.66652097, + "end_to_end_latency_ms": 36.25231981277466, + "ops_per_second": 95.67604804897967, + "bytes_per_second": 100323607.7590069, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:20", + "timestamp": "2025-09-03 13:31:54", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -972,22 +812,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1048576, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 58.18980932235718, - "ops_per_second": 90.38834648078848, - "bytes_per_second": 94779050.79943927, + "end_to_end_latency_ms": 46.89902067184448, + "ops_per_second": 96.55395948434622, + "bytes_per_second": 101244164.62025782, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:20", + "timestamp": "2025-09-03 13:31:54", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -996,22 +832,18 @@ "test_name": "concurrent", "language": "python", "data_size": 1048576, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 56.39534294605255, - "ops_per_second": 95.7876199363803, - "bytes_per_second": 100440599.3624099, + "end_to_end_latency_ms": 65.42666256427765, + "ops_per_second": 99.92930525993525, + "bytes_per_second": 104783471.19224186, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:21", + "timestamp": "2025-09-03 13:31:55", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1020,22 +852,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10485760, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 150.94387531280518, - "ops_per_second": 13.030402067812828, - "bytes_per_second": 136633668.78658903, + "end_to_end_latency_ms": 150.7880449295044, + "ops_per_second": 13.103940394826786, + "bytes_per_second": 137404774.03445894, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:22", + "timestamp": "2025-09-03 13:31:56", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1044,22 +872,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10485760, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 296.8547224998474, - "ops_per_second": 13.011930305905558, - "bytes_per_second": 136439978.32445228, + "end_to_end_latency_ms": 286.3491892814636, + "ops_per_second": 13.362948033719723, + "bytes_per_second": 140120665.97405693, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:23", + "timestamp": "2025-09-03 13:31:57", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1068,22 +892,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10485760, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 571.5517699718475, - "ops_per_second": 13.032108583090837, - "bytes_per_second": 136651562.89623058, + "end_to_end_latency_ms": 551.3835549354553, + "ops_per_second": 13.087009198620754, + "bytes_per_second": 137227237.57452956, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:26", + "timestamp": "2025-09-03 13:32:00", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1092,22 +912,18 @@ "test_name": "concurrent", "language": "python", "data_size": 10485760, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 1118.0467426776886, - "ops_per_second": 12.357680604362868, - "bytes_per_second": 129579672.97400399, + "end_to_end_latency_ms": 1093.3409422636032, + "ops_per_second": 12.943706326306428, + "bytes_per_second": 135724598.0481309, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:33", + "timestamp": "2025-09-03 13:32:07", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1116,22 +932,18 @@ "test_name": "concurrent", "language": "python", "data_size": 52428800, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 700.9313821792603, - "ops_per_second": 2.8261352066857444, - "bytes_per_second": 148170877.52428555, + "end_to_end_latency_ms": 698.815393447876, + "ops_per_second": 2.8408856310101984, + "bytes_per_second": 148944224.57110748, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:37", + "timestamp": "2025-09-03 13:32:10", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1140,22 +952,18 @@ "test_name": "concurrent", "language": "python", "data_size": 52428800, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 1404.3150186538696, - "ops_per_second": 2.796435143641608, - "bytes_per_second": 146613738.85895714, + "end_to_end_latency_ms": 1391.381275653839, + "ops_per_second": 2.81095644003493, + "bytes_per_second": 147375073.00330332, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:44", + "timestamp": "2025-09-03 13:32:18", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1164,22 +972,18 @@ "test_name": "concurrent", "language": "python", "data_size": 52428800, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 2820.2352464199066, - "ops_per_second": 2.745385669438221, - "bytes_per_second": 143937276.1858426, + "end_to_end_latency_ms": 2753.239780664444, + "ops_per_second": 2.8032581871980256, + "bytes_per_second": 146971462.84496784, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:58:59", + "timestamp": "2025-09-03 13:32:32", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1188,22 +992,18 @@ "test_name": "concurrent", "language": "python", "data_size": 52428800, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 5537.272220849991, - "ops_per_second": 2.7413794859986993, - "bytes_per_second": 143727236.7955286, + "end_to_end_latency_ms": 5447.913703322411, + "ops_per_second": 2.787988207910584, + "bytes_per_second": 146170876.15490243, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:59:28", + "timestamp": "2025-09-03 13:33:01", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1212,22 +1012,18 @@ "test_name": "concurrent", "language": "python", "data_size": 104857600, - "algorithm_suite": "", - "frame_length": null, "concurrency": 2, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 1410.4030847549438, - "ops_per_second": 1.4100913399973307, - "bytes_per_second": 147858793.6929041, + "end_to_end_latency_ms": 1391.8258666992188, + "ops_per_second": 1.4265229486044437, + "bytes_per_second": 149581772.73558533, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:59:36", + "timestamp": "2025-09-03 13:33:08", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1236,22 +1032,18 @@ "test_name": "concurrent", "language": "python", "data_size": 104857600, - "algorithm_suite": "", - "frame_length": null, "concurrency": 4, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 2806.2445402145386, - "ops_per_second": 1.4072359363354092, - "bytes_per_second": 147559382.9178838, + "end_to_end_latency_ms": 2796.0678696632385, + "ops_per_second": 1.4101954752716157, + "bytes_per_second": 147869713.06784096, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 16:59:50", + "timestamp": "2025-09-03 13:33:23", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1260,22 +1052,18 @@ "test_name": "concurrent", "language": "python", "data_size": 104857600, - "algorithm_suite": "", - "frame_length": null, "concurrency": 8, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 5545.182865858078, - "ops_per_second": 1.403170790617305, - "bytes_per_second": 147133121.49423313, + "end_to_end_latency_ms": 5783.284509181976, + "ops_per_second": 1.3486006649672388, + "bytes_per_second": 141411029.08686873, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 17:00:19", + "timestamp": "2025-09-03 13:33:53", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 @@ -1284,25 +1072,21 @@ "test_name": "concurrent", "language": "python", "data_size": 104857600, - "algorithm_suite": "", - "frame_length": null, "concurrency": 16, "encrypt_latency_ms": 0.0, "decrypt_latency_ms": 0.0, - "end_to_end_latency_ms": 11181.330859661102, - "ops_per_second": 1.3749360047730161, - "bytes_per_second": 144172489.61408702, + "end_to_end_latency_ms": 10844.419693946838, + "ops_per_second": 1.4042389516883949, + "bytes_per_second": 147245126.30056104, "peak_memory_mb": 0.0, - "avg_memory_mb": 0.0, - "cumulative_allocations_mb": 0.0, "memory_efficiency_ratio": 0.0, "p50_latency": 0.0, "p95_latency": 0.0, "p99_latency": 0.0, - "timestamp": "2025-09-02 17:01:18", + "timestamp": "2025-09-03 13:34:50", "python_version": "3.12.7", "cpu_count": 12, "total_memory_gb": 36.0 } ] -} \ No newline at end of file +} From a8936dad1711cebc8a86f081dbff7deededd7187 Mon Sep 17 00:00:00 2001 From: Shubham Chaturvedi Date: Wed, 3 Sep 2025 13:46:01 -0700 Subject: [PATCH 4/4] fix: linting --- .../benchmarks/python/benchmark.py | 5 ++++- .../benchmarks/python/results.py | 2 +- .../benchmarks/python/tests.py | 18 ++++-------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/esdk-performance-testing/benchmarks/python/benchmark.py b/esdk-performance-testing/benchmarks/python/benchmark.py index 70986c235..2d0ca5104 100644 --- a/esdk-performance-testing/benchmarks/python/benchmark.py +++ b/esdk-performance-testing/benchmarks/python/benchmark.py @@ -11,7 +11,10 @@ import psutil from aws_cryptographic_material_providers.mpl import AwsCryptographicMaterialProviders from aws_cryptographic_material_providers.mpl.config import MaterialProvidersConfig -from aws_cryptographic_material_providers.mpl.models import AesWrappingAlg, CreateRawAesKeyringInput +from aws_cryptographic_material_providers.mpl.models import ( + AesWrappingAlg, + CreateRawAesKeyringInput, +) from aws_encryption_sdk import EncryptionSDKClient, CommitmentPolicy from config import load_config diff --git a/esdk-performance-testing/benchmarks/python/results.py b/esdk-performance-testing/benchmarks/python/results.py index ff670d5b9..778d0ac95 100644 --- a/esdk-performance-testing/benchmarks/python/results.py +++ b/esdk-performance-testing/benchmarks/python/results.py @@ -9,7 +9,7 @@ import time from dataclasses import dataclass, asdict from pathlib import Path -from typing import List, Optional +from typing import List import psutil diff --git a/esdk-performance-testing/benchmarks/python/tests.py b/esdk-performance-testing/benchmarks/python/tests.py index 2a4f54eab..a0ed71b92 100644 --- a/esdk-performance-testing/benchmarks/python/tests.py +++ b/esdk-performance-testing/benchmarks/python/tests.py @@ -60,9 +60,7 @@ def run_throughput_test( timing_data = _collect_timing_data(benchmark, data, iterations) # Calculate statistics - return _create_throughput_result( - timing_data, data_size - ) + return _create_throughput_result(timing_data, data_size) def _collect_timing_data(benchmark, data, iterations): @@ -121,9 +119,7 @@ def _create_throughput_result(timing_data, data_size): ) -def run_memory_test( - benchmark, data_size: int -) -> BenchmarkResult: +def run_memory_test(benchmark, data_size: int) -> BenchmarkResult: """Run memory usage benchmark test""" data = os.urandom(data_size) iterations = 5 @@ -235,8 +231,6 @@ def _create_memory_result( ): """Create memory benchmark result""" peak_memory_mb = max(all_samples) - avg_memory_mb = sum(all_samples) / len(all_samples) - cumulative_allocations_mb = total_allocations / 1024 / 1024 memory_efficiency = ( original_data_size / (peak_memory_mb * 1024 * 1024) if peak_memory_mb > 0 else 0 ) @@ -369,10 +363,7 @@ def _get_test_parameters(config): def _calculate_total_tests(params): """Calculate total number of tests to run""" - return ( - len(params["data_sizes"]) - * (1 + len(params["concurrency_levels"]) + 1) - ) + return len(params["data_sizes"]) * (1 + len(params["concurrency_levels"]) + 1) def _run_throughput_tests(benchmark, params, results, pbar): @@ -390,8 +381,7 @@ def _run_throughput_tests(benchmark, params, results, pbar): ) results.append(result) benchmark.logger.info( - f"Throughput test completed: " - f"{result.ops_per_second:.2f} ops/sec" + f"Throughput test completed: " f"{result.ops_per_second:.2f} ops/sec" ) except Exception as e: benchmark.logger.error(f"Throughput test failed: {e}")