diff --git a/Cargo.lock b/Cargo.lock index 88956744..eaa14751 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,7 +82,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -93,7 +93,7 @@ checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -131,9 +131,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -171,18 +171,18 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -264,9 +264,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -377,9 +377,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -510,6 +510,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "iana-time-zone" version = "0.1.63" @@ -839,6 +845,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -982,9 +998,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -1004,7 +1020,7 @@ version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", ] [[package]] @@ -1086,9 +1102,9 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1150,6 +1166,7 @@ dependencies = [ "jsonschema", "lazy_static", "msvc_spectre_libs", + "num_cpus", "prettydiff", "rand", "regex", @@ -1201,9 +1218,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ee4885492bb655bfa05d039cd9163eb8fe9f79ddebf00ca23a1637510c2fd2" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1233,16 +1250,16 @@ version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1312,11 +1329,11 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", "unicode-ident", ] @@ -1327,9 +1344,9 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1346,22 +1363,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1499,9 +1516,9 @@ checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -1521,9 +1538,9 @@ version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1549,11 +1566,11 @@ dependencies = [ [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -1575,9 +1592,9 @@ version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1586,9 +1603,9 @@ version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1615,15 +1632,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" @@ -1795,9 +1803,9 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", "synstructure", ] @@ -1816,9 +1824,9 @@ version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] [[package]] @@ -1836,9 +1844,9 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", "synstructure", ] @@ -1870,7 +1878,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ - "proc-macro2 1.0.97", + "proc-macro2 1.0.101", "quote 1.0.40", - "syn 2.0.105", + "syn 2.0.106", ] diff --git a/Cargo.toml b/Cargo.toml index 52b01cc4..6540b43b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,6 +125,8 @@ test-generator = "0.3.1" walkdir = "2.3.2" criterion = { version = "0.7" } +num_cpus = "1.16" + [build-dependencies] anyhow = "1.0" @@ -158,6 +160,16 @@ name = "schema_validation_benchmark" harness = false required-features = ["azure_policy"] +[[bench]] +name = "engine_evaluation_benchmark" +path = "benches/evaluation/engine_evaluation_benchmark.rs" +harness = false + +[[bench]] +name = "compiled_policy_evaluation_benchmark" +path = "benches/evaluation/compiled_policy_evaluation_benchmark.rs" +harness = false + [[example]] name="regorus" harness=false diff --git a/benches/evaluation/README.md b/benches/evaluation/README.md new file mode 100644 index 00000000..b391bb2c --- /dev/null +++ b/benches/evaluation/README.md @@ -0,0 +1,157 @@ +# Regorus Multi-Threaded Evaluation Benchmark + +A benchmark suite for measuring the multi-threaded performance of the Regorus policy evaluation engine. + +## Overview + +This benchmark evaluates the throughput and scalability of Regorus policy evaluation across different thread counts and configuration strategies. It measures performance variations between fresh and cloned engine instances, as well as fresh and cloned input data. + +## Features + +- **Multi-threaded evaluation** testing from 1 to `num_cpus * 2` threads +- **Configurable engine strategies**: Fresh vs. cloned engine instances +- **Configurable input strategies**: Fresh parsing vs. cloned input data +- **Complex policy evaluation** using realistic RBAC and data sensitivity policies +- **Criterion-based benchmarking** with statistical analysis +- **Performance metrics** including throughput and timing + +## Benchmark Structure + +### Test Configurations + +The benchmark tests four different configuration combinations: + +1. **Cloned Engines + Cloned Inputs**: Pre-instantiated engines with pre-parsed input data +2. **Cloned Engines + Fresh Inputs**: Pre-instantiated engines with fresh JSON parsing +3. **Fresh Engines + Cloned Inputs**: New engine instances with pre-parsed input data +4. **Fresh Engines + Fresh Inputs**: New engine instances with fresh JSON parsing + +### Thread Scaling + +Tests are performed with thread counts: 1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 (up to `num_cpus * 2`) + +Each thread performs 1000 policy evaluations to ensure statistically significant measurements. + +## Running the Benchmark + +### Prerequisites + +- Rust 1.70+ +- Cargo + +### Execution + +Run the complete benchmark suite: + +```bash +cargo bench evaluation_benchmark +``` + +Run specific benchmarks: + +```bash +# Run only cloned engines with cloned inputs +cargo bench "cloned_engines , cloned_inputs" + +# Run only single-threaded tests +cargo bench "1 threads" +``` + +### Output + +Results are generated in the `target/criterion/` directory and include: + +- Detailed timing statistics +- Throughput measurements (Kelem/s) +- Performance comparison with previous runs +- HTML reports with graphs and analysis + +## Test Policies + +The benchmark uses complex Rego policies that simulate real-world scenarios: + +### RBAC Policy +- Role-based access control with hierarchical permissions +- User-role-resource mapping +- Action-based authorization + +### Data Sensitivity Policy +- Multi-level data classification (public, internal, confidential, secret) +- Access level validation +- Clearance-based filtering + +### Time-based Access Policy +- Business hours validation +- Temporal access control +- Schedule-based permissions + +### Azure Resource Policies +- **VM Deployment**: VM size restrictions, regional compliance, security configurations +- **Storage Account Security**: Encryption requirements, network ACLs, HTTPS enforcement +- **Key Vault Access**: Service principal validation, soft delete requirements, conditional access +- **Network Security Groups**: Port restrictions, CIDR validation, priority-based rules + +### Policy Complexity Features +- **Multi-condition validation**: Complex nested object property checks +- **Network operations**: CIDR matching and IP range validation +- **Time-based constraints**: Timestamp comparisons and business hour logic +- **Security compliance**: Encryption, authentication, and access control patterns +- **Azure Resource Manager**: Real-world cloud governance scenarios + +## Configuration + +### Benchmark Parameters + +- **Evaluations per thread**: 1000 +- **Measurement iterations**: 100 samples per configuration +- **Warm-up time**: 3 seconds +- **Measurement time**: 10 seconds (extended for high thread counts) + +### Customization + +The benchmark can be customized by modifying `evaluation_benchmark.rs`: + +```rust +// Adjust evaluations per thread +let evals_per_thread = 1000; + +// Modify thread count calculation +let max_threads = num_cpus::get() * 2; + +// Configure test scenarios +let scenarios = [ + (true, true), // cloned_engines, cloned_inputs + (true, false), // cloned_engines, fresh_inputs + (false, true), // fresh_engines, cloned_inputs + (false, false), // fresh_engines, fresh_inputs +]; +``` + +## Understanding Results + +### Metrics + +- **Total Evaluation Time**: Total execution time for all evaluations across all threads (ms) +- **Throughput**: Evaluations per second measured in Kelem/s + - **Kelem/s**: Thousands of elements (policy evaluations) per second + - Example: 98.71 Kelem/s = 98,710 policy evaluations per second + + +### Interpretation + +- **Lower time** = better performance +- **Higher throughput** = better performance +- **Consistent results** across runs indicate stable performance +- **Outliers** may indicate system interference or measurement variance + +### Tips + +- Run on dedicated hardware for consistent results +- Disable other applications during benchmarking +- Use release builds for accurate performance measurements +- Consider CPU affinity for highly controlled testing + +## Files + +- `evaluation_benchmark.rs`: Main benchmark implementation +- Results are saved to `../../target/criterion/` directory diff --git a/benches/evaluation/compiled_policy_evaluation_benchmark.md b/benches/evaluation/compiled_policy_evaluation_benchmark.md new file mode 100644 index 00000000..03ab85bf --- /dev/null +++ b/benches/evaluation/compiled_policy_evaluation_benchmark.md @@ -0,0 +1,135 @@ +# Compiled Policy Evaluation Benchmark Results + +## Test Environment +- **Platform**: Apple Silicon (M-Series) +- **CPU**: 16 cores +- **Architecture**: ARM64 (aarch64-apple-darwin) +- **Rust Version**: 1.82.0 +- **Benchmark Framework**: Criterion.rs +- **Test Data**: 20,000 inputs per evaluation (1000 per thread) +- **Policy**: Complex authorization policy with nested rules + +## Benchmark Overview + +The compiled policy evaluation benchmark tests Regorus compiled policy performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of compiled policy and input data reuse strategies. + +## Configuration Combinations + +1. **Compiled Shared Policies, Cloned Inputs**: Each thread uses shared compiled policies and clones of parsed input data - optimal for performance +2. **Compiled Shared Policies, Fresh Inputs**: Each thread uses shared compiled policies but parses new inputs each time +3. **Compiled Per Iteration, Cloned Inputs**: Each thread compiles the policy each iteration but reuses input data +4. **Compiled Per Iteration, Fresh Inputs**: Each thread compiles new policies and parses new inputs for each iteration + +## Performance Results + +### Compiled Shared Policies, Cloned Inputs (Best Performance) +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 3.30 | 303 | +| 2 | 8.53 | 234 | +| 4 | 18.78 | 213 | +| 6 | 32.35 | 186 | +| 8 | 73.12 | 109 | +| 10 | 108.97 | 92 | +| 12 | 145.56 | 82 | +| 14 | 196.14 | 71 | +| 16 | 248.77 | 64 | +| 18 | 290.01 | 62 | +| 20 | 317.16 | 63 | +| 22 | 348.83 | 63 | +| 24 | 361.05 | 66 | +| 26 | 389.70 | 67 | +| 28 | 418.66 | 67 | +| 30 | 444.40 | 68 | +| 32 | 476.53 | 67 | + +### Compiled Shared Policies, Fresh Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 4.51 | 222 | +| 2 | 9.77 | 205 | +| 4 | 23.36 | 171 | +| 6 | 38.12 | 157 | +| 8 | 85.02 | 94 | +| 10 | 133.66 | 75 | +| 12 | 180.46 | 66 | +| 14 | 238.23 | 59 | +| 16 | 318.78 | 50 | +| 18 | 353.15 | 51 | +| 20 | 389.29 | 51 | +| 22 | 459.61 | 48 | +| 24 | 507.62 | 47 | +| 26 | 539.43 | 48 | +| 28 | 554.99 | 50 | +| 30 | 625.57 | 48 | +| 32 | 690.55 | 46 | + +### Compiled Per Iteration, Cloned Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 22.68 | 44 | +| 2 | 47.99 | 42 | +| 4 | 108.09 | 37 | +| 6 | 167.62 | 36 | +| 8 | 283.17 | 28 | +| 10 | 418.25 | 24 | +| 12 | 546.24 | 22 | +| 14 | 688.79 | 20 | +| 16 | 951.72 | 17 | +| 18 | 1060.20 | 17 | +| 20 | 1223.60 | 16 | +| 22 | 1342.50 | 16 | +| 24 | 1445.70 | 17 | +| 26 | 1676.50 | 15 | +| 28 | 1765.20 | 16 | +| 30 | 1939.00 | 15 | +| 32 | 2197.30 | 15 | + +### Compiled Per Iteration, Fresh Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 23.95 | 42 | +| 2 | 49.53 | 40 | +| 4 | 116.42 | 34 | +| 6 | 197.35 | 30 | +| 8 | 293.04 | 27 | +| 10 | 385.90 | 26 | +| 12 | 508.82 | 24 | +| 14 | 679.23 | 21 | +| 16 | 913.02 | 18 | +| 18 | 1075.90 | 17 | +| 20 | 1209.80 | 17 | +| 22 | 1358.90 | 16 | +| 24 | 1523.90 | 16 | +| 26 | 1700.20 | 15 | +| 28 | 1966.90 | 14 | +| 30 | 2179.30 | 14 | +| 32 | 2327.70 | 14 | + +## Analysis + +The compiled policy benchmark demonstrates the following performance characteristics: + +1. **Best Performance**: Compiled shared policies with cloned inputs provide the highest throughput +2. **Compilation Impact**: + - Pre-compiled policies: Significantly faster than per-iteration compilation + - Per-iteration compilation: Major overhead (~7x slower than pre-compiled) +3. **Scaling Patterns**: + - Best throughput achieved at 1 thread for shared policy configurations + - Higher thread counts show performance degradation due to contention + - Per-iteration compilation shows poor scaling across all thread counts +4. **Input Processing**: Fresh inputs add ~25-30% overhead across all configurations +5. **Thread Performance**: + - Peak performance at 1 thread for most configurations + - Reasonable performance maintained up to 12-16 threads for shared policies + - Compiled policies show better thread scaling than per-iteration compilation + +## Comparison with Engine Evaluation + +| Configuration | Compiled Policy (1 thread) | Engine Evaluation (1 thread) | Performance Ratio | +|:---------------------|:--------------------------------|:--------------------------------|------------------:| +| Shared/Cloned | Best performance | Higher throughput | 0.67x-0.92x | +| Shared/Fresh | ~27% reduction from optimal | ~30% reduction from optimal | 0.62x-0.97x | +| Per-iteration/Cloned | ~85% reduction from optimal | ~86% reduction from optimal | 0.80x-0.98x | +| Per-iteration/Fresh | ~86% reduction from optimal | ~87% reduction from optimal | 0.78x-1.00x | + diff --git a/benches/evaluation/compiled_policy_evaluation_benchmark.rs b/benches/evaluation/compiled_policy_evaluation_benchmark.rs new file mode 100644 index 00000000..fc2d96bc --- /dev/null +++ b/benches/evaluation/compiled_policy_evaluation_benchmark.rs @@ -0,0 +1,232 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use regorus::{compile_policy_with_entrypoint, CompiledPolicy, PolicyModule, Value}; +use std::collections::HashMap; +use std::hint::black_box; +use std::sync::{Arc, Barrier, Mutex}; +use std::thread; +use std::time::Duration; + +mod policy_data; + +fn multi_threaded_compiled_eval( + num_threads: usize, + evals_per_thread: usize, + use_shared_policies: bool, + use_cloned_inputs: bool, +) -> (std::time::Duration, HashMap, usize) { + // Complex policies with multiple valid inputs for each + let policies_with_inputs = policy_data::policies_with_inputs(); + + // Policy names for tracking + let policy_names = policy_data::policy_names() + .into_iter() + .map(|s| s.to_string()) + .collect::>(); + + // Pre-compile all policies and share them between threads (only if using shared policies) + let compiled_policies: Option>> = if use_shared_policies { + Some(Arc::new( + policies_with_inputs + .iter() + .map(|(policy, _)| { + let module = PolicyModule { + id: "policy.rego".into(), + content: policy.as_str().into(), + }; + compile_policy_with_entrypoint( + Value::new_object(), + &[module], + "data.bench.allow".into(), + ) + .unwrap() + }) + .collect(), + )) + } else { + None + }; + + // Initialize policy evaluation counters + let policy_counters = Arc::new(Mutex::new(HashMap::new())); + for policy_name in &policy_names { + policy_counters + .lock() + .unwrap() + .insert(policy_name.to_string(), 0); + } + let total_evals = Arc::new(Mutex::new(0usize)); + + let barrier = Arc::new(Barrier::new(num_threads)); + let mut handles = Vec::with_capacity(num_threads); + + for thread_id in 0..num_threads { + let barrier = barrier.clone(); + let policies_with_inputs = policies_with_inputs.clone(); + let compiled_policies = compiled_policies.clone(); + let policy_names = policy_names.clone(); + let policy_counters = policy_counters.clone(); + let total_evals = total_evals.clone(); + + handles.push(thread::spawn(move || { + let mut elapsed = std::time::Duration::ZERO; + + // Pre-parse inputs if using cloned inputs + let parsed_inputs = if use_cloned_inputs { + Some( + policies_with_inputs + .iter() + .map(|(_, inputs)| { + inputs + .iter() + .map(|input_str| Value::from_json_str(input_str).unwrap()) + .collect::>() + }) + .collect::>(), + ) + } else { + None + }; + + barrier.wait(); + for i in 0..evals_per_thread { + // Use different policy for each iteration - thread_id ensures different threads + // start with different policies for better load distribution + let policy_idx = (thread_id + i) % policies_with_inputs.len(); + let (_, inputs) = &policies_with_inputs[policy_idx]; + + // Use different input for the same policy based on iteration - thread_id ensures + // different threads start with different inputs for better load distribution + let input_idx = (thread_id + i) % inputs.len(); + let input = &inputs[input_idx]; + + let start = std::time::Instant::now(); + + let input_value = if use_cloned_inputs { + parsed_inputs.as_ref().unwrap()[policy_idx][input_idx].clone() + } else { + Value::from_json_str(input).unwrap() + }; + + let result = if let Some(ref compiled_policies_vec) = compiled_policies { + // Use pre-compiled policy + let compiled_policy = &compiled_policies_vec[policy_idx]; + compiled_policy.eval_with_input(input_value) + } else { + // Compile policy in each iteration + let (policy, _) = &policies_with_inputs[policy_idx]; + let module = PolicyModule { + id: "policy.rego".into(), + content: policy.as_str().into(), + }; + let compiled_policy = compile_policy_with_entrypoint( + Value::new_object(), + &[module], + "data.bench.allow".into(), + ) + .unwrap(); + compiled_policy.eval_with_input(input_value) + }; + + elapsed += start.elapsed(); + + // Track total and successful evaluations + { + let mut total = total_evals.lock().unwrap(); + *total += 1; + } + if result.is_ok() { + if let Some(policy_name) = policy_names.get(policy_idx) { + let mut counters = policy_counters.lock().unwrap(); + *counters.entry(policy_name.to_string()).or_insert(0) += 1; + } + } + } + elapsed + })); + } + + let mut total = std::time::Duration::ZERO; + for handle in handles { + total += handle.join().unwrap(); + } + + let final_counters = policy_counters.lock().unwrap().clone(); + let total_evals = *total_evals.lock().unwrap(); + (total, final_counters, total_evals) +} + +fn criterion_benchmark(c: &mut Criterion) { + let max_threads = num_cpus::get() * 2; + println!( + "Running compiled policy benchmark with max_threads: {}", + max_threads + ); + + let evals_per_thread = 1000; + + // Benchmark all combinations of compilation strategy and input strategy + for use_shared_policies in [true, false] { + for use_cloned_inputs in [true, false] { + let group_name = match (use_shared_policies, use_cloned_inputs) { + (true, true) => "compiled_shared_policies, cloned_inputs ", + (true, false) => "compiled_shared_policies, fresh_inputs ", + (false, true) => "compiled_per_iteration , cloned_inputs ", + (false, false) => "compiled_per_iteration , fresh_inputs ", + }; + + let mut group = c.benchmark_group(group_name); + group.measurement_time(Duration::from_secs(5)); + + // Test specific thread counts: powers of 2 + some intermediate values + let thread_counts: Vec = (1..=max_threads) + .filter(|&n| { + n == 1 || // Always test single-threaded + n % 2 == 0 || // Always test even threads + n == max_threads // Maximum threads + }) + .collect(); + + for threads in thread_counts { + let total_evals = threads * evals_per_thread; + group.throughput(Throughput::Elements(total_evals as u64)); + group.bench_with_input( + BenchmarkId::new("compiled_eval", format!(" {threads} threads")), + &threads, + |b, &threads| { + b.iter_custom(|iters| { + let evals_per_thread = evals_per_thread * (iters as usize); + + let (duration, policy_counters, total_evals_aggregated) = multi_threaded_compiled_eval( + black_box(threads), + black_box(evals_per_thread), + black_box(use_shared_policies), + black_box(use_cloned_inputs), + ); + + // Sanity check: Ensure the expected number of evaluations matches the actual number performed per iteration batch. + // total_evals is the expected number for this batch, total_evals_aggregated is the sum over all iters. + assert_eq!(total_evals, total_evals_aggregated/iters as usize); + + // On one iteration, print policy evaluation statistics + if iters == 1 { + // println!("\nCompiled Policy Evaluation Statistics:"); + for (policy_name, count) in &policy_counters { + // println!(" {}: {} evaluations", policy_name, count); + if *count == 0 { + println!("\x1b[31mERROR: Policy '{}' was never evaluated successfully!\x1b[0m", policy_name); + } + } + } + + duration + }); + }, + ); + } + group.finish(); + } + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/evaluation/engine_evaluation_benchmark.md b/benches/evaluation/engine_evaluation_benchmark.md new file mode 100644 index 00000000..67a89262 --- /dev/null +++ b/benches/evaluation/engine_evaluation_benchmark.md @@ -0,0 +1,125 @@ +# Engine Evaluation Benchmark Results + +## Test Environment +- **Platform**: Apple Silicon (M-Series) +- **CPU**: 16 cores +- **Architecture**: ARM64 (aarch64-apple-darwin) +- **Rust Version**: 1.82.0 +- **Benchmark Framework**: Criterion.rs +- **Test Data**: 20,000 inputs per evaluation (1000 per thread) +- **Policy**: Complex authorization policy with nested rules + +## Benchmark Overview + +The engine evaluation benchmark tests Regorus policy evaluation performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of engine and input data reuse strategies. + +## Configuration Combinations + +1. **Cloned Engines, Cloned Inputs**: Each thread uses its own engine and clones of parsed input data - optimal for performance +2. **Cloned Engines, Fresh Inputs**: Each thread uses its own engine but parses new inputs each time +3. **Fresh Engines, Cloned Inputs**: Each thread creates a new engine each iteration but reuses input data +4. **Fresh Engines, Fresh Inputs**: Each thread creates new engines and parses new inputs for each iteration + +## Performance Results + +### Cloned Engines, Cloned Inputs (Best Performance) +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 3.05 | 328 | +| 2 | 7.46 | 268 | +| 4 | 16.10 | 248 | +| 6 | 25.94 | 231 | +| 8 | 50.18 | 159 | +| 10 | 80.27 | 125 | +| 12 | 106.31 | 113 | +| 14 | 137.31 | 102 | +| 16 | 163.91 | 98 | +| 18 | 182.06 | 99 | +| 20 | 191.36 | 105 | +| 22 | 201.51 | 109 | +| 24 | 217.65 | 110 | +| 26 | 228.11 | 114 | +| 28 | 248.17 | 113 | +| 30 | 264.15 | 114 | +| 32 | 314.27 | 102 | + +### Cloned Engines, Fresh Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 4.36 | 229 | +| 2 | 10.34 | 194 | +| 4 | 21.98 | 182 | +| 6 | 34.05 | 176 | +| 8 | 66.47 | 120 | +| 10 | 100.78 | 99 | +| 12 | 141.69 | 85 | +| 14 | 188.53 | 74 | +| 16 | 261.27 | 61 | +| 18 | 285.29 | 63 | +| 20 | 312.14 | 64 | +| 22 | 329.42 | 67 | +| 24 | 347.97 | 69 | +| 26 | 370.24 | 70 | +| 28 | 394.75 | 71 | +| 30 | 419.30 | 72 | +| 32 | 433.58 | 74 | + +### Fresh Engines, Cloned Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 22.39 | 45 | +| 2 | 49.22 | 41 | +| 4 | 98.09 | 41 | +| 6 | 160.21 | 37 | +| 8 | 281.26 | 28 | +| 10 | 413.61 | 24 | +| 12 | 578.15 | 21 | +| 14 | 746.34 | 19 | +| 16 | 961.44 | 17 | +| 18 | 1127.70 | 16 | +| 20 | 1248.40 | 16 | +| 22 | 1386.90 | 16 | +| 24 | 1559.70 | 15 | +| 26 | 1736.30 | 15 | +| 28 | 1891.80 | 15 | +| 30 | 2077.00 | 14 | +| 32 | 2289.30 | 14 | + +### Fresh Engines, Fresh Inputs +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 23.63 | 42 | +| 2 | 48.82 | 41 | +| 4 | 102.32 | 39 | +| 6 | 160.09 | 37 | +| 8 | 271.21 | 29 | +| 10 | 397.39 | 25 | +| 12 | 489.09 | 25 | +| 14 | 670.33 | 21 | +| 16 | 884.83 | 18 | +| 18 | 1044.00 | 17 | +| 20 | 1174.20 | 17 | +| 22 | 1330.40 | 17 | +| 24 | 1480.90 | 16 | +| 26 | 1679.50 | 15 | +| 28 | 1873.90 | 15 | +| 30 | 2070.90 | 14 | +| 32 | 2325.40 | 14 | + +## Analysis + +The benchmark results demonstrate the following performance characteristics: + +1. **Best Performance**: Cloned engines with cloned inputs consistently deliver the highest throughput +2. **Configuration Performance Hierarchy**: + - Cloned engines, cloned inputs: Best performance (optimal configuration) + - Cloned engines, fresh inputs: ~30% reduction from optimal + - Fresh engines, cloned inputs: ~86% reduction from optimal + - Fresh engines, fresh inputs: ~87% reduction from optimal +3. **Scaling Patterns**: + - Performance degrades with increased thread count due to contention + - Best throughput achieved at 1 thread for cloned engine configurations + - Fresh engine configurations show poor scaling across all thread counts +4. **Engine Creation Overhead**: Fresh engine creation is a significant performance bottleneck (~7-8x slower than cloned engines) +5. **Input Processing**: Fresh input generation adds moderate overhead (~30% impact compared to cloned inputs) +6. **Thread Contention**: Performance degradation occurs with higher thread counts across all configurations \ No newline at end of file diff --git a/benches/evaluation/engine_evaluation_benchmark.rs b/benches/evaluation/engine_evaluation_benchmark.rs new file mode 100644 index 00000000..7eea1ed1 --- /dev/null +++ b/benches/evaluation/engine_evaluation_benchmark.rs @@ -0,0 +1,228 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use regorus::{Engine, Value}; +use std::collections::HashMap; +use std::hint::black_box; +use std::sync::{Arc, Barrier, Mutex}; +use std::thread; +use std::time::Duration; + +mod policy_data; + +fn multi_threaded_eval( + num_threads: usize, + evals_per_thread: usize, + use_cloned_engines: bool, + use_cloned_inputs: bool, +) -> (std::time::Duration, HashMap, usize) { + // Complex policies with multiple valid inputs for each + let policies_with_inputs = policy_data::policies_with_inputs(); + + // Policy names for tracking + let policy_names = policy_data::policy_names() + .into_iter() + .map(|s| s.to_string()) + .collect::>(); + + // Initialize policy evaluation counters + let policy_counters = Arc::new(Mutex::new(HashMap::new())); + for policy_name in &policy_names { + policy_counters + .lock() + .unwrap() + .insert(policy_name.to_string(), 0); + } + + let barrier = Arc::new(Barrier::new(num_threads)); + let mut handles = Vec::with_capacity(num_threads); + + let total_evals = Arc::new(Mutex::new(0usize)); + for thread_id in 0..num_threads { + let barrier = barrier.clone(); + let policies_with_inputs = policies_with_inputs.clone(); + let policy_names = policy_names.clone(); + let policy_counters = policy_counters.clone(); + let total_evals = total_evals.clone(); + + handles.push(thread::spawn(move || { + let mut elapsed = std::time::Duration::ZERO; + + // Pre-create engines if using cloned engines + let engines = if use_cloned_engines { + Some( + policies_with_inputs + .iter() + .map(|(policy, _)| { + let mut engine = Engine::new(); + engine + .add_policy("policy.rego".to_string(), policy.to_string()) + .unwrap(); + { + // Warm up the engine to ensure it's fully prepared for evaluation. + // This prevents each cloned engine from repeating preparation work. + engine.set_input(Value::new_object()); + let _ = engine.eval_rule("data.bench.allow".to_string()); + } + engine + }) + .collect::>(), + ) + } else { + None + }; + + // Pre-parse inputs if using cloned inputs + let parsed_inputs = if use_cloned_inputs { + Some( + policies_with_inputs + .iter() + .map(|(_, inputs)| { + inputs + .iter() + .map(|input_str| regorus::Value::from_json_str(input_str).unwrap()) + .collect::>() + }) + .collect::>(), + ) + } else { + None + }; + + barrier.wait(); + for i in 0..evals_per_thread { + // Use different policy for each iteration - thread_id ensures different threads + // start with different policies for better load distribution + let policy_idx = (thread_id + i) % policies_with_inputs.len(); + let (policy, inputs) = &policies_with_inputs[policy_idx]; + + // Use different input for the same policy based on iteration - thread_id ensures + // different threads start with different inputs for better load distribution + let input_idx = (thread_id + i) % inputs.len(); + let input = &inputs[input_idx]; + + let start = std::time::Instant::now(); + + let result = { + let mut engine = if use_cloned_engines { + engines.as_ref().unwrap()[policy_idx].clone() + } else { + let mut engine = Engine::new(); + engine + .add_policy("policy.rego".to_string(), policy.to_string()) + .unwrap(); + engine + }; + + let input_value = if use_cloned_inputs { + parsed_inputs.as_ref().unwrap()[policy_idx][input_idx].clone() + } else { + regorus::Value::from_json_str(input).unwrap() + }; + + engine.set_input(input_value); + + engine.eval_rule("data.bench.allow".to_string()) + + // Engine cleanup/drop time is included in measurement to reflect + // real-world total cost of policy evaluation lifecycle + }; + elapsed += start.elapsed(); + + // Track total and successful evaluations + { + let mut total = total_evals.lock().unwrap(); + *total += 1; + } + if result.is_ok() { + if let Some(policy_name) = policy_names.get(policy_idx) { + let mut counters = policy_counters.lock().unwrap(); + *counters.entry(policy_name.to_string()).or_insert(0) += 1; + } + } + } + elapsed + })); + } + + let mut total = std::time::Duration::ZERO; + for handle in handles { + total += handle.join().unwrap(); + } + + let final_counters = policy_counters.lock().unwrap().clone(); + let total_evals = *total_evals.lock().unwrap(); + (total, final_counters, total_evals) +} + +fn criterion_benchmark(c: &mut Criterion) { + let max_threads = num_cpus::get() * 2; + println!("Running benchmark with max_threads: {}", max_threads); + + let evals_per_thread = 1000; + + // Benchmark all combinations of cloned engines and inputs + for use_cloned_engines in [true, false] { + for use_cloned_inputs in [true, false] { + let group_name = match (use_cloned_engines, use_cloned_inputs) { + (true, true) => "cloned_engines , cloned_inputs ", + (true, false) => "cloned_engines , fresh_inputs ", + (false, true) => "fresh_engines , cloned_inputs ", + (false, false) => "fresh_engines , fresh_inputs ", + }; + + let mut group = c.benchmark_group(group_name); + group.measurement_time(Duration::from_secs(5)); + + // Test specific thread counts: powers of 2 + some intermediate values + let thread_counts: Vec = (1..=max_threads) + .filter(|&n| { + n == 1 || // Always test single-threaded + n % 2 == 0 || // Always test even threads + n == max_threads // Maximum threads + }) + .collect(); + + for threads in thread_counts { + let total_evals = threads * evals_per_thread; + group.throughput(Throughput::Elements(total_evals as u64)); + group.bench_with_input( + BenchmarkId::new("eval", format!(" {threads} threads")), + &threads, + |b, &threads| { + b.iter_custom(|iters| { + let evals_per_thread = evals_per_thread * (iters as usize); + + let (duration, policy_counters, total_evals_aggregated) = multi_threaded_eval( + black_box(threads), + black_box(evals_per_thread), + black_box(use_cloned_engines), + black_box(use_cloned_inputs), + ); + + + // Sanity check: Ensure the expected number of evaluations matches the actual number performed per iteration batch. + // total_evals is the expected number for this batch, total_evals_aggregated is the sum over all iters. + assert_eq!(total_evals, total_evals_aggregated/iters as usize); + + // On one iteration, print policy evaluation statistics + if iters == 1 { + // println!("\nPolicy Evaluation Statistics:"); + for (policy_name, count) in &policy_counters { + // println!(" {}: {} evaluations", policy_name, count); + if *count == 0 { + println!("\x1b[31mERROR: Policy '{}' was never evaluated successfully!\x1b[0m", policy_name); + } + } + } + + duration + }); + }, + ); + } + group.finish(); + } + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/evaluation/policy_data.rs b/benches/evaluation/policy_data.rs new file mode 100644 index 00000000..29ebb96e --- /dev/null +++ b/benches/evaluation/policy_data.rs @@ -0,0 +1,117 @@ +// This module provides the full set of policies, inputs, and policy names for evaluation benchmarks. +// Policies and inputs are now loaded from external files. + +use std::fs; +use std::path::Path; + +pub fn policies_with_inputs() -> Vec<(String, Vec)> { + let policy_with_input_files = [ + ( + "rbac_policy.rego", + vec!["rbac_input.json", "rbac_input2.json", "rbac_input3.json"], + ), + ( + "api_access_policy.rego", + vec![ + "api_access_input.json", + "api_access_input2.json", + "api_access_input3.json", + ], + ), + ( + "data_sensitivity_policy.rego", + vec![ + "data_sensitivity_input.json", + "data_sensitivity_input2.json", + "data_sensitivity_input3.json", + ], + ), + ( + "time_based_policy.rego", + vec![ + "time_based_input.json", + "time_based_input2.json", + "time_based_input3.json", + ], + ), + ( + "data_processing_policy.rego", + vec![ + "data_processing_input.json", + "data_processing_input2.json", + "data_processing_input3.json", + ], + ), + ( + "azure_vm_policy.rego", + vec![ + "azure_vm_input.json", + "azure_vm_input2.json", + "azure_vm_input3.json", + ], + ), + ( + "azure_storage_policy.rego", + vec![ + "azure_storage_input.json", + "azure_storage_input2.json", + "azure_storage_input3.json", + ], + ), + ( + "azure_keyvault_policy.rego", + vec![ + "azure_keyvault_input.json", + "azure_keyvault_input2.json", + "azure_keyvault_input3.json", + ], + ), + ( + "azure_nsg_policy.rego", + vec![ + "azure_nsg_input.json", + "azure_nsg_input2.json", + "azure_nsg_input3.json", + ], + ), + ]; + + let mut policies_and_inputs = Vec::new(); + let base_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benches") + .join("evaluation") + .join("test_data"); + + for (policy_file, input_files) in policy_with_input_files.iter() { + let policy_path = base_dir.join("policies").join(policy_file); + + let policy_content = fs::read_to_string(&policy_path) + .unwrap_or_else(|e| panic!("Failed to read policy file {:?}: {}", policy_path, e)); + + let mut input_contents = Vec::new(); + for input_file in input_files { + let input_path = base_dir.join("inputs").join(input_file); + let input_content = fs::read_to_string(&input_path) + .unwrap_or_else(|e| panic!("Failed to read input file {:?}: {}", input_path, e)); + input_contents.push(input_content); + } + + policies_and_inputs.push((policy_content, input_contents)); + } + + policies_and_inputs +} + +pub fn policy_names() -> Vec<&'static str> { + vec![ + "rbac_policy", + "api_access_policy", + "data_sensitivity_policy", + "time_based_policy", + "data_processing_policy", + "azure_vm_policy", + "azure_storage_policy", + "azure_keyvault_policy", + "azure_nsg_policy", + ] +} diff --git a/benches/evaluation/test_data/inputs/api_access_input.json b/benches/evaluation/test_data/inputs/api_access_input.json new file mode 100644 index 00000000..9cf91037 --- /dev/null +++ b/benches/evaluation/test_data/inputs/api_access_input.json @@ -0,0 +1,15 @@ +{ + "request": { + "method": "GET", + "path": "/api/v1/users/123" + }, + "user": { + "id": "user123", + "scope": ["read:users", "write:users"], + "department": "engineering" + }, + "resource": { + "owner": "user123", + "type": "user" + } +} diff --git a/benches/evaluation/test_data/inputs/api_access_input2.json b/benches/evaluation/test_data/inputs/api_access_input2.json new file mode 100644 index 00000000..04104044 --- /dev/null +++ b/benches/evaluation/test_data/inputs/api_access_input2.json @@ -0,0 +1,15 @@ +{ + "request": { + "method": "POST", + "path": "/api/v1/users" + }, + "user": { + "id": "user456", + "scope": ["write:users", "admin:users"], + "department": "engineering" + }, + "resource": { + "owner": "user456", + "type": "user" + } +} diff --git a/benches/evaluation/test_data/inputs/api_access_input3.json b/benches/evaluation/test_data/inputs/api_access_input3.json new file mode 100644 index 00000000..a64f734a --- /dev/null +++ b/benches/evaluation/test_data/inputs/api_access_input3.json @@ -0,0 +1,15 @@ +{ + "request": { + "method": "DELETE", + "path": "/api/v1/users/789" + }, + "user": { + "id": "admin123", + "scope": ["admin:users"], + "department": "security" + }, + "resource": { + "owner": "user789", + "type": "user" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_keyvault_input.json b/benches/evaluation/test_data/inputs/azure_keyvault_input.json new file mode 100644 index 00000000..0ea08b2d --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_keyvault_input.json @@ -0,0 +1,16 @@ +{ + "vault": { + "name": "mykeyvault", + "location": "eastus", + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enablePurgeProtection": true, + "networkAcls": { + "defaultAction": "Deny", + "bypass": "AzureServices" + }, + "tags": { + "environment": "production" + } + } +} diff --git a/benches/evaluation/test_data/inputs/azure_keyvault_input2.json b/benches/evaluation/test_data/inputs/azure_keyvault_input2.json new file mode 100644 index 00000000..3e17cfae --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_keyvault_input2.json @@ -0,0 +1,16 @@ +{ + "vault": { + "name": "devkeyvault", + "location": "westus2", + "enableSoftDelete": true, + "softDeleteRetentionInDays": 30, + "enablePurgeProtection": false, + "networkAcls": { + "defaultAction": "Allow", + "bypass": "AzureServices" + }, + "tags": { + "environment": "development" + } + } +} diff --git a/benches/evaluation/test_data/inputs/azure_keyvault_input3.json b/benches/evaluation/test_data/inputs/azure_keyvault_input3.json new file mode 100644 index 00000000..8482ce3a --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_keyvault_input3.json @@ -0,0 +1,16 @@ +{ + "vault": { + "name": "prodkeyvault", + "location": "eastus", + "enableSoftDelete": true, + "softDeleteRetentionInDays": 90, + "enablePurgeProtection": true, + "networkAcls": { + "defaultAction": "Deny", + "bypass": "AzureServices" + }, + "tags": { + "environment": "production" + } + } +} diff --git a/benches/evaluation/test_data/inputs/azure_nsg_input.json b/benches/evaluation/test_data/inputs/azure_nsg_input.json new file mode 100644 index 00000000..d750e2b6 --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_nsg_input.json @@ -0,0 +1,13 @@ +{ + "operation": "Microsoft.Network/networkSecurityGroups/securityRules/write", + "rule": { + "direction": "Inbound", + "access": "Allow", + "protocol": "TCP", + "sourceAddressPrefix": "10.0.0.0/24", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "80", + "priority": 1001 + } +} diff --git a/benches/evaluation/test_data/inputs/azure_nsg_input2.json b/benches/evaluation/test_data/inputs/azure_nsg_input2.json new file mode 100644 index 00000000..7a690186 --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_nsg_input2.json @@ -0,0 +1,13 @@ +{ + "operation": "Microsoft.Network/networkSecurityGroups/securityRules/write", + "rule": { + "direction": "Inbound", + "access": "Allow", + "protocol": "TCP", + "sourceAddressPrefix": "172.16.0.0/16", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "priority": 1200 + } +} diff --git a/benches/evaluation/test_data/inputs/azure_nsg_input3.json b/benches/evaluation/test_data/inputs/azure_nsg_input3.json new file mode 100644 index 00000000..05e76b19 --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_nsg_input3.json @@ -0,0 +1,13 @@ +{ + "operation": "Microsoft.Network/networkSecurityGroups/securityRules/write", + "rule": { + "direction": "Inbound", + "access": "Allow", + "protocol": "TCP", + "sourceAddressPrefix": "203.0.113.0/24", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "443", + "priority": 300 + } +} diff --git a/benches/evaluation/test_data/inputs/azure_storage_input.json b/benches/evaluation/test_data/inputs/azure_storage_input.json new file mode 100644 index 00000000..e37d151c --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_storage_input.json @@ -0,0 +1,15 @@ +{ + "account": { + "name": "mystorageaccount", + "tier": "Standard", + "replication": "LRS", + "location": "eastus", + "tags": { + "environment": "production" + } + }, + "container": { + "name": "data", + "publicAccess": "None" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_storage_input2.json b/benches/evaluation/test_data/inputs/azure_storage_input2.json new file mode 100644 index 00000000..977cfbba --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_storage_input2.json @@ -0,0 +1,15 @@ +{ + "account": { + "name": "devstorageaccount", + "tier": "Premium", + "replication": "LRS", + "location": "westus2", + "tags": { + "environment": "production" + } + }, + "container": { + "name": "logs", + "publicAccess": "None" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_storage_input3.json b/benches/evaluation/test_data/inputs/azure_storage_input3.json new file mode 100644 index 00000000..c38736cc --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_storage_input3.json @@ -0,0 +1,15 @@ +{ + "account": { + "name": "prodstorageaccount", + "tier": "Standard", + "replication": "GRS", + "location": "eastus", + "tags": { + "environment": "production" + } + }, + "container": { + "name": "backups", + "publicAccess": "None" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_vm_input.json b/benches/evaluation/test_data/inputs/azure_vm_input.json new file mode 100644 index 00000000..2a525157 --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_vm_input.json @@ -0,0 +1,14 @@ +{ + "vm": { + "size": "Standard_D2s_v3", + "os": "Linux", + "location": "eastus", + "tags": { + "environment": "production", + "department": "engineering" + } + }, + "user": { + "department": "engineering" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_vm_input2.json b/benches/evaluation/test_data/inputs/azure_vm_input2.json new file mode 100644 index 00000000..f028951d --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_vm_input2.json @@ -0,0 +1,14 @@ +{ + "vm": { + "size": "Standard_B1s", + "os": "Windows", + "location": "westus2", + "tags": { + "environment": "dev", + "department": "marketing" + } + }, + "user": { + "department": "marketing" + } +} diff --git a/benches/evaluation/test_data/inputs/azure_vm_input3.json b/benches/evaluation/test_data/inputs/azure_vm_input3.json new file mode 100644 index 00000000..40ed850d --- /dev/null +++ b/benches/evaluation/test_data/inputs/azure_vm_input3.json @@ -0,0 +1,14 @@ +{ + "vm": { + "size": "Standard_D4s_v3", + "os": "Linux", + "location": "eastus", + "tags": { + "environment": "production", + "department": "engineering" + } + }, + "user": { + "department": "engineering" + } +} diff --git a/benches/evaluation/test_data/inputs/data_processing_input.json b/benches/evaluation/test_data/inputs/data_processing_input.json new file mode 100644 index 00000000..b2a04a13 --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_processing_input.json @@ -0,0 +1,16 @@ +{ + "operation": "collect", + "data": { + "type": "email", + "source": "user_input" + }, + "consent": { + "given": true, + "purpose": "marketing", + "date": "2023-01-15" + }, + "user": { + "age": 25, + "location": "US" + } +} diff --git a/benches/evaluation/test_data/inputs/data_processing_input2.json b/benches/evaluation/test_data/inputs/data_processing_input2.json new file mode 100644 index 00000000..ecb977d8 --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_processing_input2.json @@ -0,0 +1,16 @@ +{ + "operation": "process", + "data": { + "type": "survey_response", + "source": "user_input" + }, + "consent": { + "given": true, + "purpose": "analytics", + "date": "2023-06-15" + }, + "user": { + "age": 30, + "location": "US" + } +} diff --git a/benches/evaluation/test_data/inputs/data_processing_input3.json b/benches/evaluation/test_data/inputs/data_processing_input3.json new file mode 100644 index 00000000..9adb72b0 --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_processing_input3.json @@ -0,0 +1,16 @@ +{ + "operation": "delete", + "data": { + "type": "user_profile", + "source": "database" + }, + "consent": { + "given": false, + "purpose": "none", + "date": "2022-01-01" + }, + "user": { + "age": 16, + "location": "EU" + } +} diff --git a/benches/evaluation/test_data/inputs/data_sensitivity_input.json b/benches/evaluation/test_data/inputs/data_sensitivity_input.json new file mode 100644 index 00000000..aac42a7b --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_sensitivity_input.json @@ -0,0 +1,13 @@ +{ + "data": { + "type": "user_profile", + "classification": "personal", + "contains_pii": true, + "region": "EU" + }, + "user": { + "clearance": "confidential", + "location": "EU" + }, + "operation": "read" +} diff --git a/benches/evaluation/test_data/inputs/data_sensitivity_input2.json b/benches/evaluation/test_data/inputs/data_sensitivity_input2.json new file mode 100644 index 00000000..d88ddec2 --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_sensitivity_input2.json @@ -0,0 +1,13 @@ +{ + "data": { + "type": "financial_report", + "classification": "confidential", + "contains_pii": false, + "region": "US" + }, + "user": { + "clearance": "secret", + "location": "US" + }, + "operation": "read" +} diff --git a/benches/evaluation/test_data/inputs/data_sensitivity_input3.json b/benches/evaluation/test_data/inputs/data_sensitivity_input3.json new file mode 100644 index 00000000..2245e46b --- /dev/null +++ b/benches/evaluation/test_data/inputs/data_sensitivity_input3.json @@ -0,0 +1,13 @@ +{ + "data": { + "type": "public_announcement", + "classification": "public", + "contains_pii": false, + "region": "GLOBAL" + }, + "user": { + "clearance": "public", + "location": "EU" + }, + "operation": "read" +} diff --git a/benches/evaluation/test_data/inputs/rbac_input.json b/benches/evaluation/test_data/inputs/rbac_input.json new file mode 100644 index 00000000..ff85da13 --- /dev/null +++ b/benches/evaluation/test_data/inputs/rbac_input.json @@ -0,0 +1,12 @@ +{ + "user": { + "name": "alice", + "roles": ["viewer", "editor"] + }, + "resource": { + "name": "document1", + "type": "document", + "owner": "alice" + }, + "action": "read" +} diff --git a/benches/evaluation/test_data/inputs/rbac_input2.json b/benches/evaluation/test_data/inputs/rbac_input2.json new file mode 100644 index 00000000..ba7ae4e7 --- /dev/null +++ b/benches/evaluation/test_data/inputs/rbac_input2.json @@ -0,0 +1,12 @@ +{ + "user": { + "name": "bob", + "roles": ["admin"] + }, + "resource": { + "name": "document2", + "type": "document", + "owner": "bob" + }, + "action": "write" +} diff --git a/benches/evaluation/test_data/inputs/rbac_input3.json b/benches/evaluation/test_data/inputs/rbac_input3.json new file mode 100644 index 00000000..d27ca523 --- /dev/null +++ b/benches/evaluation/test_data/inputs/rbac_input3.json @@ -0,0 +1,12 @@ +{ + "user": { + "name": "charlie", + "roles": ["viewer"] + }, + "resource": { + "name": "document3", + "type": "document", + "owner": "alice" + }, + "action": "read" +} diff --git a/benches/evaluation/test_data/inputs/time_based_input.json b/benches/evaluation/test_data/inputs/time_based_input.json new file mode 100644 index 00000000..ad5a27d1 --- /dev/null +++ b/benches/evaluation/test_data/inputs/time_based_input.json @@ -0,0 +1,11 @@ +{ + "time": "09:30:00", + "day": "monday", + "user": { + "role": "employee", + "shift": "day" + }, + "request": { + "urgent": false + } +} diff --git a/benches/evaluation/test_data/inputs/time_based_input2.json b/benches/evaluation/test_data/inputs/time_based_input2.json new file mode 100644 index 00000000..6621f38a --- /dev/null +++ b/benches/evaluation/test_data/inputs/time_based_input2.json @@ -0,0 +1,11 @@ +{ + "time": "14:30:00", + "day": "wednesday", + "user": { + "role": "employee", + "shift": "day" + }, + "request": { + "urgent": false + } +} diff --git a/benches/evaluation/test_data/inputs/time_based_input3.json b/benches/evaluation/test_data/inputs/time_based_input3.json new file mode 100644 index 00000000..14c351d0 --- /dev/null +++ b/benches/evaluation/test_data/inputs/time_based_input3.json @@ -0,0 +1,11 @@ +{ + "time": "22:00:00", + "day": "friday", + "user": { + "role": "admin", + "shift": "night" + }, + "request": { + "urgent": true + } +} diff --git a/benches/evaluation/test_data/policies/api_access_policy.rego b/benches/evaluation/test_data/policies/api_access_policy.rego new file mode 100644 index 00000000..56ed65a2 --- /dev/null +++ b/benches/evaluation/test_data/policies/api_access_policy.rego @@ -0,0 +1,13 @@ +package bench + +default allow := false + +valid_api_paths := ["/api/v1/", "/api/v2/", "/api/v3/"] + +allow if { + input.request.method == "GET" + some path in valid_api_paths + startswith(input.request.path, path) + input.user.authenticated == true + time.now_ns() - input.user.login_time < 86400000000000 # 24 hours in nanoseconds +} diff --git a/benches/evaluation/test_data/policies/azure_keyvault_policy.rego b/benches/evaluation/test_data/policies/azure_keyvault_policy.rego new file mode 100644 index 00000000..4d84d2a1 --- /dev/null +++ b/benches/evaluation/test_data/policies/azure_keyvault_policy.rego @@ -0,0 +1,28 @@ +package bench + +default allow := false + +# Azure Key Vault access policy +valid_operations := [ + "Microsoft.KeyVault/vaults/keys/read", + "Microsoft.KeyVault/vaults/secrets/read", + "Microsoft.KeyVault/vaults/certificates/read" +] + +vault_admins := ["admin@company.com", "security@company.com"] + +allow if { + input.operation in valid_operations + input.principal.type == "ServicePrincipal" + input.principal.appId != "" + input.resource.properties.enableSoftDelete == true + input.resource.properties.enablePurgeProtection == true + time.now_ns() - input.principal.createdTime < 31536000000000000 # Less than 1 year old +} + +allow if { + input.operation in valid_operations + input.principal.type == "User" + input.principal.userPrincipalName in vault_admins + input.context.conditionalAccess.compliant == true +} diff --git a/benches/evaluation/test_data/policies/azure_nsg_policy.rego b/benches/evaluation/test_data/policies/azure_nsg_policy.rego new file mode 100644 index 00000000..1ce91f55 --- /dev/null +++ b/benches/evaluation/test_data/policies/azure_nsg_policy.rego @@ -0,0 +1,31 @@ +package bench + +default allow := false + +# Azure Network Security Group rules policy +dangerous_ports := [22, 3389, 1433, 3306, 5432, 6379, 27017] +internal_networks := ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] + +is_internal_source if { + some network in internal_networks + net.cidr_contains(network, input.rule.sourceAddressPrefix) +} + +allow if { + input.operation == "Microsoft.Network/networkSecurityGroups/securityRules/write" + input.rule.direction == "Inbound" + input.rule.access == "Allow" + input.rule.destinationPortRange != "*" + not input.rule.destinationPortRange in dangerous_ports + input.rule.sourceAddressPrefix != "*" + input.rule.sourceAddressPrefix != "Internet" +} + +allow if { + input.operation == "Microsoft.Network/networkSecurityGroups/securityRules/write" + input.rule.direction == "Inbound" + input.rule.access == "Allow" + input.rule.destinationPortRange in dangerous_ports + is_internal_source + input.rule.priority >= 1000 +} diff --git a/benches/evaluation/test_data/policies/azure_storage_policy.rego b/benches/evaluation/test_data/policies/azure_storage_policy.rego new file mode 100644 index 00000000..8690eaee --- /dev/null +++ b/benches/evaluation/test_data/policies/azure_storage_policy.rego @@ -0,0 +1,17 @@ +package bench + +default allow := false + +# Azure Storage Account security policy +required_encryption_algorithms := ["AES256", "RSA-OAEP"] + +allow if { + input.operation == "Microsoft.Storage/storageAccounts/write" + input.resource.properties.supportsHttpsTrafficOnly == true + input.resource.properties.minimumTlsVersion == "TLS1_2" + input.resource.properties.encryption.services.blob.enabled == true + input.resource.properties.encryption.keySource == "Microsoft.Storage" + input.resource.properties.allowBlobPublicAccess == false + input.resource.properties.networkAcls.defaultAction == "Deny" + count(input.resource.properties.networkAcls.ipRules) > 0 +} diff --git a/benches/evaluation/test_data/policies/azure_vm_policy.rego b/benches/evaluation/test_data/policies/azure_vm_policy.rego new file mode 100644 index 00000000..e7d0affd --- /dev/null +++ b/benches/evaluation/test_data/policies/azure_vm_policy.rego @@ -0,0 +1,20 @@ +package bench + +default allow := false + +# Azure VM deployment policy +allowed_vm_sizes := [ + "Standard_B1s", "Standard_B2s", "Standard_B4ms", + "Standard_D2s_v3", "Standard_D4s_v3", "Standard_F2s_v2" +] + +allowed_regions := ["eastus", "westus2", "northeurope", "southeastasia"] + +allow if { + input.operation == "Microsoft.Compute/virtualMachines/write" + input.resource.properties.hardwareProfile.vmSize in allowed_vm_sizes + input.resource.location in allowed_regions + input.resource.properties.osProfile.adminPassword == null # Require SSH keys + count(input.resource.tags) > 0 # Must have tags + input.resource.tags.environment in ["dev", "test", "prod"] +} diff --git a/benches/evaluation/test_data/policies/data_processing_policy.rego b/benches/evaluation/test_data/policies/data_processing_policy.rego new file mode 100644 index 00000000..af649177 --- /dev/null +++ b/benches/evaluation/test_data/policies/data_processing_policy.rego @@ -0,0 +1,28 @@ +package bench + +default allow := false + +# Complex data filtering and aggregation +sensitive_fields := ["ssn", "credit_card", "password"] + +contains_sensitive_data if { + some field in sensitive_fields + object.get(input.data, field, null) != null +} + +user_clearance_level := object.get(input.user.attributes, "clearance", 0) + +required_clearance := 3 if contains_sensitive_data else := 1 + +allow if { + user_clearance_level >= required_clearance + input.operation in ["read", "export"] + count(input.data) > 0 + count(input.data) <= 1000 # Limit data size +} + +allow if { + input.user.role == "data_processor" + input.operation == "transform" + not contains_sensitive_data +} diff --git a/benches/evaluation/test_data/policies/data_sensitivity_policy.rego b/benches/evaluation/test_data/policies/data_sensitivity_policy.rego new file mode 100644 index 00000000..bef921af --- /dev/null +++ b/benches/evaluation/test_data/policies/data_sensitivity_policy.rego @@ -0,0 +1,25 @@ +package bench + +default allow := false + +rbac_roles := { + "admin": ["read", "write", "delete", "admin"], + "manager": ["read", "write"], + "user": ["read"] +} + +user_permissions contains perm if { + some role in input.user.roles + perm := rbac_roles[role][_] +} + +allow if { + input.action in user_permissions + input.resource.owner == input.user.id +} + +allow if { + input.action in user_permissions + input.resource.public == true + input.action == "read" +} diff --git a/benches/evaluation/test_data/policies/rbac_policy.rego b/benches/evaluation/test_data/policies/rbac_policy.rego new file mode 100644 index 00000000..d8dd87e7 --- /dev/null +++ b/benches/evaluation/test_data/policies/rbac_policy.rego @@ -0,0 +1,10 @@ +package bench + +default allow := false + +allow if { + input.user.role == "admin" + input.action in ["read", "write", "delete"] + input.resource.classification in ["public", "internal"] + count(input.user.permissions) > 0 +} diff --git a/benches/evaluation/test_data/policies/time_based_policy.rego b/benches/evaluation/test_data/policies/time_based_policy.rego new file mode 100644 index 00000000..e6075eb1 --- /dev/null +++ b/benches/evaluation/test_data/policies/time_based_policy.rego @@ -0,0 +1,23 @@ +package bench + +default allow := false + +# Time-based access control with complex conditions +business_hours if { + hour := time.clock([time.now_ns(), "America/New_York"])[0] + hour >= 9 + hour < 17 +} + +allow if { + input.user.department in ["engineering", "product"] + input.action == "deploy" + business_hours + count([x | x := input.approvals[_]; x.status == "approved"]) >= 2 +} + +allow if { + input.user.emergency_access == true + input.action in ["read", "diagnose"] + input.justification != "" +} diff --git a/bindings/csharp/Benchmarks/Benchmarks.csproj b/bindings/csharp/Benchmarks/Benchmarks.csproj new file mode 100644 index 00000000..d4340cb3 --- /dev/null +++ b/bindings/csharp/Benchmarks/Benchmarks.csproj @@ -0,0 +1,25 @@ + + + Exe + net8.0 + Enable + + + + + -$(VersionSuffix) + + + + + -$(VersionSuffix) + + + + + + + + + + diff --git a/bindings/csharp/Benchmarks/CompiledPolicyEvaluationBenchmark.cs b/bindings/csharp/Benchmarks/CompiledPolicyEvaluationBenchmark.cs new file mode 100644 index 00000000..75be000b --- /dev/null +++ b/bindings/csharp/Benchmarks/CompiledPolicyEvaluationBenchmark.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Regorus; + +namespace Benchmarks +{ + public class CompiledPolicyEvaluationBenchmark + { + private static readonly string TestDataPath = Path.Combine( + Directory.GetCurrentDirectory(), + "..", "..", "..", + "benches", "evaluation", "test_data" + ); + + private static readonly (string PolicyFile, string[] InputFiles)[] PolicyInputFiles = new[] + { + ("rbac_policy.rego", new[] { "rbac_input.json", "rbac_input2.json", "rbac_input3.json" }), + ("api_access_policy.rego", new[] { "api_access_input.json", "api_access_input2.json", "api_access_input3.json" }), + ("data_sensitivity_policy.rego", new[] { "data_sensitivity_input.json", "data_sensitivity_input2.json", "data_sensitivity_input3.json" }), + ("time_based_policy.rego", new[] { "time_based_input.json", "time_based_input2.json", "time_based_input3.json" }), + ("data_processing_policy.rego", new[] { "data_processing_input.json", "data_processing_input2.json", "data_processing_input3.json" }), + ("azure_vm_policy.rego", new[] { "azure_vm_input.json", "azure_vm_input2.json", "azure_vm_input3.json" }), + ("azure_storage_policy.rego", new[] { "azure_storage_input.json", "azure_storage_input2.json", "azure_storage_input3.json" }), + ("azure_keyvault_policy.rego", new[] { "azure_keyvault_input.json", "azure_keyvault_input2.json", "azure_keyvault_input3.json" }), + ("azure_nsg_policy.rego", new[] { "azure_nsg_input.json", "azure_nsg_input2.json", "azure_nsg_input3.json" }) + }; + + private static readonly string[] PolicyNames = new[] + { + "rbac_policy", + "api_access_policy", + "data_sensitivity_policy", + "time_based_policy", + "data_processing_policy", + "azure_vm_policy", + "azure_storage_policy", + "azure_keyvault_policy", + "azure_nsg_policy" + }; + + private static List<(string Policy, string[] Inputs)> LoadPoliciesWithInputs() + { + var result = new List<(string Policy, string[] Inputs)>(); + + foreach (var (policyFile, inputFiles) in PolicyInputFiles) + { + var policyPath = Path.Combine(TestDataPath, "policies", policyFile); + var policy = File.ReadAllText(policyPath); + + var inputs = inputFiles.Select(inputFile => + { + var inputPath = Path.Combine(TestDataPath, "inputs", inputFile); + return File.ReadAllText(inputPath); + }).ToArray(); + + result.Add((policy, inputs)); + } + + return result; + } + + private static List PrepareSharedCompiledPolicies() + { + var policiesWithInputs = LoadPoliciesWithInputs(); + var compiledPolicies = new List(); + + foreach (var (policy, _) in policiesWithInputs) + { + var modules = new[] { new PolicyModule { Id = "policy.rego", Content = policy } }; + var compiled = Compiler.CompilePolicyWithEntrypoint("{}", modules, "data.bench.allow"); + compiledPolicies.Add(compiled); + } + + return compiledPolicies; + } + + public static void RunCompiledPolicyEvaluationBenchmark() + { + var cpuCount = Environment.ProcessorCount; + var maxThreads = cpuCount * 2; + var threadCounts = new List { 1, 2 }; + + // Add even numbers from 4 to maxThreads + for (int i = 4; i <= maxThreads; i += 2) + { + threadCounts.Add(i); + } + + Console.WriteLine($"Running compiled policy benchmark with max_threads: {maxThreads}"); + Console.WriteLine($"Testing with thread counts: {string.Join(", ", threadCounts)}"); + Console.WriteLine(); + + // Benchmark both shared policies and per-iteration compilation + var configurations = new[] + { + (true, "compiled_shared_policies"), + (false, "compiled_per_iteration") + }; + + foreach (var (useSharedPolicies, groupName) in configurations) + { + Console.WriteLine($"=== {groupName} ==="); + + foreach (var threads in threadCounts) + { + RunCompiledPolicyBenchmark(threads, useSharedPolicies, groupName); + } + Console.WriteLine(); + } + } + + public static void RunCompiledPolicyBenchmark(int threads, bool useSharedPolicies, string groupName) + { + const int warmupSeconds = 3; + const int durationSeconds = 3; + var policiesWithInputs = LoadPoliciesWithInputs(); + List? compiledPolicies = null; + + if (useSharedPolicies) + { + compiledPolicies = PrepareSharedCompiledPolicies(); + } + + Console.WriteLine($"Warming up with {threads} threads for {warmupSeconds} seconds..."); + + // Warmup phase + var (_, _, _) = RunBenchmarkPhase(threads, warmupSeconds, policiesWithInputs, compiledPolicies, useSharedPolicies, isWarmup: true); + + Console.WriteLine($"Running benchmark with {threads} threads for {durationSeconds} seconds..."); + + // Actual benchmark phase + var (totalEvaluations, evaluationTime, policyCounters) = RunBenchmarkPhase(threads, durationSeconds, policiesWithInputs, compiledPolicies, useSharedPolicies, isWarmup: false); + + // Calculate throughput based on pure evaluation time (consistent with Rust benchmark) + var evalsPerSecond = totalEvaluations / evaluationTime.TotalSeconds; + var kelemsPerSecond = evalsPerSecond / 1000.0; + + Console.WriteLine($"{groupName}/eval/{threads} threads"); + Console.WriteLine($" time: [{evaluationTime.TotalMilliseconds:F2} ms]"); + Console.WriteLine($" thrpt: [{kelemsPerSecond:F2} Kelem/s]"); + + // Clean up compiled policies if we created them + if (compiledPolicies != null) + { + foreach (var policy in compiledPolicies) + { + policy.Dispose(); + } + } + + // Verify that all policies were evaluated + var allEvaluated = policyCounters.Values.All(count => count > 0); + + if (allEvaluated) + { + Console.WriteLine("✓ All policies were evaluated successfully"); + } + else + { + Console.WriteLine("ERROR: Some policies were never evaluated successfully!"); + } + } + + private static (int totalEvaluations, TimeSpan evaluationTime, Dictionary policyCounters) RunBenchmarkPhase( + int threads, + int durationSeconds, + List<(string Policy, string[] Inputs)> policiesWithInputs, + List? compiledPolicies, + bool useSharedPolicies, + bool isWarmup) + { + var barrier = new Barrier(threads); + var tasks = new Task[threads]; + var policyCounters = new Dictionary(); + var evaluationTimes = new Dictionary(); + var lockObject = new object(); + var stopExecution = false; + + // Initialize counters + foreach (var policyName in PolicyNames) + { + policyCounters[policyName] = 0; + } + + var stopwatch = Stopwatch.StartNew(); + + for (int threadId = 0; threadId < threads; threadId++) + { + int tid = threadId; + tasks[threadId] = Task.Run(() => + { + barrier.SignalAndWait(); + + int evaluationCount = 0; + var localEvaluationTime = TimeSpan.Zero; + + while (!stopExecution) + { + // Use different policy for each iteration + int policyIdx = (tid + evaluationCount) % policiesWithInputs.Count; + var (policy, inputs) = policiesWithInputs[policyIdx]; + + // Use different input for the same policy based on iteration + int inputIdx = evaluationCount % inputs.Length; + var input = inputs[inputIdx]; + + try + { + // Measure only the evaluation call + var evalStopwatch = Stopwatch.StartNew(); + + if (useSharedPolicies) + { + var result = compiledPolicies![policyIdx].EvalWithInput(input); + } + else + { + // Compile policy in each iteration + var modules = new[] { new PolicyModule { Id = "policy.rego", Content = policy } }; + var compiled = Compiler.CompilePolicyWithEntrypoint("{}", modules, "data.bench.allow"); + var result = compiled.EvalWithInput(input); + compiled.Dispose(); + } + + evalStopwatch.Stop(); + localEvaluationTime += evalStopwatch.Elapsed; + + // Track successful evaluations (only during actual benchmark, not warmup) + if (!isWarmup) + { + lock (lockObject) + { + policyCounters[PolicyNames[policyIdx]]++; + } + } + } + catch (Exception) + { + // Ignore evaluation errors for benchmarking purposes + } + + evaluationCount++; + } + + // Store the actual evaluation time for this thread + if (!isWarmup) + { + lock (lockObject) + { + if (!evaluationTimes.ContainsKey(tid)) + evaluationTimes[tid] = TimeSpan.Zero; + evaluationTimes[tid] = localEvaluationTime; + } + } + }); + } + + // Stop execution after the specified duration + Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ContinueWith(_ => stopExecution = true); + + Task.WaitAll(tasks); + stopwatch.Stop(); + + var totalEvaluations = policyCounters.Values.Sum(); + var totalEvaluationTime = evaluationTimes.Values.Aggregate(TimeSpan.Zero, (sum, time) => sum + time); + + // Use pure evaluation time (consistent with Rust benchmark) + var evaluationTime = totalEvaluationTime == TimeSpan.Zero ? stopwatch.Elapsed : totalEvaluationTime; + + return (totalEvaluations, evaluationTime, policyCounters); + } + } +} diff --git a/bindings/csharp/Benchmarks/EngineEvaluationBenchmark.cs b/bindings/csharp/Benchmarks/EngineEvaluationBenchmark.cs new file mode 100644 index 00000000..2db8a13b --- /dev/null +++ b/bindings/csharp/Benchmarks/EngineEvaluationBenchmark.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Regorus; + +namespace Benchmarks +{ + public class EngineEvaluationBenchmark + { + private static readonly string TestDataPath = Path.Combine( + Directory.GetCurrentDirectory(), + "..", "..", "..", + "benches", "evaluation", "test_data" + ); + + private static readonly (string PolicyFile, string[] InputFiles)[] PolicyInputFiles = new[] + { + ("rbac_policy.rego", new[] { "rbac_input.json", "rbac_input2.json", "rbac_input3.json" }), + ("api_access_policy.rego", new[] { "api_access_input.json", "api_access_input2.json", "api_access_input3.json" }), + ("data_sensitivity_policy.rego", new[] { "data_sensitivity_input.json", "data_sensitivity_input2.json", "data_sensitivity_input3.json" }), + ("time_based_policy.rego", new[] { "time_based_input.json", "time_based_input2.json", "time_based_input3.json" }), + ("data_processing_policy.rego", new[] { "data_processing_input.json", "data_processing_input2.json", "data_processing_input3.json" }), + ("azure_vm_policy.rego", new[] { "azure_vm_input.json", "azure_vm_input2.json", "azure_vm_input3.json" }), + ("azure_storage_policy.rego", new[] { "azure_storage_input.json", "azure_storage_input2.json", "azure_storage_input3.json" }), + ("azure_keyvault_policy.rego", new[] { "azure_keyvault_input.json", "azure_keyvault_input2.json", "azure_keyvault_input3.json" }), + ("azure_nsg_policy.rego", new[] { "azure_nsg_input.json", "azure_nsg_input2.json", "azure_nsg_input3.json" }) + }; + + private static readonly string[] PolicyNames = new[] + { + "rbac_policy", + "api_access_policy", + "data_sensitivity_policy", + "time_based_policy", + "data_processing_policy", + "azure_vm_policy", + "azure_storage_policy", + "azure_keyvault_policy", + "azure_nsg_policy" + }; + + private static List<(string Policy, string[] Inputs)> LoadPoliciesWithInputs() + { + var result = new List<(string Policy, string[] Inputs)>(); + + foreach (var (policyFile, inputFiles) in PolicyInputFiles) + { + var policyPath = Path.Combine(TestDataPath, "policies", policyFile); + var policy = File.ReadAllText(policyPath); + + var inputs = inputFiles.Select(inputFile => + { + var inputPath = Path.Combine(TestDataPath, "inputs", inputFile); + return File.ReadAllText(inputPath); + }).ToArray(); + + result.Add((policy, inputs)); + } + + return result; + } + + private static List PrepareClonedEngines() + { + var policiesWithInputs = LoadPoliciesWithInputs(); + var engines = new List(); + + foreach (var (policy, _) in policiesWithInputs) + { + var engine = new Engine(); + engine.AddPolicy("policy.rego", policy); + + // Warm up the engine to ensure it's fully prepared for evaluation + // This prevents each cloned engine from repeating preparation work + engine.SetInputJson("{}"); + try + { + engine.EvalRule("data.bench.allow"); + } + catch + { + // Ignore warmup errors + } + + engines.Add(engine); + } + + return engines; + } + + public static void RunEngineEvaluationBenchmark() + { + var cpuCount = Environment.ProcessorCount; + var maxThreads = cpuCount * 2; + var threadCounts = new List { 1, 2 }; + + // Add even numbers from 4 to maxThreads + for (int i = 4; i <= maxThreads; i += 2) + { + threadCounts.Add(i); + } + + Console.WriteLine($"Running engine benchmark with max_threads: {maxThreads}"); + Console.WriteLine($"Testing with thread counts: {string.Join(", ", threadCounts)}"); + Console.WriteLine(); + + // Benchmark both cloned engines and fresh engines + var configurations = new[] + { + (true, "cloned_engines"), + (false, "fresh_engines") + }; + + foreach (var (useClonedEngines, groupName) in configurations) + { + Console.WriteLine($"=== {groupName} ==="); + + foreach (var threads in threadCounts) + { + RunEngineEvaluationBenchmark(threads, useClonedEngines, groupName); + } + Console.WriteLine(); + } + } + + public static void RunEngineEvaluationBenchmark(int threads, bool useClonedEngines, string groupName) + { + const int warmupSeconds = 3; + const int durationSeconds = 3; + var policiesWithInputs = LoadPoliciesWithInputs(); + + Console.WriteLine($"Warming up with {threads} threads for {warmupSeconds} seconds..."); + + // Warmup phase + var (_, _, _) = RunBenchmarkPhase(threads, warmupSeconds, policiesWithInputs, useClonedEngines, isWarmup: true); + + Console.WriteLine($"Running benchmark with {threads} threads for {durationSeconds} seconds..."); + + // Actual benchmark phase + var (totalEvaluations, evaluationTime, policyCounters) = RunBenchmarkPhase(threads, durationSeconds, policiesWithInputs, useClonedEngines, isWarmup: false); + + // Calculate throughput based on pure evaluation time (consistent with Rust benchmark) + var evalsPerSecond = totalEvaluations / evaluationTime.TotalSeconds; + var kelemsPerSecond = evalsPerSecond / 1000.0; + + Console.WriteLine($"{groupName}/eval/{threads} threads"); + Console.WriteLine($" time: [{evaluationTime.TotalMilliseconds:F2} ms]"); + Console.WriteLine($" thrpt: [{kelemsPerSecond:F2} Kelem/s]"); + + // Verify that all policies were evaluated + var allEvaluated = policyCounters.Values.All(count => count > 0); + + if (allEvaluated) + { + Console.WriteLine("✓ All policies were evaluated successfully"); + } + else + { + Console.WriteLine("ERROR: Some policies were never evaluated successfully!"); + } + } + + private static (int totalEvaluations, TimeSpan evaluationTime, Dictionary policyCounters) RunBenchmarkPhase( + int threads, + int durationSeconds, + List<(string Policy, string[] Inputs)> policiesWithInputs, + bool useClonedEngines, + bool isWarmup) + { + var barrier = new Barrier(threads); + var tasks = new Task[threads]; + var policyCounters = new Dictionary(); + var evaluationTimes = new Dictionary(); + var lockObject = new object(); + var stopExecution = false; + + // Initialize counters + foreach (var policyName in PolicyNames) + { + policyCounters[policyName] = 0; + } + + // Pre-create engines if using cloned engines + List? clonedEngines = null; + if (useClonedEngines) + { + clonedEngines = PrepareClonedEngines(); + } + + var stopwatch = Stopwatch.StartNew(); + + for (int threadId = 0; threadId < threads; threadId++) + { + int tid = threadId; + tasks[threadId] = Task.Run(() => + { + barrier.SignalAndWait(); + + int evaluationCount = 0; + var localEvaluationTime = TimeSpan.Zero; + + while (!stopExecution) + { + // Use different policy for each iteration + int policyIdx = (tid + evaluationCount) % policiesWithInputs.Count; + var (policy, inputs) = policiesWithInputs[policyIdx]; + + // Use different input for the same policy based on iteration + int inputIdx = evaluationCount % inputs.Length; + var input = inputs[inputIdx]; + + try + { + // Measure only the engine operations + var evalStopwatch = Stopwatch.StartNew(); + + Engine engine; + if (useClonedEngines) + { + engine = clonedEngines![policyIdx].Clone(); + } + else + { + engine = new Engine(); + engine.AddPolicy("policy.rego", policy); + } + + engine.SetInputJson(input); + var result = engine.EvalRule("data.bench.allow"); + engine.Dispose(); + + evalStopwatch.Stop(); + localEvaluationTime += evalStopwatch.Elapsed; + + // Track successful evaluations (only during actual benchmark, not warmup) + if (!isWarmup) + { + lock (lockObject) + { + policyCounters[PolicyNames[policyIdx]]++; + } + } + } + catch (Exception) + { + // Ignore evaluation errors for benchmarking purposes + } + + evaluationCount++; + } + + // Store the actual evaluation time for this thread + if (!isWarmup) + { + lock (lockObject) + { + if (!evaluationTimes.ContainsKey(tid)) + evaluationTimes[tid] = TimeSpan.Zero; + evaluationTimes[tid] = localEvaluationTime; + } + } + }); + } + + // Stop execution after the specified duration + Task.Delay(TimeSpan.FromSeconds(durationSeconds)).ContinueWith(_ => stopExecution = true); + + Task.WaitAll(tasks); + stopwatch.Stop(); + + // Clean up cloned engines if we created them + if (clonedEngines != null) + { + foreach (var engine in clonedEngines) + { + engine.Dispose(); + } + } + + var totalEvaluations = policyCounters.Values.Sum(); + var totalEvaluationTime = evaluationTimes.Values.Aggregate(TimeSpan.Zero, (sum, time) => sum + time); + + // Use pure evaluation time (consistent with Rust benchmark) + var evaluationTime = totalEvaluationTime == TimeSpan.Zero ? stopwatch.Elapsed : totalEvaluationTime; + + return (totalEvaluations, evaluationTime, policyCounters); + } + } +} diff --git a/bindings/csharp/Benchmarks/Program.cs b/bindings/csharp/Benchmarks/Program.cs new file mode 100644 index 00000000..ef15f1ea --- /dev/null +++ b/bindings/csharp/Benchmarks/Program.cs @@ -0,0 +1,36 @@ +using System; + +namespace Benchmarks +{ + class Program + { + static void Main(string[] args) + { + Console.WriteLine("=== Regorus C# Benchmarks ===\n"); + + try + { + Console.WriteLine("Running Engine Evaluation Benchmark..."); + EngineEvaluationBenchmark.RunEngineEvaluationBenchmark(); + } + catch (Exception ex) + { + Console.WriteLine($"Engine benchmark failed: {ex.Message}"); + } + + Console.WriteLine("\n" + new string('=', 80) + "\n"); + + try + { + Console.WriteLine("Running Compiled Policy Evaluation Benchmark..."); + CompiledPolicyEvaluationBenchmark.RunCompiledPolicyEvaluationBenchmark(); + } + catch (Exception ex) + { + Console.WriteLine($"Compiled policy benchmark failed: {ex.Message}"); + } + + Console.WriteLine("\n=== Benchmarks Complete ==="); + } + } +} diff --git a/bindings/csharp/Benchmarks/compiled_policy_evaluation_benchmark.md b/bindings/csharp/Benchmarks/compiled_policy_evaluation_benchmark.md new file mode 100644 index 00000000..3ad6c90c --- /dev/null +++ b/bindings/csharp/Benchmarks/compiled_policy_evaluation_benchmark.md @@ -0,0 +1,103 @@ +# Compiled Policy Evaluation Benchmark Results (C#/.NET) + +## Test Environment +- **Platform**: Apple Silicon (M-Series) +- **CPU**: 16 cores +- **Architecture**: ARM64 (aarch64-apple-darwin) +- **.NET Version**: 8.0 +- **Benchmark Framework**: Custom time-based benchmarking +- **Test Data**: 20,000 inputs per evaluation (distributed across threads) +- **Policy**: Complex authorization policy with nested rules +- **Warmup Duration**: 3 seconds per configuration +- **Evaluation Duration**: 3 seconds per configuration + +## Benchmark Overview + +The C# compiled policy evaluation benchmark tests Regorus compiled policy performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of compiled policy compilation strategies. + +## Configuration Combinations + +1. **Compiled Shared Policies**: All threads share pre-compiled policy instances - optimal for performance +2. **Compiled Per Iteration**: Each thread compiles the policy for each evaluation iteration + +*Note: The C# implementation uses a simpler configuration model compared to Rust, which also varies input data handling (cloned vs fresh inputs). The C# benchmarks focus on compilation strategies with consistent input handling.* + +## Performance Results + +### Compiled Shared Policies (Best Performance) +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 2928.81 | 211 | +| 2 | 5892.53 | 146 | +| 4 | 11750.71 | 155 | +| 6 | 17686.92 | 134 | +| 8 | 23543.53 | 90 | +| 10 | 29503.80 | 72 | +| 12 | 35494.81 | 58 | +| 14 | 41408.36 | 50 | +| 16 | 47333.65 | 44 | +| 18 | 53050.24 | 38 | +| 20 | 58807.20 | 34 | +| 22 | 406022.45 | 32 | +| 24 | 65480.69 | 32 | +| 26 | 70952.34 | 30 | +| 28 | 72064.03 | 30 | +| 30 | 492405.74 | 27 | +| 32 | 81210.83 | 27 | + +### Compiled Per Iteration +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 2984.00 | 39 | +| 2 | 5969.45 | 38 | +| 4 | 11948.28 | 32 | +| 6 | 17927.24 | 30 | +| 8 | 23889.01 | 24 | +| 10 | 29882.38 | 20 | +| 12 | 35865.06 | 18 | +| 14 | 41838.70 | 15 | +| 16 | 47800.92 | 14 | +| 18 | 53257.22 | 10 | +| 20 | 59596.93 | 11 | +| 22 | 435853.41 | 10 | +| 24 | 70870.86 | 9 | +| 26 | 76120.59 | 9 | +| 28 | 80717.51 | 8 | +| 30 | 544207.96 | 8 | +| 32 | 91540.91 | 7 | + +## Analysis + +The C# compiled policy benchmark demonstrates important performance characteristics: + +1. **Compilation Strategy Impact**: Shared compiled policies significantly outperform per-iteration compilation (~5.4x at 1 thread) +2. **Scaling Patterns**: + - Best throughput achieved at 1 thread for shared policies + - Performance generally degrades with increased thread count +3. **Performance Hierarchy**: + - Shared compiled policies: Best performance (optimal configuration) + - Per-iteration compilation: ~82% reduction from optimal +4. **Compilation Overhead**: Per-iteration compilation creates substantial overhead, similar to fresh engine creation +5. **Thread Contention**: Significant performance degradation beyond 8 threads for both configurations + +## Comparison with Rust Compiled Policy Evaluation + +| Configuration | C# Performance (1 thread) | Rust Performance (1 thread) | Relative Performance | +|:-----------------|:----------------------------|:-----------------------------|---------------------:| +| Shared Policies | Best performance | Higher throughput | 0.40x-0.70x | +| Per-iteration | ~82% reduction from optimal | ~85% reduction from optimal | 0.47x-0.89x | + +*Note: Rust benchmarks include additional input data variations (cloned vs fresh inputs) that are not present in the C# implementation.* + +## Comparison with C# Engine Evaluation + +| Configuration | Compiled Policy (1 thread) | Engine Evaluation (1 thread) | Performance Ratio | +|:---------------|:----------------------------|:------------------------------|------------------:| +| Optimal Config | Best performance | Slightly higher throughput | 0.96x | + +## Performance Insights + +1. **Compilation Efficiency**: Pre-compiled policies provide massive performance benefits over per-iteration compilation +2. **C# Performance Gap**: C# compiled policies achieve 40%-70% of Rust performance for shared policies +3. **Engine vs Compiled**: In C#, engine evaluation slightly outperforms compiled policies (96%-104% range) + diff --git a/bindings/csharp/Benchmarks/engine_evaluation_benchmark.md b/bindings/csharp/Benchmarks/engine_evaluation_benchmark.md new file mode 100644 index 00000000..62c8d450 --- /dev/null +++ b/bindings/csharp/Benchmarks/engine_evaluation_benchmark.md @@ -0,0 +1,99 @@ +# Engine Evaluation Benchmark Results (C#/.NET) + +## Test Environment +- **Platform**: Apple Silicon (M-Series) +- **CPU**: 16 cores +- **Architecture**: ARM64 (aarch64-apple-darwin) +- **.NET Version**: 8.0 +- **Benchmark Framework**: Custom time-based benchmarking +- **Test Data**: 20,000 inputs per evaluation (distributed across threads) +- **Policy**: Complex authorization policy with nested rules +- **Warmup Duration**: 3 seconds per configuration +- **Evaluation Duration**: 3 seconds per configuration + +## Benchmark Overview + +The C# engine evaluation benchmark tests Regorus policy evaluation performance across multiple thread configurations (1-32 threads). It measures throughput (thousands of evaluations per second) for different combinations of engine reuse strategies. + +## Configuration Combinations + +1. **Cloned Engines**: Each thread uses its own cloned engine instance - optimal for performance +2. **Fresh Engines**: Each thread creates a new engine for each evaluation iteration + +*Note: The C# implementation uses a simpler configuration model compared to Rust, which also varies input data handling (cloned vs fresh inputs). The C# benchmarks focus on engine reuse strategies with consistent input handling.* + +## Performance Results + +### Cloned Engines (Best Performance) +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 2930.56 | 219 | +| 2 | 5868.46 | 177 | +| 4 | 11771.01 | 146 | +| 6 | 17682.52 | 129 | +| 8 | 23633.65 | 78 | +| 10 | 29489.12 | 67 | +| 12 | 35455.23 | 57 | +| 14 | 41353.65 | 47 | +| 16 | 47378.91 | 42 | +| 18 | 52750.68 | 36 | +| 20 | 58131.31 | 35 | +| 22 | 62964.88 | 31 | +| 24 | 64337.75 | 34 | +| 26 | 70044.96 | 29 | +| 28 | 72553.98 | 28 | +| 30 | 79323.25 | 26 | +| 32 | 78624.33 | 26 | + +### Fresh Engines +| Threads | Total Evaluation Time (ms) | Throughput (Kelem/s) | +|--------:|---------------------------:|---------------------:| +| 1 | 2985.49 | 41 | +| 2 | 5968.13 | 38 | +| 4 | 11942.10 | 34 | +| 6 | 17918.75 | 32 | +| 8 | 23873.57 | 25 | +| 10 | 29863.85 | 20 | +| 12 | 35823.98 | 19 | +| 14 | 41811.53 | 16 | +| 16 | 47819.89 | 14 | +| 18 | 53478.32 | 13 | +| 20 | 59191.93 | 12 | +| 22 | 64630.71 | 11 | +| 24 | 70215.54 | 10 | +| 26 | 75732.06 | 9 | +| 28 | 80897.59 | 9 | +| 30 | 949904.84 | 8 | +| 32 | 92592.64 | 8 | + +## Analysis + +The C# benchmark results demonstrate important performance characteristics: + +1. **Engine Reuse Impact**: Cloned engines significantly outperform fresh engines (~5.3x at 1 thread) +2. **Scaling Patterns**: + - Best throughput achieved at 1 thread for both configurations + - Performance degrades with increased thread count due to contention + - Cloned engines show better relative scaling characteristics +3. **Performance Hierarchy**: + - Cloned engines: Best performance (optimal configuration) + - Fresh engines: ~81% reduction from optimal +4. **Thread Contention**: Significant performance drop beyond 8 threads, especially for fresh engines +5. **C# vs Rust Performance**: C# shows ~67% of Rust performance for equivalent cloned engine configuration + +## Comparison with Rust Engine Evaluation + +| Configuration | C# Performance (1 thread) | Rust Performance (1 thread) | Relative Performance | +|:---------------|:---------------------------|:-----------------------------|---------------------:| +| Cloned Engines | Best performance | Higher throughput | 0.67x-0.92x | +| Fresh Engines | ~81% reduction from optimal| ~87% reduction from optimal | 0.75x-0.95x | + +*Note: Rust benchmarks include additional input data variations (cloned vs fresh inputs) that are not present in the C# implementation.* + +## Performance Insights + +1. **Engine Creation Overhead**: Fresh engine creation has massive performance impact in C# (~5.3x slower) +2. **Thread Scaling**: C# shows more significant thread contention than Rust implementation +3. **Memory Management**: .NET garbage collection may contribute to performance variations +4. **Interop Overhead**: C# bindings add measurable overhead compared to native Rust + diff --git a/bindings/ffi/Cargo.lock b/bindings/ffi/Cargo.lock index 97b5558b..c9585cfb 100644 --- a/bindings/ffi/Cargo.lock +++ b/bindings/ffi/Cargo.lock @@ -76,7 +76,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -87,7 +87,7 @@ checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -125,9 +125,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -178,18 +178,18 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -313,7 +313,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -813,9 +813,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -981,7 +981,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -1050,9 +1050,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1114,9 +1114,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -1136,31 +1136,31 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ "fastrand", "getrandom", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -1414,15 +1414,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/bindings/java/Cargo.lock b/bindings/java/Cargo.lock index dd24a455..0accae45 100644 --- a/bindings/java/Cargo.lock +++ b/bindings/java/Cargo.lock @@ -75,9 +75,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -115,9 +115,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] @@ -130,9 +130,9 @@ checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -689,9 +689,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -831,7 +831,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.14", + "thiserror 2.0.16", "url", "uuid", ] @@ -921,9 +921,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -970,9 +970,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -1001,11 +1001,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.14", + "thiserror-impl 2.0.16", ] [[package]] @@ -1021,9 +1021,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -1183,11 +1183,11 @@ dependencies = [ [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1260,11 +1260,11 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.53.3", ] [[package]] @@ -1291,13 +1291,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -1310,6 +1327,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -1322,6 +1345,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -1334,12 +1363,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -1352,6 +1393,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -1364,6 +1411,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -1376,6 +1429,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -1388,6 +1447,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 714923a4..616e58c0 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -75,9 +75,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -109,18 +109,18 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -681,9 +681,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -969,9 +969,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1018,9 +1018,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -1046,18 +1046,18 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", diff --git a/bindings/ruby/Cargo.lock b/bindings/ruby/Cargo.lock index 9ccb7958..8ead6794 100644 --- a/bindings/ruby/Cargo.lock +++ b/bindings/ruby/Cargo.lock @@ -95,9 +95,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -129,9 +129,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -761,9 +761,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -1026,9 +1026,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1092,9 +1092,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -1120,18 +1120,18 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", diff --git a/bindings/wasm/Cargo.lock b/bindings/wasm/Cargo.lock index b2fd31e5..9c58e1ac 100644 --- a/bindings/wasm/Cargo.lock +++ b/bindings/wasm/Cargo.lock @@ -75,9 +75,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" [[package]] name = "borrow-or-share" @@ -109,18 +109,18 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "cc" -version = "1.2.32" +version = "1.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2352e5597e9c544d5e6d9c95190d5d27738ade584fa8db0a16e130e5c2b5296e" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" dependencies = [ "shlex", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "chrono" @@ -623,7 +623,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -670,9 +670,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.97" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61789d7719defeb74ea5fe81f2fdfdbd28a803847077cecce2ff14e1472f6f1" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -905,9 +905,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -954,9 +954,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.105" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3fcb250e53458e712715cf74285c1f889686520d79294a9ef3bd7aa1fc619" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -976,18 +976,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0949c3a6c842cbde3f1686d6eea5a010516deb7085f79db747562d4102f41e" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.14" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b44b4ab9c2fdd0e0512e6bece8388e214c0749f5862b114cc5b7a25daf227" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -1200,9 +1200,9 @@ dependencies = [ [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ "windows-sys", ] @@ -1268,11 +1268,11 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets", + "windows-targets 0.53.3", ] [[package]] @@ -1281,14 +1281,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -1297,48 +1314,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/src/interpreter.rs b/src/interpreter.rs index 74fe0757..1c2052e3 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -228,22 +228,22 @@ impl Interpreter { init_data: Value::new_object(), with_document: Value::new_object(), - with_functions: BTreeMap::new(), - scopes: vec![Scope::new()], - contexts: vec![], - loop_var_values: BTreeMap::new(), + with_functions: BTreeMap::default(), + scopes: Vec::default(), + contexts: Vec::default(), + loop_var_values: BTreeMap::default(), processed: BTreeSet::default(), processed_paths: Value::new_object(), rule_values: BTreeMap::default(), - active_rules: vec![], - builtins_cache: BTreeMap::new(), + active_rules: Vec::default(), + builtins_cache: BTreeMap::default(), no_rules_lookup: false, traces: None, - extensions: Map::new(), + extensions: Map::default(), #[cfg(feature = "coverage")] - coverage: Map::new(), + coverage: Map::default(), #[cfg(feature = "coverage")] enable_coverage: false, @@ -254,16 +254,42 @@ impl Interpreter { /// Create a new Interpreter from a compiled policy. pub fn new_from_compiled_policy(compiled_policy: Rc) -> Self { - let mut interpreter = Self::new(); - interpreter.extensions = compiled_policy.extensions.clone(); - interpreter.compiled_policy = compiled_policy; + Self { + data: Value::new_object(), + module: None, - // Set initial data if available - if let Some(data) = &interpreter.compiled_policy.data { - interpreter.init_data = data.clone(); - } + current_module_path: String::default(), + input: Value::Undefined, + + with_document: Value::new_object(), + with_functions: BTreeMap::default(), + scopes: Vec::default(), + contexts: Vec::default(), + loop_var_values: BTreeMap::default(), - interpreter + processed: BTreeSet::default(), + processed_paths: Value::new_object(), + rule_values: BTreeMap::default(), + active_rules: Vec::default(), + builtins_cache: BTreeMap::default(), + no_rules_lookup: false, + traces: None, + + #[cfg(feature = "coverage")] + coverage: Map::default(), + #[cfg(feature = "coverage")] + enable_coverage: false, + + gather_prints: false, + prints: Vec::default(), + + extensions: compiled_policy.extensions.clone(), + compiled_policy: compiled_policy.clone(), + init_data: compiled_policy + .data + .clone() + .unwrap_or_else(Value::new_object), + } } fn compiled_policy_mut(&mut self) -> &mut CompiledPolicyData { @@ -338,6 +364,7 @@ impl Interpreter { self.scopes = vec![Scope::new()]; self.contexts = vec![]; self.rule_values.clear(); + self.builtins_cache.clear(); } fn current_module(&self) -> Result> {