feat(cargo): reuse native compiler outputs across clean roots

Expand verified reuse to non-incremental dependency and workspace metadata/rlib units on graduated Apple Silicon macOS and ARM64 Linux. Preserve fail-closed class boundaries and existing wrappers, and add representative fixtures plus reproducible performance evidence.

Author: loadingalias

.changes/native-compiler-cache-9fd7.md

@@ -2,4 +2,11 @@
 "cargo-rail" = "minor"
 ---
 
-Add compiler observation and verified native compiler-result caching for non-incremental workspace library metadata checks on Apple Silicon macOS with Cargo/rustc 1.97.1, preserving custom wrappers and sccache while bypassing unproven classes. Unsupported platforms remain buildable and execute cold through explicit fail-closed boundaries.
+Add portable, verified native compiler-result caching for non-incremental dependency and workspace library
+metadata/rlib units on Apple Silicon macOS and ARM64 Linux with Cargo/rustc 1.97.1. Ordinary `cargo rail run` check
+and build actions can reuse byte-exact outputs across clean roots without restoring or fabricating Cargo target state,
+incremental state, or fingerprints.
+
+Preserve custom wrappers and sccache, keep incremental builds and unproven linker/build-script/proc-macro classes
+explicitly bypassed, and fail closed on input, toolchain, environment, SDK/linker, cache-object, and output mutations.
+Add a representative registry/Git/native/proc-macro fixture plus reproducible cold/warm benchmarks and cache evidence.

.config/nextest.toml

@@ -11,6 +11,11 @@ retries = 0
 [test-groups]
 cargo-subprocess = { max-threads = 2 }
 
+[[profile.default.overrides]]
+filter = 'binary(integration) & test(/test_native_cache_fixture::/)'
+test-group = 'cargo-subprocess'
+slow-timeout = { period = "30s", terminate-after = 6 }
+
 [[profile.default.overrides]]
 filter = 'binary(integration) & test(/test_(instrumentation|scale_fixture|source_snapshot|unify|unused_detection|undeclared_features)/)'
 test-group = 'cargo-subprocess'
@@ -25,6 +30,10 @@ slow-timeout = { period = "60s", terminate-after = 4 }
 leak-timeout = { period = "2s", result = "fail" }
 retries = 0
 
+[[profile.commit.overrides]]
+filter = 'binary(integration) & test(/test_native_cache_fixture::/)'
+test-group = 'cargo-subprocess'
+
 [[profile.commit.overrides]]
 filter = 'binary(integration) & test(/test_(instrumentation|scale_fixture|source_snapshot|unify|unused_detection|undeclared_features)/)'
 test-group = 'cargo-subprocess'

justfile

@@ -24,6 +24,9 @@ build-all:
 bench-unify packages="25" runs="10":
     @scripts/bench/unify.sh "{{ packages }}" "{{ runs }}"
 
+bench-native-cache runs="10":
+    @scripts/bench/native-cache.sh "{{ runs }}"
+
 gen-fixture members output:
     @scripts/fixtures/generate-workspace.sh "{{ members }}" "{{ output }}"
 

scripts/bench/native-cache.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+timing_dir="${CARGO_RAIL_UNIT_TIMINGS:?CARGO_RAIL_UNIT_TIMINGS must name an output directory}"
+workspace="${CARGO_RAIL_BENCH_WORKSPACE:?CARGO_RAIL_BENCH_WORKSPACE must name the fixture root}"
+mkdir -p "$timing_dir"
+record="$timing_dir/unit-$$"
+
+crate_name="unknown"
+crate_type="unknown"
+emit="unknown"
+source="unknown"
+test_mode="false"
+native_link="false"
+incremental="false"
+
+arguments=("$@")
+index=1
+while ((index < ${#arguments[@]})); do
+  argument="${arguments[$index]}"
+  next=""
+  if ((index + 1 < ${#arguments[@]})); then
+    next="${arguments[$((index + 1))]}"
+  fi
+  case "$argument" in
+    --crate-name)
+      crate_name="$next"
+      ((index += 2))
+      continue
+      ;;
+    --crate-name=*) crate_name="${argument#--crate-name=}" ;;
+    --crate-type)
+      crate_type="$next"
+      ((index += 2))
+      continue
+      ;;
+    --crate-type=*) crate_type="${argument#--crate-type=}" ;;
+    --emit)
+      emit="$next"
+      ((index += 2))
+      continue
+      ;;
+    --emit=*) emit="${argument#--emit=}" ;;
+    --test) test_mode="true" ;;
+    -l | -l* | -Lnative* | *linker=* | *link-arg=* | *link-args=*) native_link="true" ;;
+    *incremental=*) incremental="true" ;;
+    *.rs)
+      if [[ "$source" == unknown ]]; then
+        source="$argument"
+      fi
+      ;;
+  esac
+  ((index += 1))
+done
+
+case "$source" in
+  "$workspace"/*) source_class="workspace" ;;
+  */registry/src/*) source_class="registry" ;;
+  */git/checkouts/*) source_class="git" ;;
+  /*) source_class="host" ;;
+  *.rs) source_class="workspace" ;;
+  *) source_class="host" ;;
+esac
+
+printf '%s\0' "$@" >"$record.argv"
+/usr/bin/time -p -o "$record.time" "$@"
+status=$?
+real="$(awk '$1 == "real" { print $2 }' "$record.time")"
+user="$(awk '$1 == "user" { print $2 }' "$record.time")"
+sys="$(awk '$1 == "sys" { print $2 }' "$record.time")"
+printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+  "$crate_name" "$crate_type" "$emit" "$source_class" "$test_mode" "$native_link" "$incremental" \
+  "${real:-0}" "${user:-0}" "${sys:-0}" "$status" >"$record.tsv"
+exit "$status"

scripts/bench/rustc-timing-wrapper.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  echo "usage: $0 <destination> [shared-git-source]" >&2
+  exit 2
+}
+
+[[ $# -ge 1 && $# -le 2 ]] || usage
+destination="$1"
+[[ -n "$destination" && "$destination" != -* ]] || usage
+shared_git_source="${2:-${destination}.git-source}"
+[[ -n "$shared_git_source" && "$shared_git_source" != -* ]] || usage
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+template="$repo_root/tests/fixtures/native_cache/real_world"
+
+if [[ -e "$destination" ]]; then
+  [[ -d "$destination" ]] || { echo "fixture destination is not a directory: $destination" >&2; exit 2; }
+else
+  mkdir -p "$destination"
+fi
+destination="$(cd "$destination" && pwd -P)"
+[[ -z "$(find "$destination" -mindepth 1 -maxdepth 1 -print -quit)" ]] || {
+  echo "fixture destination must be empty: $destination" >&2
+  exit 2
+}
+
+cp -R "$template/." "$destination"
+
+git_source_template="$destination/git-source"
+if [[ -e "$shared_git_source" ]]; then
+  [[ -d "$shared_git_source/.git" ]] || {
+    echo "shared Git source is not a fixture repository: $shared_git_source" >&2
+    exit 2
+  }
+  rm -r -- "$git_source_template"
+else
+  mkdir -p "$(dirname "$shared_git_source")"
+  mv "$git_source_template" "$shared_git_source"
+  git -C "$shared_git_source" init --quiet --initial-branch=main --object-format=sha1
+  git -C "$shared_git_source" config core.autocrlf false
+  git -C "$shared_git_source" config core.filemode false
+  git -C "$shared_git_source" add --all
+  git_tree="$(git -C "$shared_git_source" write-tree)"
+  git_commit="$({
+    GIT_AUTHOR_NAME='cargo-rail fixture' \
+    GIT_AUTHOR_EMAIL='fixture@cargo-rail.invalid' \
+    GIT_AUTHOR_DATE='2000-01-01T00:00:00Z' \
+    GIT_COMMITTER_NAME='cargo-rail fixture' \
+    GIT_COMMITTER_EMAIL='fixture@cargo-rail.invalid' \
+    GIT_COMMITTER_DATE='2000-01-01T00:00:00Z' \
+      git -C "$shared_git_source" commit-tree "$git_tree" -m 'Create native-cache Git dependency'
+  })"
+  git -C "$shared_git_source" update-ref refs/heads/main "$git_commit"
+fi
+git_source="$(cd "$shared_git_source" && pwd -P)"
+git_commit="$(git -C "$git_source" rev-parse --verify HEAD)"
+
+manifest="$destination/Cargo.toml"
+rendered="$destination/Cargo.toml.rendered"
+git_url="file://$git_source"
+sed -e "s|__FIXTURE_GIT_URL__|$git_url|" -e "s|__FIXTURE_GIT_REV__|$git_commit|" "$manifest" >"$rendered"
+mv "$rendered" "$manifest"
+
+prefetch_manifest="$destination/git-prefetch/Cargo.toml"
+prefetch_rendered="$destination/git-prefetch/Cargo.toml.rendered"
+sed -e "s|__FIXTURE_GIT_URL__|$git_url|" -e "s|__FIXTURE_GIT_REV__|$git_commit|" \
+  "$prefetch_manifest" >"$prefetch_rendered"
+mv "$prefetch_rendered" "$prefetch_manifest"
+cargo metadata --manifest-path "$prefetch_manifest" --format-version=1 >/dev/null
+rm -r -- "$destination/git-prefetch"
+
+cargo generate-lockfile --manifest-path "$manifest" --offline --quiet
+git -C "$destination" init --quiet --initial-branch=main --object-format=sha1
+git -C "$destination" config core.autocrlf false
+git -C "$destination" config core.filemode false
+git -C "$destination" add --all
+fixture_tree="$(git -C "$destination" write-tree)"
+fixture_commit="$({
+  GIT_AUTHOR_NAME='cargo-rail fixture' \
+  GIT_AUTHOR_EMAIL='fixture@cargo-rail.invalid' \
+  GIT_AUTHOR_DATE='2000-01-02T00:00:00Z' \
+  GIT_COMMITTER_NAME='cargo-rail fixture' \
+  GIT_COMMITTER_EMAIL='fixture@cargo-rail.invalid' \
+  GIT_COMMITTER_DATE='2000-01-02T00:00:00Z' \
+    git -C "$destination" commit-tree "$fixture_tree" -m 'Materialize native-cache workload'
+})"
+git -C "$destination" update-ref refs/heads/main "$fixture_commit"
+
+echo "materialized native-cache fixture at $destination" >&2

scripts/fixtures/materialize-native-cache.sh

@@ -266,8 +266,8 @@ pub enum Commands {
     /// Execute supported Rust actions in fresh isolated roots
     #[arg(long)]
     hermetic: bool,
-    /// Disable local action-result reuse for this hermetic execution
-    #[arg(long, requires = "hermetic")]
+    /// Disable local result reuse for this execution
+    #[arg(long)]
     no_cache: bool,
     /// Dry-run action plan format (json/github require --dry-run)
     #[arg(long, short = 'f', default_value_t, value_enum)]

src/commands/cli.rs

@@ -10,6 +10,7 @@ use crate::action::{
 use crate::action_key::{analyze as analyze_action_key, resolution_identity};
 use crate::cargo::{ResolutionFeatures, ResolutionPackages, ResolutionRequest, TargetSpecificationIdentity};
 use crate::commands::common::{ActionOutputFormat, PlanOutputFormat, format_preview_list};
+use crate::compiler::collector::prepare_direct_cargo_action;
 use crate::config::MAX_ACTIONS;
 use crate::error::{RailError, RailResult};
 use crate::git::detect_default_base_ref;
@@ -1480,8 +1481,36 @@ fn run_or_print_action(opts: &RunOptions, ctx: &WorkspaceContext, action: &Expan
     .argv()
     .split_first()
     .ok_or_else(|| RailError::message("expanded action argv cannot be empty"))?;
+  let native_cache = if matches!(action.kind(), ActionKind::Build | ActionKind::Distribution) {
+    Some(if opts.no_cache {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("native_cache_disabled_by_request")
+    } else if cargo_cli_configuration_override(arguments) {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("cargo_cli_configuration_not_graduated")
+    } else if action_compiler_wrapper_capability(action) {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("action_compiler_wrapper_preserved")
+    } else if !action.environment().inherit() || !action.environment().entries().is_empty() {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("action_environment_not_graduated")
+    } else if std::env::var_os("CARGO_INCREMENTAL").as_deref() != Some(std::ffi::OsStr::new("0")) {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("native_cache_incremental_policy_not_graduated")
+    } else if std::env::var_os("RUSTC_FORCE_INCREMENTAL").is_some() {
+      crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("forced_incremental_compilation_not_graduated")
+    } else {
+      match prepare_direct_cargo_action(ctx.snapshot()?) {
+        Ok(setup) => setup,
+        Err(_) => crate::compiler::native_cache::DirectNativeCacheSetup::Bypassed("native_cache_identity_unavailable"),
+      }
+    })
+  } else {
+    None
+  };
   let mut command = Command::new(program);
   let working_directory = action.validate_paths(ctx.workspace_root())?;
+  if let Some(configuration) = native_cache
+    .as_ref()
+    .and_then(crate::compiler::native_cache::DirectNativeCacheSetup::cargo_config_argument)
+  {
+    command.arg("--config").arg(configuration);
+  }
   command.args(arguments).current_dir(&working_directory);
   if !action.environment().inherit() {
     command.env_clear();
@@ -1519,6 +1548,31 @@ fn run_or_print_action(opts: &RunOptions, ctx: &WorkspaceContext, action: &Expan
   let status = command
     .status()
     .map_err(|error| RailError::message(format!("{} failed: {}", action.id(), error)))?;
+  if opts.explain
+    && let Some(native_cache) = &native_cache
+  {
+    if let Some(reason) = native_cache.bypass_reason() {
+      println!("action `{}` native compiler cache: bypassed ({reason})", action.id());
+    } else if let Some(report) = native_cache.report() {
+      let reasons = report
+        .reasons
+        .iter()
+        .map(|(reason, count)| format!("{reason}={count}"))
+        .collect::<Vec<_>>()
+        .join(",");
+      println!(
+        "action `{}` native compiler cache: hits={} misses={} bypasses={} setup_bytes_hashed={} bytes_hashed={} bytes_restored={} reasons={}",
+        action.id(),
+        report.hits,
+        report.misses,
+        report.bypasses,
+        report.setup_bytes_hashed,
+        report.bytes_hashed,
+        report.bytes_restored,
+        reasons,
+      );
+    }
+  }
   if !status.success() {
     return Err(RailError::ExitWithCode {
       code: status.code().unwrap_or(1),
@@ -1527,6 +1581,27 @@ fn run_or_print_action(opts: &RunOptions, ctx: &WorkspaceContext, action: &Expan
   Ok(())
 }
 
+fn cargo_cli_configuration_override(arguments: &[String]) -> bool {
+  arguments
+    .iter()
+    .any(|argument| argument == "--config" || argument.starts_with("--config="))
+}
+
+fn action_compiler_wrapper_capability(action: &ExpandedAction) -> bool {
+  action.environment().entries().iter().any(|entry| {
+    let name = match entry {
+      ActionEnvironmentEntry::Fixed { name, .. }
+      | ActionEnvironmentEntry::Pass { name }
+      | ActionEnvironmentEntry::Secret { name }
+      | ActionEnvironmentEntry::Cargo { name, .. } => name,
+    };
+    matches!(
+      name.as_str(),
+      "RUSTC_WRAPPER" | "RUSTC_WORKSPACE_WRAPPER" | "CARGO_BUILD_RUSTC_WRAPPER" | "CARGO_BUILD_RUSTC_WORKSPACE_WRAPPER"
+    )
+  })
+}
+
 fn print_hermetic_result(
   action_id: &str,
   report: &crate::hermetic::HermeticExecutionReport,