diff --git a/CLAUDE.md b/CLAUDE.md index 9e43749c8..25602bbd5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,6 +9,7 @@ A monorepo task runner (like Nx/Turbo) with intelligent caching and dependency r - `crates/vt_graph` — Task dependency graph construction and config loading - `crates/vt_plan` — Execution planning (resolves env vars, working dirs, commands) - `crates/vt_workspace` — Workspace detection and package dependency graph +- `crates/vt_fs_fingerprint` — Filesystem fingerprinting for task caching (pre-run snapshot, traced-access judgment, cache-entry validation) - `crates/fspy*` — File system access tracing (9 crates: supervisor, preload libs, platform backends) - `crates/pty_terminal*` — Cross-platform headless terminal emulator (3 crates) - `crates/vt_path` — Type-safe absolute/relative path system diff --git a/Cargo.lock b/Cargo.lock index b5fc5484b..6d0cdb2a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4158,7 +4158,6 @@ dependencies = [ "owo-colors", "petgraph", "pty_terminal_test_client", - "rayon", "rusqlite", "rustc-hash", "serde", @@ -4170,9 +4169,9 @@ dependencies = [ "tokio", "tokio-util", "tracing", - "twox-hash", "uuid", "vt_client_napi", + "vt_fs_fingerprint", "vt_glob", "vt_graph", "vt_ipc_shared", @@ -4182,7 +4181,6 @@ dependencies = [ "vt_server", "vt_str", "vt_workspace", - "wax", "winapi", "wincode", "zstd", @@ -4258,6 +4256,28 @@ dependencies = [ "vt_str", ] +[[package]] +name = "vt_fs_fingerprint" +version = "0.0.0" +dependencies = [ + "anyhow", + "fspy_shared", + "nix 0.31.2", + "rayon", + "rustc-hash", + "serde", + "tempfile", + "thiserror 2.0.18", + "tracing", + "twox-hash", + "vt_glob", + "vt_graph", + "vt_path", + "vt_str", + "wax", + "wincode", +] + [[package]] name = "vt_glob" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index ee6abee66..ddc39daaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,6 +149,7 @@ tui-term = "0.3.1" twox-hash = "2.1.1" uuid = "1.18.1" vec1 = "1.12.1" +vt_fs_fingerprint = { path = "crates/vt_fs_fingerprint" } vt_glob = { path = "crates/vt_glob" } vt_graph_ser = { path = "crates/vt_graph_ser" } vt_path = { path = "crates/vt_path" } diff --git a/crates/vt/Cargo.toml b/crates/vt/Cargo.toml index f12d53bfa..5040c8ac2 100644 --- a/crates/vt/Cargo.toml +++ b/crates/vt/Cargo.toml @@ -25,7 +25,6 @@ once_cell = { workspace = true } owo-colors = { workspace = true } petgraph = { workspace = true } pty_terminal_test_client = { workspace = true } -rayon = { workspace = true } rusqlite = { workspace = true, features = ["bundled"] } rustc-hash = { workspace = true } serde = { workspace = true, features = ["derive", "rc"] } @@ -43,9 +42,9 @@ tokio = { workspace = true, features = [ ] } tokio-util = { workspace = true } tracing = { workspace = true } -twox-hash = { workspace = true } materialized_artifact = { workspace = true } uuid = { workspace = true, features = ["v4"] } +vt_fs_fingerprint = { workspace = true } vt_glob = { workspace = true } vt_path = { workspace = true } vt_select = { workspace = true } @@ -55,7 +54,6 @@ vt_ipc_shared = { workspace = true } vt_plan = { workspace = true } vt_server = { workspace = true } vt_workspace = { workspace = true } -wax = { workspace = true } zstd = { workspace = true } # Artifact build-deps must be unconditional: cargo's resolver panics when diff --git a/crates/vt/docs/task-cache.md b/crates/vt/docs/task-cache.md index c26838859..a74173cad 100644 --- a/crates/vt/docs/task-cache.md +++ b/crates/vt/docs/task-cache.md @@ -567,18 +567,19 @@ Each `&&` separated command is cached independently. If only terser config chang ### Core Cache Components ``` +crates/vt_fs_fingerprint/src/ # The run's filesystem story (own crate) +├── task_run.rs # TaskFs (pre_run/post_run), Conclusion +├── fingerprint.rs # InputFingerprints, PathFingerprint, InputChange +├── tracked_accesses.rs # fspy access normalization +├── glob.rs # Glob walking + input hashing +└── hash.rs # Content hashing + crates/vt/src/session/ ├── cache/ │ ├── mod.rs # ExecutionCache, CacheEntryKey/Value, FingerprintMismatch │ └── display.rs # Cache status display formatting ├── execute/ │ ├── mod.rs # execute_spawn, SpawnOutcome -│ ├── task_fs/ # The run's filesystem story -│ │ ├── task_run.rs # TaskFs (pre_run/post_run), Conclusion -│ │ ├── fingerprint.rs # InputFingerprints, PathFingerprint, InputChange -│ │ ├── tracked_accesses.rs # fspy access normalization -│ │ ├── glob.rs # Glob walking + input hashing -│ │ └── hash.rs # Content hashing │ ├── cache_update.rs # Post-run cache update decision │ ├── post_run.rs # TrackedEnvFingerprints (tracked env validation) │ └── spawn.rs # spawn_with_tracking, fspy integration diff --git a/crates/vt/src/session/cache/mod.rs b/crates/vt/src/session/cache/mod.rs index 597fbfd37..29297221d 100644 --- a/crates/vt/src/session/cache/mod.rs +++ b/crates/vt/src/session/cache/mod.rs @@ -14,6 +14,8 @@ pub use display::{ use rusqlite::{Connection, OptionalExtension as _}; use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; +pub use vt_fs_fingerprint::InputChangeKind; +use vt_fs_fingerprint::{InputChange, InputFingerprints}; use vt_graph::config::ResolvedGlobConfig; use vt_path::{AbsolutePath, RelativePathBuf}; use vt_plan::cache_metadata::{CacheMetadata, ExecutionCacheKey, SpawnFingerprint}; @@ -25,11 +27,9 @@ use wincode::{ io::{Reader, Writer}, }; -pub use super::execute::task_fs::InputChangeKind; use super::execute::{ pipe::StdOutput, post_run::{PostRunMismatch, TrackedEnvFingerprints, TrackedEnvQuery}, - task_fs::{InputChange, InputFingerprints}, }; const TASK_CACHE_PREALLOCATION_SIZE_LIMIT: usize = 256 * 1024 * 1024; diff --git a/crates/vt/src/session/execute/cache_update.rs b/crates/vt/src/session/execute/cache_update.rs index 24026c752..8fc8983dd 100644 --- a/crates/vt/src/session/execute/cache_update.rs +++ b/crates/vt/src/session/execute/cache_update.rs @@ -4,6 +4,7 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; use rustc_hash::FxHashSet; +use vt_fs_fingerprint::{Conclusion, PostRunError}; use vt_path::{AbsolutePath, RelativePathBuf}; use vt_plan::cache_metadata::{CacheMetadata, EnvValueHash}; use vt_server::Reports; @@ -13,7 +14,6 @@ use super::{ CacheState, post_run::{TrackedEnvFingerprints, TrackedEnvQuery}, spawn::ChildOutcome, - task_fs::{Conclusion, PostRunError}, }; use crate::session::{ cache::{CacheEntryValue, ExecutionCache, archive}, diff --git a/crates/vt/src/session/execute/mod.rs b/crates/vt/src/session/execute/mod.rs index d3599c197..7a136f3af 100644 --- a/crates/vt/src/session/execute/mod.rs +++ b/crates/vt/src/session/execute/mod.rs @@ -3,7 +3,6 @@ pub mod pipe; pub mod post_run; mod scheduler; pub mod spawn; -pub mod task_fs; #[cfg(windows)] mod win_job; @@ -15,6 +14,7 @@ use std::{ use futures_util::future::LocalBoxFuture; use tokio_util::sync::CancellationToken; +use vt_fs_fingerprint::TaskFs; use vt_ipc_shared::NODE_CLIENT_PATH_ENV_NAME; use vt_path::AbsolutePath; use vt_plan::{SpawnExecution, cache_metadata::CacheMetadata}; @@ -23,7 +23,6 @@ use vt_server::{Recorder, Reports, ServerHandle, StopAccepting, serve}; use self::{ pipe::{PipeSinks, StdOutput, pipe_stdio}, spawn::{ChildHandle, ChildOutcome, SpawnStdio, spawn}, - task_fs::TaskFs, }; use super::{ cache::{CacheEntryValue, CacheMiss, ExecutionCache, archive}, diff --git a/crates/vt/src/session/execute/post_run.rs b/crates/vt/src/session/execute/post_run.rs index bdc6c1fec..998c04fe5 100644 --- a/crates/vt/src/session/execute/post_run.rs +++ b/crates/vt/src/session/execute/post_run.rs @@ -1,7 +1,7 @@ //! Post-run environment fingerprinting: env values and bulk env queries //! observed by runner-aware tools during execution, validated again at cache //! lookup. The filesystem half of post-run fingerprinting lives in -//! [`super::task_fs`]. +//! [`vt_fs_fingerprint`]. use std::{collections::BTreeMap, ffi::OsStr, sync::Arc}; diff --git a/crates/vt/src/session/execute/spawn.rs b/crates/vt/src/session/execute/spawn.rs index b8adff88e..c91c6c1df 100644 --- a/crates/vt/src/session/execute/spawn.rs +++ b/crates/vt/src/session/execute/spawn.rs @@ -2,7 +2,7 @@ //! //! [`spawn`] does one thing: hand back the child's stdio pipes plus a //! cancellation-aware `wait` future. Draining the pipes is [`super::pipe`]'s -//! job; the raw path accesses are judged by [`super::task_fs`] during the +//! job; the raw path accesses are judged by [`vt_fs_fingerprint`] during the //! cache update. use std::{ffi::OsStr, io, process::Stdio}; diff --git a/crates/vt_fs_fingerprint/Cargo.toml b/crates/vt_fs_fingerprint/Cargo.toml new file mode 100644 index 000000000..9494a76ce --- /dev/null +++ b/crates/vt_fs_fingerprint/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "vt_fs_fingerprint" +version = "0.0.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +publish = false +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +fspy_shared = { workspace = true } +rayon = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true, features = ["derive"] } +thiserror = { workspace = true } +tracing = { workspace = true } +twox-hash = { workspace = true } +vt_glob = { workspace = true } +vt_graph = { workspace = true } +vt_path = { workspace = true } +vt_str = { workspace = true } +wax = { workspace = true } +wincode = { workspace = true, features = ["derive"] } + +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } + +[lints] +workspace = true + +[lib] +doctest = false diff --git a/crates/vt_fs_fingerprint/README.md b/crates/vt_fs_fingerprint/README.md new file mode 100644 index 000000000..434b0ed55 --- /dev/null +++ b/crates/vt_fs_fingerprint/README.md @@ -0,0 +1,41 @@ +# vt_fs_fingerprint + +The filesystem story of one task run, extracted from the execution engine so +it can be reasoned about and tested on its own: what a run read, what it +produced, and whether a previous run's cached result still matches the +filesystem. Everything else about caching — storage, archiving, env tracking, +process spawning — stays with the engine; this crate is the part that decides +what filesystem facts _mean_. + +The API is one typestate with a call per stage: + +- **`TaskFs::pre_run`** — before the task executes: validate its io + configuration, capture the state of its listed inputs, and (when a previous + run's fingerprints were fetched) report the first input that changed since + that run. +- **`TaskFs::post_run`** — after it finished: judge the traced file accesses + and either declare caching unsound (`Conclusion::InputModified` — the task + wrote a path it also read) or return everything the cache should remember + (`Conclusion::Cacheable`: the run's `InputFingerprints` and its outputs). + +`InputFingerprints` is the opaque record linking runs together: `post_run` +produces it, the caller stores it in the cache entry, and a later run's +`pre_run` checks the filesystem against it. + +Why a separate crate: the policy for a path that a task both reads and writes +is unsettled and expected to be rewritten several times. Here it can be driven +directly with synthetic traces and temp directories, instead of only being +observable by running a whole task through the engine. + +Design notes worth knowing: + +- The pre-run snapshot is taken before the task runs as a **soundness** + requirement, not a convenience: a task that modifies one of its own listed + inputs without tracing to catch it must perpetually miss. A snapshot taken + after the run would capture the post-run state and turn that safe perpetual + miss into a false cache hit. +- `pre_run` is handed the previous fingerprints (fetch-first) rather than + exposing a separate comparison method, so comparing after the run — or + against the wrong run's record — is unrepresentable. +- "No filesystem change" from `pre_run` is not yet a cache hit: the engine + still validates tracked environment state, which lives outside this crate. diff --git a/crates/vt/src/session/execute/task_fs/collections.rs b/crates/vt_fs_fingerprint/src/collections.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/collections.rs rename to crates/vt_fs_fingerprint/src/collections.rs diff --git a/crates/vt/src/session/execute/task_fs/fingerprint.rs b/crates/vt_fs_fingerprint/src/fingerprint.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/fingerprint.rs rename to crates/vt_fs_fingerprint/src/fingerprint.rs diff --git a/crates/vt/src/session/execute/task_fs/glob.rs b/crates/vt_fs_fingerprint/src/glob.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/glob.rs rename to crates/vt_fs_fingerprint/src/glob.rs diff --git a/crates/vt/src/session/execute/task_fs/hash.rs b/crates/vt_fs_fingerprint/src/hash.rs similarity index 81% rename from crates/vt/src/session/execute/task_fs/hash.rs rename to crates/vt_fs_fingerprint/src/hash.rs index 0bbfb0fac..2ba55b56d 100644 --- a/crates/vt/src/session/execute/task_fs/hash.rs +++ b/crates/vt_fs_fingerprint/src/hash.rs @@ -1,7 +1,7 @@ use std::{hash::Hasher as _, io}; /// Hash content using 8 KiB buffered `xxHash3_64`. -pub(super) fn hash_content(mut stream: impl io::Read) -> io::Result { +pub fn hash_content(mut stream: impl io::Read) -> io::Result { let mut hasher = twox_hash::XxHash3_64::default(); let mut buf = [0u8; 8192]; loop { diff --git a/crates/vt/src/session/execute/task_fs/mod.rs b/crates/vt_fs_fingerprint/src/lib.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/mod.rs rename to crates/vt_fs_fingerprint/src/lib.rs diff --git a/crates/vt/src/session/execute/task_fs/task_run.rs b/crates/vt_fs_fingerprint/src/task_run.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/task_run.rs rename to crates/vt_fs_fingerprint/src/task_run.rs diff --git a/crates/vt/src/session/execute/task_fs/tracked_accesses.rs b/crates/vt_fs_fingerprint/src/tracked_accesses.rs similarity index 100% rename from crates/vt/src/session/execute/task_fs/tracked_accesses.rs rename to crates/vt_fs_fingerprint/src/tracked_accesses.rs