Skip to content

Commit

Permalink
Use files instead of junctions on Windows
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Feb 6, 2025
1 parent 306fcfe commit 4e21181
Show file tree
Hide file tree
Showing 11 changed files with 97 additions and 85 deletions.
11 changes: 0 additions & 11 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@ indicatif = { version = "0.17.8" }
indoc = { version = "2.0.5" }
itertools = { version = "0.14.0" }
jiff = { version = "0.1.14", features = ["serde"] }
junction = { version = "1.2.0" }
mailparse = { version = "0.15.0" }
md-5 = { version = "0.10.6" }
memchr = { version = "2.7.4" }
Expand Down
37 changes: 28 additions & 9 deletions crates/uv-cache/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -483,19 +483,29 @@ impl Cache {
continue;
}

// Remove any symlinks and directories in the revision. The symlinks represent
// unzipped wheels, and the directories represent the source distribution archives.
// Remove everything except the built wheel archive and the metadata.
for entry in fs_err::read_dir(entry.path())? {
let entry = entry?;
let path = entry.path();

if path.is_dir() {
debug!("Removing unzipped built wheel entry: {}", path.display());
summary += rm_rf(path)?;
} else if path.is_symlink() {
debug!("Removing unzipped built wheel entry: {}", path.display());
summary += rm_rf(path)?;
// Retain the resolved metadata (`metadata.msgpack`).
if path
.file_name()
.is_some_and(|file_name| file_name == "metadata.msgpack")
{
continue;
}

// Retain any built wheel archives.
if path
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
{
continue;
}

debug!("Removing unzipped built wheel entry: {}", path.display());
summary += rm_rf(path)?;
}
}
}
Expand All @@ -508,8 +518,17 @@ impl Cache {
if bucket.is_dir() {
for entry in walkdir::WalkDir::new(bucket) {
let entry = entry?;

#[cfg(unix)]
if entry.file_type().is_symlink() {
if let Ok(target) = fs_err::canonicalize(entry.path()) {
if let Ok(target) = uv_fs::resolve_symlink(entry.path()) {
references.insert(target);
}
}

#[cfg(windows)]
if entry.file_type().is_file() {
if let Ok(target) = uv_fs::resolve_symlink(entry.path()) {
references.insert(target);
}
}
Expand Down
9 changes: 9 additions & 0 deletions crates/uv-distribution-filename/src/wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,13 @@ impl WheelFilename {

/// Parse a wheel filename from the stem (e.g., `foo-1.2.3-py3-none-any`).
pub fn from_stem(stem: &str) -> Result<Self, WheelFilenameError> {
// The wheel stem should not contain the `.whl` extension.
if std::path::Path::new(stem)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
{
return Err(WheelFilenameError::UnexpectedExtension(stem.to_string()));
}
Self::parse(stem, stem)
}

Expand Down Expand Up @@ -328,6 +335,8 @@ pub enum WheelFilenameError {
MissingAbiTag(String),
#[error("The wheel filename \"{0}\" is missing a platform tag")]
MissingPlatformTag(String),
#[error("The wheel stem \"{0}\" has an unexpected extension")]
UnexpectedExtension(String),
}

#[cfg(test)]
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-distribution/src/distribution_database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {

// If the wheel was unzipped previously, respect it. Source distributions are
// cached under a unique revision ID, so unzipped directories are never stale.
match built_wheel.target.canonicalize() {
match uv_fs::resolve_symlink(&built_wheel.target) {
Ok(archive) => {
return Ok(LocalWheel {
dist: Dist::Source(dist.clone()),
Expand Down
12 changes: 6 additions & 6 deletions crates/uv-distribution/src/index/built_wheel_index.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
use crate::index::cached_wheel::CachedWheel;
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
use crate::Error;
use uv_cache::{Cache, CacheBucket, CacheShard, WheelCache};
use uv_cache_info::CacheInfo;
use uv_cache_key::cache_digest;
use uv_configuration::ConfigSettings;
use uv_distribution_types::{
DirectUrlSourceDist, DirectorySourceDist, GitSourceDist, Hashed, PathSourceDist,
};
use uv_fs::symlinks;
use uv_platform_tags::Tags;
use uv_types::HashStrategy;

use crate::index::cached_wheel::CachedWheel;
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};
use crate::Error;

/// A local index of built distributions for a specific source distribution.
#[derive(Debug)]
pub struct BuiltWheelIndex<'a> {
Expand Down Expand Up @@ -203,8 +203,8 @@ impl<'a> BuiltWheelIndex<'a> {
let mut candidate: Option<CachedWheel> = None;

// Unzipped wheels are stored as symlinks into the archive directory.
for subdir in symlinks(shard) {
match CachedWheel::from_built_source(&subdir) {
for wheel_dir in uv_fs::entries(shard) {
match CachedWheel::from_built_source(&wheel_dir) {
None => {}
Some(dist_info) => {
// Pick the wheel with the highest priority
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-distribution/src/index/cached_wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ impl CachedWheel {
let filename = WheelFilename::from_stem(filename).ok()?;

// Convert to a cached wheel.
let archive = path.canonicalize().ok()?;
let archive = uv_fs::resolve_symlink(path).ok()?;
let entry = CacheEntry::from_path(archive);
let hashes = Vec::new();
let cache_info = CacheInfo::default();
Expand Down
6 changes: 4 additions & 2 deletions crates/uv-distribution/src/index/registry_wheel_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_cache_key::cache_digest;
use uv_configuration::ConfigSettings;
use uv_distribution_types::{CachedRegistryDist, Hashed, Index, IndexLocations, IndexUrl};
use uv_fs::{directories, files, symlinks};
use uv_fs::{directories, files};
use uv_normalize::PackageName;
use uv_platform_tags::Tags;
use uv_types::HashStrategy;
Expand Down Expand Up @@ -205,7 +205,9 @@ impl<'a> RegistryWheelIndex<'a> {
cache_shard.shard(cache_digest(build_configuration))
};

for wheel_dir in symlinks(cache_shard) {
// STOPSHIP(charlie): On Windows, this could also be... any entry that has a valid wheel
// filename?
for wheel_dir in uv_fs::entries(cache_shard) {
if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
if wheel.filename.compatibility(tags).is_compatible() {
// Enforce hash-checking based on the source distribution.
Expand Down
1 change: 0 additions & 1 deletion crates/uv-fs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ rustix = { workspace = true }

[target.'cfg(windows)'.dependencies]
backon = { workspace = true }
junction = { workspace = true }

[features]
default = []
Expand Down
99 changes: 47 additions & 52 deletions crates/uv-fs/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use fs2::FileExt;
use std::fmt::Display;
use std::path::{Path, PathBuf};

use fs2::FileExt;
use tempfile::NamedTempFile;
use tracing::{debug, error, info, trace, warn};

Expand Down Expand Up @@ -45,40 +46,34 @@ pub async fn read_to_string_transcode(path: impl AsRef<Path>) -> std::io::Result

/// Create a symlink at `dst` pointing to `src`, replacing any existing symlink.
///
/// On Windows, this uses the `junction` crate to create a junction point. The
/// operation is _not_ atomic, as we first delete the junction, then create a
/// junction at the same path.
///
/// Note that because junctions are used, the source must be a directory.
/// On Windows, we emulate symlinks by writing a file with the target path.
#[cfg(windows)]
pub fn replace_symlink(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io::Result<()> {
// If the source is a file, we can't create a junction
if src.as_ref().is_file() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!(
"Cannot create a junction for {}: is not a directory",
src.as_ref().display()
),
));
}
// First, attempt to create a file at the location, but fail if it doesn't exist.
match fs_err::OpenOptions::new()
.write(true)
.create_new(true)
.open(dst.as_ref())
{
Ok(mut file) => {
// Write the target path to the file.
use std::io::Write;
file.write_all(src.as_ref().to_string_lossy().as_bytes())?;
Ok(())
}
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {
// Write to a temporary file, then move it into place.
let temp_dir = tempfile::tempdir_in(dst.as_ref().parent().unwrap())?;
let temp_file = temp_dir.path().join("link");
fs_err::write(&temp_file, src.as_ref().to_string_lossy().as_bytes())?;

// Remove the existing symlink, if any.
match junction::delete(dunce::simplified(dst.as_ref())) {
Ok(()) => match fs_err::remove_dir_all(dst.as_ref()) {
Ok(()) => {}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
},
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
};
// Move the symlink into the target location.
fs_err::rename(&temp_file, dst.as_ref())?;

// Replace it with a new symlink.
junction::create(
dunce::simplified(src.as_ref()),
dunce::simplified(dst.as_ref()),
)
Ok(())
}
Err(err) => Err(err),
}
}

/// Create a symlink at `dst` pointing to `src`, replacing any existing symlink if necessary.
Expand Down Expand Up @@ -106,11 +101,29 @@ pub fn replace_symlink(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io:
}
}

#[cfg(unix)]
/// Remove the symlink at `path`.
pub fn remove_symlink(path: impl AsRef<Path>) -> std::io::Result<()> {
fs_err::remove_file(path.as_ref())
}

/// Canonicalize a symlink, returning the fully-resolved path.
///
/// If the symlink target does not exist, returns an error.
#[cfg(unix)]
pub fn resolve_symlink(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
path.as_ref().canonicalize()
}

/// Canonicalize a symlink, returning the fully-resolved path.
///
/// If the symlink target does not exist, returns an error.
#[cfg(windows)]
pub fn resolve_symlink(path: impl AsRef<Path>) -> std::io::Result<PathBuf> {
// On Windows, we emulate symlinks by writing a file with the target path.
let contents = fs_err::read_to_string(path.as_ref())?;
PathBuf::from(contents.trim()).canonicalize()
}

/// Create a symlink at `dst` pointing to `src` on Unix or copy `src` to `dst` on Windows
///
/// This does not replace an existing symlink or file at `dst`.
Expand All @@ -132,19 +145,6 @@ pub fn symlink_or_copy_file(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std
Ok(())
}

#[cfg(windows)]
pub fn remove_symlink(path: impl AsRef<Path>) -> std::io::Result<()> {
match junction::delete(dunce::simplified(path.as_ref())) {
Ok(()) => match fs_err::remove_dir_all(path.as_ref()) {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(err),
},
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(err),
}
}

/// Return a [`NamedTempFile`] in the specified directory.
///
/// Sets the permissions of the temporary file to `0o666`, to match the non-temporary file default.
Expand Down Expand Up @@ -497,10 +497,10 @@ pub fn directories(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
.map(|entry| entry.path())
}

/// Iterate over the symlinks in a directory.
/// Iterate over the entries in a directory.
///
/// If the directory does not exist, returns an empty iterator.
pub fn symlinks(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
pub fn entries(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
path.as_ref()
.read_dir()
.ok()
Expand All @@ -513,11 +513,6 @@ pub fn symlinks(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
None
}
})
.filter(|entry| {
entry
.file_type()
.is_ok_and(|file_type| file_type.is_symlink())
})
.map(|entry| entry.path())
}

Expand Down
2 changes: 1 addition & 1 deletion crates/uv/src/commands/project/environment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ impl CachedEnvironment {
let cache_entry = cache.entry(CacheBucket::Environments, interpreter_hash, resolution_hash);

if cache.refresh().is_none() {
if let Ok(root) = fs_err::read_link(cache_entry.path()) {
if let Ok(root) = uv_fs::resolve_symlink(cache_entry.path()) {
if let Ok(environment) = PythonEnvironment::from_root(root, cache) {
return Ok(Self(environment));
}
Expand Down

0 comments on commit 4e21181

Please sign in to comment.