fix: pdm add requests pip-system-certs fails nondeterministically (#2765)

* fix: `pdm add requests pip-system-certs` fails nondeterministically
Fixes #2762

Signed-off-by: Frost Ming <[email protected]>

* fix pth files after done

Signed-off-by: Frost Ming <[email protected]>

---------

Signed-off-by: Frost Ming <[email protected]>

Author: frostming

news/2762.bugfix.md

@@ -0,0 +1 @@
+Fix a race condition where pth files take effect when multiple packages are installed in parallel.

src/pdm/installers/installers.py

@@ -9,7 +9,7 @@
 
 from installer import install as _install
 from installer._core import _process_WHEEL_file
-from installer.destinations import SchemeDictionaryDestination
+from installer.destinations import SchemeDictionaryDestination, WheelDestination
 from installer.exceptions import InvalidWheelSource
 from installer.records import RecordEntry
 from installer.sources import WheelContentElement, WheelSource
@@ -97,12 +97,14 @@ def __init__(
         self,
         *args: Any,
         link_method: LinkMethod = "copy",
+        rename_pth: bool = False,
         **kwargs: Any,
     ) -> None:
         super().__init__(*args, **kwargs)
         self.link_method = link_method
+        self.rename_pth = rename_pth
 
-    def write_to_fs(self, scheme: Scheme, path: str | Path, stream: BinaryIO, is_executable: bool) -> RecordEntry:
+    def write_to_fs(self, scheme: Scheme, path: str, stream: BinaryIO, is_executable: bool) -> RecordEntry:
         from installer.records import Hash
         from installer.utils import copyfileobj_with_hashing, make_file_executable
 
@@ -112,6 +114,10 @@ def write_to_fs(self, scheme: Scheme, path: str | Path, stream: BinaryIO, is_exe
 
         os.makedirs(os.path.dirname(target_path), exist_ok=True)
 
+        if self.rename_pth and target_path.endswith(".pth") and "/" not in path:
+            # Postpone the creation of pth files since it may cause race condition
+            # when multiple packages are installed at the same time.
+            target_path += ".pdmtmp"
         if self.link_method == "copy" or not hasattr(stream, "name"):
             with open(target_path, "wb") as f:
                 hash_, size = copyfileobj_with_hashing(stream, f, self.hash_algorithm)
@@ -147,6 +153,7 @@ def install_package(
     environment: BaseEnvironment,
     direct_url: dict[str, Any] | None = None,
     install_links: bool = True,
+    rename_pth: bool = False,
 ) -> str:
     """Only create .pth files referring to the cached package.
     If the cache doesn't exist, create one.
@@ -175,6 +182,7 @@ def install_package(
         interpreter=interpreter,
         script_kind=script_kind,
         link_method=link_method,
+        rename_pth=rename_pth,
     )
     source = PackageWheelSource(package)
     dist_info_dir = install(source, destination=destination, additional_metadata=additional_metadata)
@@ -184,7 +192,7 @@ def install_package(
 
 
 def install(
-    source: WheelSource, destination: InstallDestination, additional_metadata: dict[str, bytes] | None = None
+    source: WheelSource, destination: WheelDestination, additional_metadata: dict[str, bytes] | None = None
 ) -> str:
     """A lower level installation method that is copied from installer
     but is controlled by extra parameters.
@@ -198,7 +206,7 @@ def install(
 
 def install_wheel(wheel: str, environment: BaseEnvironment, direct_url: dict[str, Any] | None = None) -> str:
     """Install a wheel into the environment, return the .dist-info path"""
-    destination = InstallDestination(
+    destination = SchemeDictionaryDestination(
         scheme_dict=environment.get_paths(_get_dist_name(wheel)),
         interpreter=str(environment.interpreter.executable),
         script_kind=environment.script_kind,

src/pdm/installers/manager.py

@@ -19,15 +19,22 @@ class InstallManager:
     # The packages below are needed to load paths and thus should not be cached.
     NO_CACHE_PACKAGES = ("editables",)
 
-    def __init__(self, environment: BaseEnvironment, *, use_install_cache: bool = False) -> None:
+    def __init__(
+        self, environment: BaseEnvironment, *, use_install_cache: bool = False, rename_pth: bool = False
+    ) -> None:
         self.environment = environment
         self.use_install_cache = use_install_cache
+        self.rename_pth = rename_pth
 
     def install(self, candidate: Candidate) -> Distribution:
         """Install a candidate into the environment, return the distribution"""
         prepared = candidate.prepare(self.environment)
         dist_info = install_package(
-            prepared.get_cached_package(), self.environment, prepared.direct_url(), self.use_install_cache
+            prepared.get_cached_package(),
+            self.environment,
+            direct_url=prepared.direct_url(),
+            install_links=self.use_install_cache,
+            rename_pth=self.rename_pth,
         )
         return Distribution.at(dist_info)
 

src/pdm/installers/synchronizers.py

@@ -122,7 +122,7 @@ def __init__(
         self.no_editable = no_editable
         self.install_self = install_self
         if use_install_cache is None:
-            use_install_cache = environment.project.config["install.cache"]
+            use_install_cache = bool(environment.project.config["install.cache"])
         self.use_install_cache = use_install_cache
         self.reinstall = reinstall
         self.only_keep = only_keep
@@ -177,7 +177,7 @@ def manager(self) -> InstallManager:
 
     def get_manager(self) -> InstallManager:
         return self.environment.project.core.install_manager_class(
-            self.environment, use_install_cache=self.use_install_cache
+            self.environment, use_install_cache=self.use_install_cache, rename_pth=True
         )
 
     @property
@@ -378,6 +378,19 @@ def _show_summary(self, packages: dict[str, list[str]]) -> None:
         else:
             self.ui.echo("All packages are synced to date, nothing to do.")
 
+    def _fix_pth_files(self) -> None:
+        """Remove the .pdmtmp suffix from the installed packages"""
+        from pathlib import Path
+
+        lib_paths = self.environment.get_paths()
+        for scheme in ["purelib", "platlib"]:
+            for path in list(Path(lib_paths[scheme]).iterdir()):
+                if path.suffix == ".pdmtmp":
+                    target_path = path.with_suffix("")
+                    if target_path.exists():
+                        target_path.unlink()
+                    path.rename(target_path)
+
     def synchronize(self) -> None:
         to_add, to_update, to_remove = self.compare_with_working_set()
         to_do = {"remove": to_remove, "update": to_update, "add": to_add}
@@ -443,21 +456,25 @@ def update_progress(future: Future | DummyFuture, kind: str, key: str) -> None:
                 state.errors.clear()
                 live.console.print("Retry failed jobs")
 
-            if state.errors:
-                if self.ui.verbosity < termui.Verbosity.DETAIL:
-                    live.console.print("\n[error]ERRORS[/]:")
-                    live.console.print("".join(state.errors), end="")
-                raise InstallationError("Some package operations are not complete yet")
-
-            if self.install_self:
-                self_key = self.self_key
-                assert self_key
-                self.candidates[self_key] = self.self_candidate
-                word = "a" if self.no_editable else "an editable"
-                live.console.print(f"Installing the project as {word} package...")
-                if self_key in self.working_set:
-                    self.update_candidate(self_key, progress)
-                else:
-                    self.install_candidate(self_key, progress)
+            try:
+                if state.errors:
+                    if self.ui.verbosity < termui.Verbosity.DETAIL:
+                        live.console.print("\n[error]ERRORS[/]:")
+                        live.console.print("".join(state.errors), end="")
+                    raise InstallationError("Some package operations are not complete yet")
+
+                if self.install_self:
+                    self_key = self.self_key
+                    assert self_key
+                    self.candidates[self_key] = self.self_candidate
+                    word = "a" if self.no_editable else "an editable"
+                    live.console.print(f"Installing the project as {word} package...")
+                    if self_key in self.working_set:
+                        self.update_candidate(self_key, progress)
+                    else:
+                        self.install_candidate(self_key, progress)
 
-            live.console.print(f"\n{termui.Emoji.POPPER} All complete!")
+                live.console.print(f"\n{termui.Emoji.POPPER} All complete!")
+            finally:
+                # Now we remove the .pdmtmp suffix from the installed packages
+                self._fix_pth_files()