Refactor pythonfinder for improved efficiency and PEP 514 support (#6360)

* Complete python finder 3.x rewrite (with new tests-and updated docs).

* Support py command on windows as well


* PR feedback from testing

* More conversions to pathlib to help with the pythonfinder integration

* Fix issue related to python bug 35144 now that we dropped python3.8

* fix ruff

Author: matteius

news/6360.feature.rst

@@ -0,0 +1,25 @@
+# Refactor pythonfinder for improved efficiency and PEP 514 support
+
+## Summary
+This PR completely refactors the pythonfinder module to improve efficiency, reduce logical errors, and fix support for PEP 514 (Python registration in the Windows registry). The refactoring replaces the complex object hierarchy with a more modular, composition-based approach that is easier to maintain and extend.
+
+## Motivation
+The original pythonfinder implementation had several issues:
+* Complex object wrapping with paths as objects, leading to excessive recursion
+* Tight coupling between classes making the code difficult to follow and maintain
+* Broken Windows registry support (PEP 514)
+* Performance issues due to redundant path scanning and inefficient caching
+
+## Changes
+* **Architecture**: Replaced inheritance-heavy design with a composition-based approach using specialized finders
+* **Data Model**: Simplified the data model with a clean ``PythonInfo`` dataclass
+* **Windows Support**: Implemented proper PEP 514 support for Windows registry
+* **Performance**: Improved caching and reduced redundant operations
+* **Error Handling**: Added more specific exceptions and better error handling
+
+## Features
+The refactored implementation continues to support all required features:
+* System and user PATH searches
+* pyenv installations
+* asdf installations
+* Windows registry (PEP 514) - now working correctly

pipenv/__init__.py

@@ -2,11 +2,12 @@
 import os
 import sys
 import warnings
+from pathlib import Path
 
 # This has to come before imports of pipenv
-PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
-PIP_ROOT = os.sep.join([PIPENV_ROOT, "patched", "pip"])
-sys.path.insert(0, PIPENV_ROOT)
+PIPENV_ROOT = Path(__file__).resolve().parent.absolute()
+PIP_ROOT = str(PIPENV_ROOT / "patched" / "pip")
+sys.path.insert(0, str(PIPENV_ROOT))
 sys.path.insert(0, PIP_ROOT)
 
 # Load patched pip instead of system pip
@@ -15,9 +16,10 @@
 
 def _ensure_modules():
     # Ensure when pip gets invoked it uses our patched version
+    location = Path(__file__).parent / "patched" / "pip" / "__init__.py"
     spec = importlib.util.spec_from_file_location(
         "pip",
-        location=os.path.join(os.path.dirname(__file__), "patched", "pip", "__init__.py"),
+        location=str(location),
     )
     pip = importlib.util.module_from_spec(spec)
     sys.modules["pip"] = pip

pipenv/cli/command.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from pathlib import Path
 
 from pipenv import environments
 from pipenv.__version__ import __version__
@@ -98,8 +99,8 @@ def cli(
 
     if man:
         if system_which("man"):
-            path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "pipenv.1")
-            os.execle(system_which("man"), "man", path, os.environ)
+            path = Path(__file__).parent.parent / "pipenv.1"
+            os.execle(system_which("man"), "man", str(path), os.environ)
             return 0
         else:
             err.print(
@@ -393,8 +394,8 @@ def shell(state, fancy=False, shell_args=None, anyway=False, quiet=False):
     # Use fancy mode for Windows or pwsh on *nix.
     if (
         os.name == "nt"
-        or (os.environ.get("PIPENV_SHELL") or "").split(os.path.sep)[-1] == "pwsh"
-        or (os.environ.get("SHELL") or "").split(os.path.sep)[-1] == "pwsh"
+        or Path(os.environ.get("PIPENV_SHELL") or "").name == "pwsh"
+        or Path(os.environ.get("SHELL") or "").name == "pwsh"
     ):
         fancy = True
     do_shell(
@@ -624,7 +625,7 @@ def run_open(state, module, *args, **kwargs):
         console.print("Module not found!", style="red")
         sys.exit(1)
     if "__init__.py" in c.stdout:
-        p = os.path.dirname(c.stdout.strip().rstrip("cdo"))
+        p = Path(c.stdout.strip().rstrip("cdo")).parent
     else:
         p = c.stdout.strip().rstrip("cdo")
     console.print(f"Opening {p!r} in your EDITOR.", style="bold")

pipenv/cli/options.py

@@ -1,5 +1,5 @@
-import os
 import re
+from pathlib import Path
 
 from pipenv.project import Project
 from pipenv.utils import console, err
@@ -460,7 +460,8 @@ def validate_python_path(ctx, param, value):
     # the path or an absolute path. To report errors as early as possible
     # we'll report absolute paths which do not exist:
     if isinstance(value, (str, bytes)):
-        if os.path.isabs(value) and not os.path.isfile(value):
+        path = Path(value)
+        if path.is_absolute() and not path.is_file():
             raise BadParameter(f"Expected Python at path {value} does not exist")
     return value
 

pipenv/environment.py

@@ -113,7 +113,7 @@ def python_version(self) -> str | None:
     @property
     def python_info(self) -> dict[str, str]:
         include_dir = self.prefix / "include"
-        if not os.path.exists(include_dir):
+        if not include_dir.exists():
             include_dirs = self.get_include_path()
             if include_dirs:
                 include_path = include_dirs.get(
@@ -130,15 +130,17 @@ def python_info(self) -> dict[str, str]:
         return {}
 
     def _replace_parent_version(self, path: str, replace_version: str) -> str:
-        if not os.path.exists(path):
-            base, leaf = os.path.split(path)
-            base, parent = os.path.split(base)
-            leaf = os.path.join(parent, leaf).replace(
+        path_obj = Path(path)
+        if not path_obj.exists():
+            parent = path_obj.parent
+            grandparent = parent.parent
+            leaf = f"{parent.name}/{path_obj.name}"
+            leaf = leaf.replace(
                 replace_version,
                 self.python_info.get("py_version_short", get_python_version()),
             )
-            return os.path.join(base, leaf)
-        return path
+            return str(grandparent / leaf)
+        return str(path_obj)
 
     @cached_property
     def install_scheme(self):

pipenv/environments.py

@@ -1,8 +1,8 @@
 import glob
 import os
-import pathlib
 import re
 import sys
+from pathlib import Path
 
 from pipenv.patched.pip._vendor.platformdirs import user_cache_dir
 from pipenv.utils.fileutils import normalize_drive
@@ -53,18 +53,18 @@ def get_from_env(arg, prefix="PIPENV", check_for_negation=True, default=None):
 def normalize_pipfile_path(p):
     if p is None:
         return None
-    loc = pathlib.Path(p)
+    loc = Path(p)
     try:
         loc = loc.resolve()
     except OSError:
         loc = loc.absolute()
     # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728
     if os.name == "nt":
         matches = glob.glob(re.sub(r"([^:/\\])(?=[/\\]|$)", r"[\1]", str(loc)))
-        path_str = matches and matches[0] or str(loc)
+        path = Path(matches[0] if matches else str(loc))
     else:
-        path_str = str(loc)
-    return normalize_drive(os.path.abspath(path_str))
+        path = loc
+    return normalize_drive(str(path.absolute()))
 
 
 # HACK: Prevent invalid shebangs with Homebrew-installed Python:
@@ -433,8 +433,8 @@ def is_in_virtualenv() -> bool:
     :rtype: bool
     """
 
-    pipenv_active = os.environ.get("PIPENV_ACTIVE")
-    virtual_env = bool(os.environ.get("VIRTUAL_ENV"))
+    pipenv_active = is_env_truthy("PIPENV_ACTIVE")
+    virtual_env = bool(os.getenv("VIRTUAL_ENV"))
     ignore_virtualenvs = bool(get_from_env("IGNORE_VIRTUALENVS"))
     return virtual_env and not (pipenv_active or ignore_virtualenvs)
 

pipenv/help.py

@@ -32,7 +32,7 @@ def get_pipenv_diagnostics(project):
     finder = pythonfinder.Finder(system=False, global_search=True)
     python_paths = finder.find_all_python_versions()
     for python in python_paths:
-        print(f"  - `{python.py_version.version}`: `{python.path}`")
+        print(f"  - `{python.version_str}`: `{python.path}`")
 
     print("")
     print("PEP 508 Information:")