fix(toolchain): symlink all toolchain files for the host toolchain (#1745)

aignas · web-flow · commit 74d576f7a2c9 · 2024-02-08T00:37:52.000Z
Previously we would only symlink the interpreter binary itself when creating the hermetic host toolchain used in setting up the `whl_library` repositories. This works on UNIX platforms and Windows if they have the following in their `.bazelrc`: ``` startup --windows_enable_symlinks ``` In our CI we had the same lines but the users did not need to add them until the `0.29.0` forced them to have them because we actually started using symlinks on Windows. If the symlinks are not enabled on the host platform `bazel` tries to be helpful and copies the files over instead of making the links. We are leveraging this to just symlink all of the contents of the python interpreter repository for the host platform to the `_host` toolchain repository. Fixes #1723
diff --git a/.bazelrc b/.bazelrc
@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 
@@ -19,7 +19,6 @@ build --incompatible_default_to_explicit_init_py
 
 # Windows makes use of runfiles for some rules
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Make Bazel 6 use bzlmod by default
 common --enable_bzlmod
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -36,13 +36,18 @@ A brief description of the categories of changes:
 ### Fixed
 
 * (bzlmod) pip.parse now does not fail with an empty `requirements.txt`.
+
 * (py_wheel) Wheels generated by `py_wheel` now preserve executable bits when
   being extracted by `installer` and/or `pip`.
+
 * (coverage) During the running of lcov, the stdout/stderr was causing test
   failures.  By default, suppress output when generating lcov.  This can be
   overridden by setting 'VERBOSE_COVERAGE'.  This change only affect bazel
   7.x.x and above.
 
+* (toolchain) Changed the `host_toolchain` to symlink all files to support
+  Windows host environments without symlink support.
+
 ### Added
 
 * (py_wheel) Added `requires_file` and `extra_requires_files` attributes.
diff --git a/docs/sphinx/pypi-dependencies.md b/docs/sphinx/pypi-dependencies.md
@@ -27,6 +27,18 @@ use_repo(pip, "my_deps")
 For more documentation, including how the rules can update/create a requirements
 file, see the bzlmod examples under the {gh-path}`examples` folder.
 
+We are using a host-platform compatible toolchain by default to setup pip dependencies.
+During the setup phase, we create some symlinks, which may be inefficient on Windows
+by default. In that case use the following `.bazelrc` options to improve performance if
+you have admin privileges:
+```
+startup --windows_enable_symlinks
+```
+
+This will enable symlinks on Windows and help with bootstrap performance of setting up the 
+hermetic host python interpreter on this platform. Linux and OSX users should see no
+difference.
+
 ### Using a WORKSPACE file
 
 To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and
diff --git a/examples/build_file_generation/.bazelrc b/examples/build_file_generation/.bazelrc
@@ -2,7 +2,6 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # The bzlmod version of this example is in examples/bzlmod_build_file_generation
 # Once WORKSPACE support is dropped, this example can be entirely deleted.
diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc
@@ -6,5 +6,3 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-
-startup --windows_enable_symlinks
diff --git a/examples/bzlmod_build_file_generation/.bazelrc b/examples/bzlmod_build_file_generation/.bazelrc
@@ -2,7 +2,6 @@ test --test_output=errors --enable_runfiles
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 common --experimental_enable_bzlmod
 
diff --git a/examples/multi_python_versions/.bazelrc b/examples/multi_python_versions/.bazelrc
@@ -2,6 +2,5 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 coverage --java_runtime_version=remotejdk_11
diff --git a/examples/pip_parse_vendored/.bazelrc b/examples/pip_parse_vendored/.bazelrc
@@ -2,7 +2,6 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Vendoring requirements.bzl files isn't necessary under bzlmod
 # When workspace support is dropped, this example can be removed.
diff --git a/gazelle/.bazelrc b/gazelle/.bazelrc
@@ -10,7 +10,6 @@ build --incompatible_default_to_explicit_init_py
 
 # Windows makes use of runfiles for some rules
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # Do NOT implicitly create empty __init__.py files in the runfiles tree.
 # By default, these are created in every directory containing Python source code
diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl
@@ -223,7 +223,7 @@ def _outputs_to_str(result):
 repo_utils = struct(
     execute_checked = _execute_checked,
     execute_unchecked = _execute_unchecked,
-    execute_check_stdout = _execute_checked_stdout,
+    execute_checked_stdout = _execute_checked_stdout,
     is_repo_debug_enabled = _is_repo_debug_enabled,
     debug_print = _debug_print,
     which_checked = _which_checked,
diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
@@ -238,15 +238,64 @@ exports_files(["python"], visibility = ["//visibility:public"])
 
     (os_name, arch) = get_host_os_arch(rctx)
     host_platform = get_host_platform(os_name, arch)
-    host_python = rctx.path(
-        Label(
-            "@@{py_repository}_{host_platform}//:python".format(
-                py_repository = rctx.attr.name[:-len("_host")],
-                host_platform = host_platform,
-            ),
-        ),
+    repo = "@@{py_repository}_{host_platform}".format(
+        py_repository = rctx.attr.name[:-len("_host")],
+        host_platform = host_platform,
+    )
+
+    rctx.report_progress("Symlinking interpreter files to the target platform")
+    host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo)))
+
+    # The interpreter might not work on platfroms that don't have symlink support if
+    # we just symlink the interpreter itself. rctx.symlink does a copy in such cases
+    # so we can just attempt to symlink all of the directories in the host interpreter
+    # repo, which should be faster than re-downloading it.
+    for p in host_python_repo.dirname.readdir():
+        if p.basename in [
+            # ignore special files created by the repo rule automatically
+            "BUILD.bazel",
+            "MODULE.bazel",
+            "REPO.bazel",
+            "WORKSPACE",
+            "WORKSPACE.bazel",
+            "WORKSPACE.bzlmod",
+        ]:
+            continue
+
+        # symlink works on all platforms that bazel supports, so it should work on
+        # UNIX and Windows with and without symlink support. For better performance
+        # users should enable the symlink startup option, however that requires admin
+        # privileges.
+        rctx.symlink(p, p.basename)
+
+    is_windows = (os_name == WINDOWS_NAME)
+    python_binary = "python.exe" if is_windows else "python"
+
+    # Ensure that we can run the interpreter and check that we are not
+    # using the host interpreter.
+    python_tester_contents = """\
+from pathlib import Path
+import sys
+
+python = Path(sys.executable)
+want_python = str(Path("{python}").resolve())
+got_python = str(Path(sys.executable).resolve())
+
+assert want_python == got_python, \
+    "Expected to use a different interpreter:\\nwant: '{{}}'\\n got: '{{}}'".format(
+        want_python,
+        got_python,
+    )
+""".format(repo = repo.strip("@"), python = python_binary)
+    python_tester = rctx.path("python_tester.py")
+    rctx.file(python_tester, python_tester_contents)
+    repo_utils.execute_checked(
+        rctx,
+        op = "CheckHostInterpreter",
+        arguments = [rctx.path(python_binary), python_tester],
     )
-    rctx.symlink(host_python, "python")
+    if not rctx.delete(python_tester):
+        fail("Failed to delete the python tester")
 
 host_toolchain = repository_rule(
     _host_toolchain_impl,
diff --git a/tests/integration/compile_pip_requirements/.bazelrc b/tests/integration/compile_pip_requirements/.bazelrc
@@ -2,4 +2,3 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
diff --git a/tests/integration/ignore_root_user_error/.bazelrc b/tests/integration/ignore_root_user_error/.bazelrc
@@ -4,4 +4,3 @@ test --test_output=errors
 
 # Windows requires these for multi-python support:
 build --enable_runfiles
-startup --windows_enable_symlinks
diff --git a/tests/integration/pip_parse/.bazelrc b/tests/integration/pip_parse/.bazelrc
@@ -1,7 +1,6 @@
 # Bazel configuration flags
 
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
 try-import %workspace%/user.bazelrc
diff --git a/tests/integration/pip_repository_entry_points/.bazelrc b/tests/integration/pip_repository_entry_points/.bazelrc
@@ -1,7 +1,6 @@
 # Bazel configuration flags
 
 build --enable_runfiles
-startup --windows_enable_symlinks
 
 # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file
 try-import %workspace%/user.bazelrc
