diff --git a/aws_lambda_builders/workflows/python_uv/DESIGN.md b/aws_lambda_builders/workflows/python_uv/DESIGN.md index cfdba8fb2..3ef5a60d6 100644 --- a/aws_lambda_builders/workflows/python_uv/DESIGN.md +++ b/aws_lambda_builders/workflows/python_uv/DESIGN.md @@ -214,7 +214,7 @@ config = { "no_cache": False, # Disable caching "prerelease": "disallow", # Handle pre-release versions "resolution": "highest", # Resolution strategy - "compile_bytecode": True, # Compile .pyc files + "compile_bytecode": False, # Opt in to unchecked-hash .pyc compiled with the target Python "exclude_newer": None, # Exclude packages newer than date "generate_hashes": False, # Generate package hashes } diff --git a/aws_lambda_builders/workflows/python_uv/packager.py b/aws_lambda_builders/workflows/python_uv/packager.py index 1ca8c07c7..4c1251c83 100644 --- a/aws_lambda_builders/workflows/python_uv/packager.py +++ b/aws_lambda_builders/workflows/python_uv/packager.py @@ -57,6 +57,17 @@ def get_uv_version(self) -> Optional[str]: pass return None + def find_python(self, python_version: str) -> Optional[str]: + """Find an already-installed interpreter matching the requested Python version.""" + rc, stdout, stderr = self.run_uv_command( + ["python", "find", "--no-project", "--no-python-downloads", python_version] + ) + if rc == 0 and stdout: + return stdout.strip() + diagnostic = (stderr or "").strip() or (stdout or "").strip() or "no diagnostic output" + LOG.warning("Could not locate target Python %s via uv (exit code %d): %s", python_version, rc, diagnostic) + return None + def run_uv_command(self, args: List[str], cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> tuple: """ Execute UV command with given arguments. @@ -165,15 +176,33 @@ def install_requirements( # Resolve --target to an absolute path: UV runs from the project or workspace directory, # so a relative target (e.g. the incremental-build dependencies dir) would otherwise be # created under the UV's cwd instead of the build root. - args.extend(["--target", os.path.abspath(target_dir)]) + target_dir = os.path.abspath(target_dir) + args.extend(["--target", target_dir]) - # Add configuration arguments + target_python = None + if config.compile_bytecode: + if python_version: + target_python = self._uv.find_python(python_version) + else: + LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + + # Add configuration arguments that are independent of interpreter selection. args.extend(config.to_uv_args()) # Add platform-specific arguments if python_version: args.extend(["--python-version", python_version]) + # Use the exact interpreter found by UV instead of repeating a version request that + # could resolve differently when the install command runs. + if target_python: + args.extend(["--python", target_python]) + + # UV currently produces timestamp-based bytecode. SAM packages files with normalized + # timestamps, so that bytecode would be rejected by the Lambda runtime after deployment. + # Keep UV compilation disabled and generate unchecked-hash bytecode below instead. + args.append("--no-compile-bytecode") + if platform and architecture: # UV pip install uses --python-platform format # Map Lambda architectures to UV platform strings @@ -193,14 +222,47 @@ def install_requirements( python_platform = f"{target_arch}-manylinux_{manylinux_version}" args.extend(["--python-platform", python_platform]) + # uv rejects --no-compile-bytecode when UV_COMPILE_BYTECODE is also set ("cannot be used + # with"), so drop the variable for this command; bytecode is handled by _compile_bytecode. + env = self._osutils.original_environ() + env.pop("UV_COMPILE_BYTECODE", None) + # Execute UV pip install - rc, stdout, stderr = self._uv.run_uv_command(args, cwd) + rc, stdout, stderr = self._uv.run_uv_command(args, cwd, env=env) if rc != 0: raise UvInstallationError(reason=f"UV pip install failed: {stderr}") LOG.debug("UV pip install completed successfully: %s", stdout) + if target_python: + self._compile_bytecode(target_python, target_dir) + + def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: + """Compile reusable bytecode with the interpreter selected for the target runtime.""" + command = [ + python_executable, + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + target_dir, + ] + LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command)) + rc, stdout, stderr = self._osutils.run_subprocess(command) + if rc != 0: + diagnostic = stderr.strip() or stdout.strip() or "no diagnostic output" + LOG.warning( + "Could not compile unchecked-hash bytecode with target Python (exit code %d): %s", + rc, + diagnostic, + ) + return + + LOG.debug("Python bytecode compilation completed successfully: %s", stdout) + class PythonUvDependencyBuilder: """High-level dependency builder that orchestrates UV operations.""" diff --git a/aws_lambda_builders/workflows/python_uv/utils.py b/aws_lambda_builders/workflows/python_uv/utils.py index aa0bc12ab..4802085ee 100644 --- a/aws_lambda_builders/workflows/python_uv/utils.py +++ b/aws_lambda_builders/workflows/python_uv/utils.py @@ -28,7 +28,16 @@ def run_subprocess(self, cmd, cwd=None, env=None): env = self.original_environ() try: - result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=False) + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) return result.returncode, result.stdout, result.stderr except Exception as e: return 1, "", str(e) @@ -133,7 +142,7 @@ def __init__( no_cache: bool = False, prerelease: str = "disallow", resolution: str = "highest", - compile_bytecode: bool = True, + compile_bytecode: bool = False, exclude_newer: Optional[str] = None, generate_hashes: bool = False, groups: Optional[List[str]] = None, diff --git a/tests/functional/workflows/python_uv/test_packager.py b/tests/functional/workflows/python_uv/test_packager.py index 79049557e..37797bc5a 100644 --- a/tests/functional/workflows/python_uv/test_packager.py +++ b/tests/functional/workflows/python_uv/test_packager.py @@ -1,4 +1,5 @@ import os +import sys from collections import defaultdict, namedtuple import pytest @@ -659,6 +660,41 @@ def test_can_build_with_hash_verification(self, tmpdir, uv_runner, osutils): assert "certifi" in installed_packages + def test_can_compile_unchecked_hash_bytecode_when_enabled(self, tmpdir, uv_runner, osutils): + """Opt-in bytecode uses the interpreter from `uv python find` and writes unchecked-hash .pyc""" + reqs = ["requests"] + fake_uv, runner = uv_runner + appdir, builder, manifest_path = self._make_appdir_and_dependency_builder( + reqs, tmpdir, runner, manifest_type="requirements" + ) + + # Every fake UV call returns this str; `uv python find` uses it as the interpreter path. + fake_uv.set_return_tuple(0, sys.executable, "") + + site_packages = os.path.join(appdir, "site-packages") + os.makedirs(site_packages, exist_ok=True) + fake_uv.packages_to_install(expected_args=["pip", "install"], packages=reqs, install_location=site_packages) + + with osutils.tempdir() as scratch_dir: + builder.build_dependencies( + artifacts_dir_path=site_packages, + scratch_dir_path=scratch_dir, + manifest_path=manifest_path, + architecture=X86_64, + config=UvConfig(compile_bytecode=True), + ) + + python_find_calls = [args for args, _, _ in fake_uv._calls["python"] if args[:2] == ["python", "find"]] + assert python_find_calls + + cache_dir = os.path.join(site_packages, "requests", "__pycache__") + pyc_files = os.listdir(cache_dir) + assert pyc_files + with open(os.path.join(cache_dir, pyc_files[0]), "rb") as f: + header = f.read(8) + # PEP 552 flags word: 0b01 = hash-based, source not checked (unchecked-hash). + assert int.from_bytes(header[4:8], "little") == 1 + def test_lock_export_uses_build_options(self, tmpdir, uv_runner, osutils): """Build options select groups and extras""" fake_uv, runner = uv_runner diff --git a/tests/unit/workflows/python_uv/test_packager.py b/tests/unit/workflows/python_uv/test_packager.py index 95cb92445..518acd440 100644 --- a/tests/unit/workflows/python_uv/test_packager.py +++ b/tests/unit/workflows/python_uv/test_packager.py @@ -147,11 +147,57 @@ def test_get_uv_version_failure(self, mock_osutils_class): self.assertIsNone(version) + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_success(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (0, "/usr/bin/python3.13\n", "") + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + self.assertEqual(subprocess_uv.find_python("3.13"), "/usr/bin/python3.13") + mock_osutils.run_subprocess.assert_called_once_with( + ["/usr/bin/uv", "python", "find", "--no-project", "--no-python-downloads", "3.13"], + cwd=None, + env=None, + ) + + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_failure(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (2, "", "interpreter not found") + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.assertIsNone(subprocess_uv.find_python("3.9")) + + self.assertIn("Could not locate target Python 3.9 via uv (exit code 2)", logs.output[0]) + self.assertIn("interpreter not found", logs.output[0]) + + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_failure_without_diagnostics(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (1, None, None) + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.assertIsNone(subprocess_uv.find_python("3.9")) + + self.assertIn("no diagnostic output", logs.output[0]) + class TestUvRunner(TestCase): def setUp(self): self.mock_subprocess_uv = Mock() self.mock_subprocess_uv.uv_executable = "/usr/bin/uv" + self.mock_subprocess_uv.find_python.return_value = None self.mock_osutils = Mock() self.uv_runner = UvRunner(uv_subprocess=self.mock_subprocess_uv, osutils=self.mock_osutils) @@ -196,6 +242,129 @@ def test_install_requirements_success(self): self.assertIn("install", args_called) self.assertIn("-r", args_called) self.assertEqual(args_called[args_called.index("-r") + 1], os.path.abspath("/path/to/requirements.txt")) + self.assertIn("--no-compile-bytecode", args_called) + # Bytecode is opt-in: the default config must not look up a Python or run compileall. + self.mock_subprocess_uv.find_python.assert_not_called() + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_drops_uv_compile_bytecode_env(self): + # uv refuses --no-compile-bytecode when UV_COMPILE_BYTECODE is set in the environment. + self.mock_osutils.original_environ.return_value = {"UV_COMPILE_BYTECODE": "1", "PATH": "/usr/bin"} + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + python_version="3.13", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + env_called = self.mock_subprocess_uv.run_uv_command.call_args.kwargs["env"] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("UV_COMPILE_BYTECODE", env_called) + self.assertEqual(env_called["PATH"], "/usr/bin") + + def test_install_requirements_compiles_unchecked_hash_bytecode_when_enabled(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (0, "", "") + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + python_version="3.13", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertEqual(args_called[args_called.index("--python") + 1], "/usr/bin/python3.13") + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13") + self.mock_subprocess_uv.find_python.assert_called_once_with("3.13") + self.mock_osutils.run_subprocess.assert_called_once_with( + [ + "/usr/bin/python3.13", + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + os.path.abspath("/target"), + ] + ) + + def test_install_requirements_skips_bytecode_when_target_python_is_unavailable(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = None + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + python_version="3.9", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.9") + self.mock_subprocess_uv.find_python.assert_called_once_with("3.9") + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_skips_bytecode_without_target_python_version(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.mock_subprocess_uv.find_python.assert_not_called() + self.assertIn("Target Python version is unavailable", logs.output[0]) + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_does_not_select_python_when_bytecode_disabled(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=False), + python_version="3.13", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13") + self.mock_subprocess_uv.find_python.assert_not_called() + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fails(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (1, "", "compile error") + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + python_version="3.13", + ) + + self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0]) + self.assertIn("compile error", logs.output[0]) def test_install_requirements_resolves_relative_target_to_absolute(self): # UV runs from project_dir or workspace_dir, diff --git a/tests/unit/workflows/python_uv/test_utils.py b/tests/unit/workflows/python_uv/test_utils.py index 85a622a34..e1966a7a4 100644 --- a/tests/unit/workflows/python_uv/test_utils.py +++ b/tests/unit/workflows/python_uv/test_utils.py @@ -38,6 +38,25 @@ def test_run_subprocess_failure(self): rc, stdout, stderr = self.osutils.run_subprocess(["false"]) self.assertEqual(rc, 1) + @patch("aws_lambda_builders.workflows.python_uv.utils.subprocess.run") + def test_run_subprocess_decodes_utf8_with_replacement(self, mock_run): + mock_run.return_value = Mock(returncode=0, stdout="success", stderr="") + env = {"UV_TEST": "1"} + + result = self.osutils.run_subprocess(["uv", "--version"], cwd="/work", env=env) + + self.assertEqual(result, (0, "success", "")) + mock_run.assert_called_once_with( + ["uv", "--version"], + cwd="/work", + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + class TestDetectUvManifest(TestCase): def test_detect_uv_manifest_no_files(self): @@ -172,6 +191,7 @@ def test_uv_config_defaults(self): config = UvConfig() args = config.to_uv_args() self.assertEqual(args, []) + self.assertFalse(config.compile_bytecode) def test_uv_config_with_index_url(self): config = UvConfig(index_url="https://pypi.org/simple/")