Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion aws_lambda_builders/workflows/python_uv/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ config = {
"no_cache": False, # Disable caching
"prerelease": "disallow", # Handle pre-release versions
"resolution": "highest", # Resolution strategy
"compile_bytecode": True, # Compile .pyc files
"compile_bytecode": False, # Opt in to unchecked-hash .pyc compiled with the target Python
"exclude_newer": None, # Exclude packages newer than date
"generate_hashes": False, # Generate package hashes
}
Expand Down
68 changes: 65 additions & 3 deletions aws_lambda_builders/workflows/python_uv/packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,17 @@ def get_uv_version(self) -> Optional[str]:
pass
return None

def find_python(self, python_version: str) -> Optional[str]:
"""Find an already-installed interpreter matching the requested Python version."""
rc, stdout, stderr = self.run_uv_command(
["python", "find", "--no-project", "--no-python-downloads", python_version]

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

uv python find <version> checks discovered virtualenvs before managed and system interpreters. That's fine, because the magic number depends only on the minor version and .pyc is arch-independent. If you want to rule out a non-CPython interpreter, cpython@<version> is an explicit request. Not required.

)
if rc == 0 and stdout:
return stdout.strip()
diagnostic = (stderr or "").strip() or (stdout or "").strip() or "no diagnostic output"
LOG.warning("Could not locate target Python %s via uv (exit code %d): %s", python_version, rc, diagnostic)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[GENERAL] A missing target interpreter is the normal case for cross-version builds, so LOG.warning here will surface a scary message on healthy builds.

LOG.warning("Could not locate target Python %s via uv (exit code %d): %s", python_version, rc, diagnostic)

Building a python3.9 function on a host that only has 3.12 installed — or any build in an image whose interpreter differs from the runtime — hits this path, and --no-python-downloads guarantees uv will not fill the gap. Since bytecode compilation is a best-effort optimization whose absence has no effect on the produced artifact, this belongs at LOG.debug (or LOG.info), otherwise users see a warning in sam build output for a build that succeeded exactly as intended. Note this is amplified by the default in comment 1: with compilation on by default, most cross-version builds will emit it.

The same applies to the "Target Python version is unavailable" warning at line 155 — _extract_python_version() raises when there is no runtime, so a falsy python_version reaching install_requirements() only happens for direct library callers.

return None

def run_uv_command(self, args: List[str], cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> tuple:
"""
Execute UV command with given arguments.
Expand Down Expand Up @@ -165,15 +176,33 @@ def install_requirements(
# Resolve --target to an absolute path: UV runs from the project or workspace directory,
# so a relative target (e.g. the incremental-build dependencies dir) would otherwise be
# created under the UV's cwd instead of the build root.
args.extend(["--target", os.path.abspath(target_dir)])
target_dir = os.path.abspath(target_dir)
args.extend(["--target", target_dir])

# Add configuration arguments
target_python = None
if config.compile_bytecode:
Comment thread
aws-sam-tooling-bot[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UvConfig.compile_bytecode still defaults to True. On develop the flag is unused, so this change makes every build run compileall and ship .pyc files. Please change the default in UvConfig to False, update test_uv_config_defaults, and fix the DESIGN.md example.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing upstream sets compile_bytecode=True yet. UvConfig is built from the uv_* build options (workflow.py:153), and they don't include it, so this branch only runs in tests today. OK for this PR; worth a line in the description.

if python_version:
target_python = self._uv.find_python(python_version)
else:
LOG.warning("Target Python version is unavailable; skipping bytecode compilation")

# Add configuration arguments that are independent of interpreter selection.
args.extend(config.to_uv_args())

# Add platform-specific arguments
if python_version:
args.extend(["--python-version", python_version])

# Use the exact interpreter found by UV instead of repeating a version request that
# could resolve differently when the install command runs.
if target_python:
args.extend(["--python", target_python])

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[GENERAL] The --python pin is no longer needed and makes an optional optimization change how the install itself runs.

_compile_bytecode() invokes the discovered interpreter directly:

command = [python_executable, "-m", "compileall", "-f", "-q", "--invalidation-mode", "unchecked-hash", target_dir]

The resulting .pyc files depend only on that interpreter and the installed sources — not on which interpreter uv used. Meanwhile the installed sources are already governed by --python-version and --python-platform (both handlers always pass platform="linux" plus an architecture). So --python contributes nothing to the bytecode outcome, but it does override uv's own interpreter selection for the whole uv pip install invocation — including the interpreter used for any sdist build — replacing whatever uv would have chosen (e.g. an active virtualenv) with whatever uv python find happens to return.

Dropping the pin keeps the install behavior identical to today and confines the new code path to the compilation step:

if python_version:
   args.extend(["--python-version", python_version])

with target_python used only for _compile_bytecode().


# UV currently produces timestamp-based bytecode. SAM packages files with normalized
# timestamps, so that bytecode would be rejected by the Lambda runtime after deployment.
# Keep UV compilation disabled and generate unchecked-hash bytecode below instead.
args.append("--no-compile-bytecode")

if platform and architecture:
# UV pip install uses --python-platform format
# Map Lambda architectures to UV platform strings
Expand All @@ -193,14 +222,47 @@ def install_requirements(
python_platform = f"{target_arch}-manylinux_{manylinux_version}"
args.extend(["--python-platform", python_platform])

# uv rejects --no-compile-bytecode when UV_COMPILE_BYTECODE is also set ("cannot be used
# with"), so drop the variable for this command; bytecode is handled by _compile_bytecode.
env = self._osutils.original_environ()
env.pop("UV_COMPILE_BYTECODE", None)

# Execute UV pip install
rc, stdout, stderr = self._uv.run_uv_command(args, cwd)
rc, stdout, stderr = self._uv.run_uv_command(args, cwd, env=env)

if rc != 0:
raise UvInstallationError(reason=f"UV pip install failed: {stderr}")

LOG.debug("UV pip install completed successfully: %s", stdout)

if target_python:
self._compile_bytecode(target_python, target_dir)

def _compile_bytecode(self, python_executable: str, target_dir: str) -> None:
"""Compile reusable bytecode with the interpreter selected for the target runtime."""
command = [

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider -j 0 for compileall. Large dependency trees (#924 measured 977 files) compile serially otherwise.

python_executable,
"-m",
"compileall",
"-f",
"-q",
"--invalidation-mode",
"unchecked-hash",
target_dir,
]
LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command))
rc, stdout, stderr = self._osutils.run_subprocess(command)
if rc != 0:
diagnostic = stderr.strip() or stdout.strip() or "no diagnostic output"
LOG.warning(
"Could not compile unchecked-hash bytecode with target Python (exit code %d): %s",
rc,
diagnostic,
)
return

LOG.debug("Python bytecode compilation completed successfully: %s", stdout)


class PythonUvDependencyBuilder:
"""High-level dependency builder that orchestrates UV operations."""
Expand Down
13 changes: 11 additions & 2 deletions aws_lambda_builders/workflows/python_uv/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,16 @@ def run_subprocess(self, cmd, cwd=None, env=None):
env = self.original_environ()

try:
result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=False)
result = subprocess.run(
cmd,
cwd=cwd,
env=env,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=False,
)
return result.returncode, result.stdout, result.stderr
except Exception as e:
return 1, "", str(e)
Expand Down Expand Up @@ -133,7 +142,7 @@ def __init__(
no_cache: bool = False,
prerelease: str = "disallow",
resolution: str = "highest",
compile_bytecode: bool = True,
compile_bytecode: bool = False,
exclude_newer: Optional[str] = None,
generate_hashes: bool = False,
groups: Optional[List[str]] = None,
Expand Down
36 changes: 36 additions & 0 deletions tests/functional/workflows/python_uv/test_packager.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import sys
from collections import defaultdict, namedtuple

import pytest
Expand Down Expand Up @@ -659,6 +660,41 @@ def test_can_build_with_hash_verification(self, tmpdir, uv_runner, osutils):

assert "certifi" in installed_packages

def test_can_compile_unchecked_hash_bytecode_when_enabled(self, tmpdir, uv_runner, osutils):
"""Opt-in bytecode uses the interpreter from `uv python find` and writes unchecked-hash .pyc"""
reqs = ["requests"]
fake_uv, runner = uv_runner
appdir, builder, manifest_path = self._make_appdir_and_dependency_builder(
reqs, tmpdir, runner, manifest_type="requirements"
)

# Every fake UV call returns this str; `uv python find` uses it as the interpreter path.
fake_uv.set_return_tuple(0, sys.executable, "")

site_packages = os.path.join(appdir, "site-packages")
os.makedirs(site_packages, exist_ok=True)
fake_uv.packages_to_install(expected_args=["pip", "install"], packages=reqs, install_location=site_packages)

with osutils.tempdir() as scratch_dir:
builder.build_dependencies(
artifacts_dir_path=site_packages,
scratch_dir_path=scratch_dir,
manifest_path=manifest_path,
architecture=X86_64,
config=UvConfig(compile_bytecode=True),
)

python_find_calls = [args for args, _, _ in fake_uv._calls["python"] if args[:2] == ["python", "find"]]
assert python_find_calls

cache_dir = os.path.join(site_packages, "requests", "__pycache__")
pyc_files = os.listdir(cache_dir)
assert pyc_files
with open(os.path.join(cache_dir, pyc_files[0]), "rb") as f:
header = f.read(8)
# PEP 552 flags word: 0b01 = hash-based, source not checked (unchecked-hash).
assert int.from_bytes(header[4:8], "little") == 1

def test_lock_export_uses_build_options(self, tmpdir, uv_runner, osutils):
"""Build options select groups and extras"""
fake_uv, runner = uv_runner
Expand Down
169 changes: 169 additions & 0 deletions tests/unit/workflows/python_uv/test_packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,57 @@ def test_get_uv_version_failure(self, mock_osutils_class):

self.assertIsNone(version)

@patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils")
def test_find_python_success(self, mock_osutils_class):
mock_osutils = Mock()
mock_osutils.which.return_value = "/usr/bin/uv"
mock_osutils.run_subprocess.return_value = (0, "/usr/bin/python3.13\n", "")
mock_osutils_class.return_value = mock_osutils

subprocess_uv = SubprocessUv()

self.assertEqual(subprocess_uv.find_python("3.13"), "/usr/bin/python3.13")
mock_osutils.run_subprocess.assert_called_once_with(
["/usr/bin/uv", "python", "find", "--no-project", "--no-python-downloads", "3.13"],
cwd=None,
env=None,
)

@patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils")
def test_find_python_failure(self, mock_osutils_class):
mock_osutils = Mock()
mock_osutils.which.return_value = "/usr/bin/uv"
mock_osutils.run_subprocess.return_value = (2, "", "interpreter not found")
mock_osutils_class.return_value = mock_osutils

subprocess_uv = SubprocessUv()

with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs:
self.assertIsNone(subprocess_uv.find_python("3.9"))

self.assertIn("Could not locate target Python 3.9 via uv (exit code 2)", logs.output[0])
self.assertIn("interpreter not found", logs.output[0])

@patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils")
def test_find_python_failure_without_diagnostics(self, mock_osutils_class):
mock_osutils = Mock()
mock_osutils.which.return_value = "/usr/bin/uv"
mock_osutils.run_subprocess.return_value = (1, None, None)
mock_osutils_class.return_value = mock_osutils

subprocess_uv = SubprocessUv()

with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs:
self.assertIsNone(subprocess_uv.find_python("3.9"))

self.assertIn("no diagnostic output", logs.output[0])


class TestUvRunner(TestCase):
def setUp(self):
self.mock_subprocess_uv = Mock()
self.mock_subprocess_uv.uv_executable = "/usr/bin/uv"
self.mock_subprocess_uv.find_python.return_value = None
self.mock_osutils = Mock()
self.uv_runner = UvRunner(uv_subprocess=self.mock_subprocess_uv, osutils=self.mock_osutils)

Expand Down Expand Up @@ -196,6 +242,129 @@ def test_install_requirements_success(self):
self.assertIn("install", args_called)
self.assertIn("-r", args_called)
self.assertEqual(args_called[args_called.index("-r") + 1], os.path.abspath("/path/to/requirements.txt"))
self.assertIn("--no-compile-bytecode", args_called)
# Bytecode is opt-in: the default config must not look up a Python or run compileall.
self.mock_subprocess_uv.find_python.assert_not_called()
self.mock_osutils.run_subprocess.assert_not_called()

def test_install_requirements_drops_uv_compile_bytecode_env(self):
# uv refuses --no-compile-bytecode when UV_COMPILE_BYTECODE is set in the environment.
self.mock_osutils.original_environ.return_value = {"UV_COMPILE_BYTECODE": "1", "PATH": "/usr/bin"}
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")

self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
python_version="3.13",
)

args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0]
env_called = self.mock_subprocess_uv.run_uv_command.call_args.kwargs["env"]
self.assertIn("--no-compile-bytecode", args_called)
self.assertNotIn("UV_COMPILE_BYTECODE", env_called)
self.assertEqual(env_called["PATH"], "/usr/bin")

def test_install_requirements_compiles_unchecked_hash_bytecode_when_enabled(self):
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")
self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13"
self.mock_osutils.run_subprocess.return_value = (0, "", "")

self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
config=UvConfig(compile_bytecode=True),
python_version="3.13",
)

args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0]
self.assertIn("--no-compile-bytecode", args_called)
self.assertNotIn("--compile-bytecode", args_called)
self.assertEqual(args_called[args_called.index("--python") + 1], "/usr/bin/python3.13")
self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13")
self.mock_subprocess_uv.find_python.assert_called_once_with("3.13")
self.mock_osutils.run_subprocess.assert_called_once_with(
[
"/usr/bin/python3.13",
"-m",
"compileall",
"-f",
"-q",
"--invalidation-mode",
"unchecked-hash",
os.path.abspath("/target"),
]
)

def test_install_requirements_skips_bytecode_when_target_python_is_unavailable(self):
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")
self.mock_subprocess_uv.find_python.return_value = None

self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
config=UvConfig(compile_bytecode=True),
python_version="3.9",
)

args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0]
self.assertIn("--no-compile-bytecode", args_called)
self.assertNotIn("--compile-bytecode", args_called)
self.assertNotIn("--python", args_called)
self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.9")
self.mock_subprocess_uv.find_python.assert_called_once_with("3.9")
self.mock_osutils.run_subprocess.assert_not_called()

def test_install_requirements_skips_bytecode_without_target_python_version(self):
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")

with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs:
self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
config=UvConfig(compile_bytecode=True),
)

args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0]
self.assertIn("--no-compile-bytecode", args_called)
self.assertNotIn("--compile-bytecode", args_called)
self.assertNotIn("--python", args_called)
self.mock_subprocess_uv.find_python.assert_not_called()
self.assertIn("Target Python version is unavailable", logs.output[0])
self.mock_osutils.run_subprocess.assert_not_called()

def test_install_requirements_does_not_select_python_when_bytecode_disabled(self):
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")

self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
config=UvConfig(compile_bytecode=False),
python_version="3.13",
)

args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0]
self.assertIn("--no-compile-bytecode", args_called)
self.assertNotIn("--compile-bytecode", args_called)
self.assertNotIn("--python", args_called)
self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13")
self.mock_subprocess_uv.find_python.assert_not_called()
self.mock_osutils.run_subprocess.assert_not_called()

def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fails(self):
self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "")
self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13"
self.mock_osutils.run_subprocess.return_value = (1, "", "compile error")

with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs:
self.uv_runner.install_requirements(
requirements_path="/path/to/requirements.txt",
target_dir="/target",
config=UvConfig(compile_bytecode=True),
python_version="3.13",
)

self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0])
self.assertIn("compile error", logs.output[0])

def test_install_requirements_resolves_relative_target_to_absolute(self):
# UV runs from project_dir or workspace_dir,
Expand Down
Loading
Loading