Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions aws_lambda_builders/workflows/python_uv/DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ SAM CLI maps them from `Metadata.BuildProperties`.
| `uv_extras` | `UvExtras` | list of str | `--extra <name>` per entry |
| `uv_all_extras` | `UvAllExtras` | bool | `--all-extras` |
| `uv_prune` | `UvPrune` | list of str | `--prune <name>` per entry: drops the package and any dependency only it needs |
| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; after install, runs `compileall --invalidation-mode unchecked-hash` with the target Python (also for `requirements.txt` builds) |
| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; runs `compileall --invalidation-mode unchecked-hash` with the target Python over the dependencies and the function's source (also for `requirements.txt` builds) |

Default groups stay excluded (`--no-default-groups`); `uv_groups` adds groups on top of that.
`uv_prune` is meant for packages the Lambda runtime already ships, such as `boto3`. The function then
Expand All @@ -258,7 +258,9 @@ timestamps in the deployment package. So the workflow always passes `--no-compil
`uv pip install`, removes `UV_COMPILE_BYTECODE` from that command's environment (uv refuses the flag and
the variable together), and, when bytecode is enabled, runs the target Python
(`uv python find --no-project --no-python-downloads <version>`) as
`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory.
`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory. The function's
own source is compiled the same way after it is copied into the artifacts directory, so it gets bytecode
too, including when the build has no manifest.

Bytecode is enabled by `uv_compile_bytecode: true`. When that option isn't given, a truthy
`UV_COMPILE_BYTECODE` in the build environment enables it too, and the build logs that at INFO. Values are
Expand Down
41 changes: 41 additions & 0 deletions aws_lambda_builders/workflows/python_uv/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
"""

import logging
import os
import shutil
from typing import Optional

from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose
Expand Down Expand Up @@ -74,3 +76,42 @@ def execute(self) -> None:
)
except (LockFileError, MissingUvError, UvInstallationError, UvBuildError) as ex:
raise ActionFailedError(str(ex))


class PythonUvCompileSourceAction(BaseAction):
"""Action for compiling the function's own source to bytecode, once it is in the artifacts directory."""

NAME = "CompileSource"
DESCRIPTION = "Compiling source to bytecode"
PURPOSE = Purpose.COMPILE_SOURCE
LANGUAGE = "python"

def __init__(self, source_dir, artifacts_dir, runtime, excludes=None):
self.source_dir = source_dir
self.artifacts_dir = artifacts_dir
self.runtime = runtime
self.excludes = excludes or []

self._os_utils = OSUtils()

def execute(self) -> None:
"""Compile what CopySourceAction copied; the dependency build has already compiled the rest."""
# Match CopySourceAction: a missing source was skipped there, so there is nothing to compile
if not os.path.isdir(self.source_dir):
return

names = os.listdir(self.source_dir)
ignored = shutil.ignore_patterns(*self.excludes)(self.source_dir, names)
copied = [os.path.join(self.artifacts_dir, name) for name in sorted(names) if name not in ignored]
Comment thread
tomaDev marked this conversation as resolved.
Resolved
paths = [path for path in copied if os.path.isdir(path) or (path.endswith(".py") and os.path.isfile(path))]
if not paths:
return

python_version = self.runtime.replace("python", "") if self.runtime else None
try:
uv_runner = UvRunner(uv_subprocess=SubprocessUv(osutils=self._os_utils), osutils=self._os_utils)
except MissingUvError as ex:
# Only reached when no dependencies were installed. Bytecode is optional, so don't fail the build
LOG.warning("Skipping bytecode compilation of the source: %s", ex)
return
uv_runner.compile_bytecode(python_version, paths)
24 changes: 16 additions & 8 deletions aws_lambda_builders/workflows/python_uv/packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,12 +179,7 @@ def install_requirements(
target_dir = os.path.abspath(target_dir)
args.extend(["--target", target_dir])

target_python = None
if config.compile_bytecode:
if python_version:
target_python = self._uv.find_python(python_version)
else:
LOG.warning("Target Python version is unavailable; skipping bytecode compilation")
target_python = self._find_target_python(python_version) if config.compile_bytecode else None

# Add configuration arguments that are independent of interpreter selection.
args.extend(config.to_uv_args())
Expand Down Expand Up @@ -238,7 +233,20 @@ def install_requirements(
if target_python:
self._compile_bytecode(target_python, target_dir)

def _compile_bytecode(self, python_executable: str, target_dir: str) -> None:
def compile_bytecode(self, python_version: Optional[str], paths: List[str]) -> None:
"""Compile unchecked-hash bytecode for the given files and directories with the target Python."""
target_python = self._find_target_python(python_version)
if target_python:
self._compile_bytecode(target_python, *paths)

def _find_target_python(self, python_version: Optional[str]) -> Optional[str]:
"""Find the interpreter that compiles bytecode for the target runtime, or None if there isn't one."""
if not python_version:
LOG.warning("Target Python version is unavailable; skipping bytecode compilation")
return None
return self._uv.find_python(python_version)

def _compile_bytecode(self, python_executable: str, *paths: str) -> None:
"""Compile reusable bytecode with the interpreter selected for the target runtime."""
command = [
python_executable,
Expand All @@ -248,7 +256,7 @@ def _compile_bytecode(self, python_executable: str, target_dir: str) -> None:
"-q",
"--invalidation-mode",
"unchecked-hash",
target_dir,
*paths,
]
LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command))
rc, stdout, stderr = self._osutils.run_subprocess(command)
Expand Down
12 changes: 9 additions & 3 deletions aws_lambda_builders/workflows/python_uv/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from aws_lambda_builders.exceptions import WorkflowFailedError
from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability

from .actions import PythonUvBuildAction
from .actions import PythonUvBuildAction, PythonUvCompileSourceAction
from .utils import OSUtils, UvConfig, detect_uv_manifest, normalize_package_name

LOG = logging.getLogger(__name__)
Expand Down Expand Up @@ -38,7 +38,7 @@ def _compile_bytecode_from_env():
if value is None or value.lower() in UV_ENV_FALSE:
return False
if value.lower() in UV_ENV_TRUE:
LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode for dependencies", value)
LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode", value)
return True
LOG.warning("Ignoring UV_COMPILE_BYTECODE=%r: expected one of %s", value, ", ".join(UV_ENV_TRUE + UV_ENV_FALSE))
return False
Expand Down Expand Up @@ -256,9 +256,15 @@ def _setup_build_actions(self, source_dir, artifacts_dir, scratch_dir, manifest_
)
)

# Always copy source code (final step)
# Always copy source code
self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))

# The dependency build compiles what it installs; the source is only in place now
if config and config.compile_bytecode:
self.actions.append(
PythonUvCompileSourceAction(source_dir, artifacts_dir, runtime, excludes=self.EXCLUDED_FILES)
)

def get_resolvers(self):
"""
Get path resolvers for the workflow.
Expand Down
13 changes: 9 additions & 4 deletions tests/integration/workflows/python_uv/test_python_uv.py
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,9 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self):
os.environ.pop("UV_COMPILE_BYTECODE", None)
requirements = os.path.join(source_dir, "requirements.txt")
pathlib.Path(requirements).write_text("six==1.16.0\n")
pathlib.Path(source_dir, "app.py").write_text("import six\n")
pathlib.Path(source_dir, "pkg").mkdir()
pathlib.Path(source_dir, "pkg", "mod.py").write_text("VALUE = 1\n")
builder = LambdaBuilder(language="python", dependency_manager="uv", application_framework=None)
runtime = f"python{sys.version_info.major}.{sys.version_info.minor}"

Expand All @@ -757,10 +760,12 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self):
experimental_flags=self.experimental_flags,
options={"uv_compile_bytecode": True},
)
pyc_files = list(pathlib.Path(compiled_dir, "__pycache__").glob("six.*.pyc"))
self.assertEqual(len(pyc_files), 1)
# PEP 552 flags word: 1 = hash-based, unchecked
self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1)
# The dependency, and the function's own source (top level and in a package)
for pycache, module in (("__pycache__", "six"), ("__pycache__", "app"), ("pkg/__pycache__", "mod")):
pyc_files = list(pathlib.Path(compiled_dir, pycache).glob(f"{module}.*.pyc"))
self.assertEqual(len(pyc_files), 1, module)
# PEP 552 flags word: 1 = hash-based, unchecked
self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1, module)

@skipIf(which("uv") is None, "uv not available")
def test_workflow_shows_uv_warnings(self):
Expand Down
75 changes: 75 additions & 0 deletions tests/unit/workflows/python_uv/test_actions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import os
import shutil
import tempfile
from pathlib import Path
from unittest import TestCase
from unittest.mock import patch

from aws_lambda_builders.workflows.python_uv.actions import PythonUvCompileSourceAction
from aws_lambda_builders.workflows.python_uv.exceptions import MissingUvError


class TestPythonUvCompileSourceAction(TestCase):
def setUp(self):
self.source_dir = tempfile.mkdtemp()
self.artifacts_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.source_dir)
self.addCleanup(shutil.rmtree, self.artifacts_dir)

def _execute(self, runtime="python3.13", excludes=("venv", "*.egg-info")):
with (
patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv"),
patch("aws_lambda_builders.workflows.python_uv.actions.UvRunner") as runner_class,
):
PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, runtime, excludes).execute()
return runner_class.return_value.compile_bytecode

def test_compiles_only_what_was_copied_from_the_source(self):
for root in (self.source_dir, self.artifacts_dir):
Path(root, "app.py").touch()
Path(root, "pkg").mkdir()
Path(root, "requirements.txt").touch()
# Excluded from the copy, so not part of the function's source, even where the
# artifacts directory has an entry of the same name
for root in (self.source_dir, self.artifacts_dir):
Path(root, "venv").mkdir()
Path(root, "app.egg-info").mkdir()
# Installed dependencies, already compiled by the dependency build
Path(self.artifacts_dir, "six.py").touch()
Path(self.artifacts_dir, "requests").mkdir()

compile_bytecode = self._execute()

compile_bytecode.assert_called_once_with(
"3.13", [os.path.join(self.artifacts_dir, "app.py"), os.path.join(self.artifacts_dir, "pkg")]
)

def test_does_nothing_without_python_source(self):
Path(self.source_dir, "requirements.txt").touch()
Path(self.artifacts_dir, "requirements.txt").touch()

self._execute().assert_not_called()

def test_passes_no_python_version_without_a_runtime(self):
Path(self.source_dir, "app.py").touch()
Path(self.artifacts_dir, "app.py").touch()

self._execute(runtime=None).assert_called_once_with(None, [os.path.join(self.artifacts_dir, "app.py")])

def test_does_nothing_without_a_source_directory(self):
shutil.rmtree(self.source_dir)
self.addCleanup(os.mkdir, self.source_dir)

self._execute().assert_not_called()

def test_warns_instead_of_failing_when_uv_is_missing(self):
Path(self.source_dir, "app.py").touch()
Path(self.artifacts_dir, "app.py").touch()

with (
patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv", side_effect=MissingUvError()),
self.assertLogs("aws_lambda_builders.workflows.python_uv.actions", level="WARNING") as logs,
):
PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, "python3.13").execute()

self.assertIn("Skipping bytecode compilation of the source: uv executable not found", logs.output[0])
29 changes: 29 additions & 0 deletions tests/unit/workflows/python_uv/test_packager.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,35 @@ def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fa
self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0])
self.assertIn("compile error", logs.output[0])

def test_compile_bytecode_compiles_the_given_paths_with_the_target_python(self):
self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13"
self.mock_osutils.run_subprocess.return_value = (0, "", "")

self.uv_runner.compile_bytecode("3.13", ["/artifacts/app.py", "/artifacts/pkg"])

self.mock_subprocess_uv.find_python.assert_called_once_with("3.13")
self.mock_osutils.run_subprocess.assert_called_once_with(
[
"/usr/bin/python3.13",
"-m",
"compileall",
"-f",
"-q",
"--invalidation-mode",
"unchecked-hash",
"/artifacts/app.py",
"/artifacts/pkg",
]
)

@parameterized.expand([("no_target_python", "3.13"), ("no_python_version", None)])
def test_compile_bytecode_skips_when_the_target_python_is_unknown(self, _, python_version):
self.mock_subprocess_uv.find_python.return_value = None

self.uv_runner.compile_bytecode(python_version, ["/artifacts/app.py"])

self.mock_osutils.run_subprocess.assert_not_called()

def test_install_requirements_resolves_relative_target_to_absolute(self):
# UV runs from project_dir or workspace_dir,
# so a relative --target must be resolved to an absolute path first,
Expand Down
22 changes: 17 additions & 5 deletions tests/unit/workflows/python_uv/test_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from aws_lambda_builders.actions import CleanUpAction, CopyDependenciesAction, CopySourceAction
from aws_lambda_builders.exceptions import WorkflowFailedError
from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction
from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction, PythonUvCompileSourceAction
from aws_lambda_builders.workflows.python_uv.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE, OSUtils
from aws_lambda_builders.workflows.python_uv.workflow import PythonUvWorkflow

Expand Down Expand Up @@ -251,6 +251,20 @@ def test_compile_bytecode_option_is_passed_to_the_build_action(self):
config = self._workflow({"uv_compile_bytecode": True}).actions[0].config
self.assertTrue(config.compile_bytecode)

def test_compile_bytecode_compiles_the_source_after_copying_it(self):
workflow = self._workflow({"uv_compile_bytecode": True})
self.assertIsInstance(workflow.actions[-2], CopySourceAction)
compile_action = workflow.actions[-1]
self.assertIsInstance(compile_action, PythonUvCompileSourceAction)
self.assertEqual(compile_action.source_dir, self.source_dir)
self.assertEqual(compile_action.artifacts_dir, "artifacts")
self.assertEqual(compile_action.runtime, "python3.12")
self.assertEqual(compile_action.excludes, PythonUvWorkflow.EXCLUDED_FILES)

def test_source_is_not_compiled_by_default(self):
workflow = self._workflow({"build_logical_id": "Fn"})
self.assertIsInstance(workflow.actions[-1], CopySourceAction)

def test_compile_bytecode_on_requirements_build_is_allowed(self):
requirements = os.path.join(self.source_dir, "requirements.txt")
open(requirements, "w").close()
Expand All @@ -260,15 +274,13 @@ def test_compile_bytecode_on_requirements_build_is_allowed(self):
def test_compile_bytecode_without_manifest_builds_without_dependencies(self):
os.remove(self.pyproject)
workflow = self._workflow({"uv_compile_bytecode": True}, manifest=os.path.join(self.source_dir, "missing.txt"))
self.assertEqual(len(workflow.actions), 1)
self.assertIsInstance(workflow.actions[0], CopySourceAction)
self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction])

def test_truthy_env_without_manifest_builds_without_dependencies(self):
os.environ["UV_COMPILE_BYTECODE"] = "1"
os.remove(self.pyproject)
workflow = self._workflow({"build_logical_id": "Fn"}, manifest=os.path.join(self.source_dir, "missing.txt"))
self.assertEqual(len(workflow.actions), 1)
self.assertIsInstance(workflow.actions[0], CopySourceAction)
self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction])

def test_export_option_with_compile_bytecode_on_requirements_build_fails(self):
requirements = os.path.join(self.source_dir, "requirements.txt")
Expand Down