From 677b67a6953cf807b0130d22f2dae7316833b50e Mon Sep 17 00:00:00 2001 From: tomaDev Date: Fri, 20 Nov 2026 12:18:39 +0200 Subject: [PATCH 1/2] fix(python-uv): compile the function's source to bytecode too With uv_compile_bytecode on, only the dependencies were compiled: compileall ran inside the dependency install, before the source was copied into the artifacts directory. Add PythonUvCompileSourceAction after CopySourceAction. It compiles what was copied from the source, so dependencies are not compiled twice, and it also covers builds that have no manifest. A missing uv or source directory skips it with a warning instead of failing the build. Co-Authored-By: Claude Fable 5.2 --- .../workflows/python_uv/DESIGN.md | 6 +- .../workflows/python_uv/actions.py | 41 +++++++++++ .../workflows/python_uv/packager.py | 24 ++++-- .../workflows/python_uv/workflow.py | 12 ++- .../workflows/python_uv/test_python_uv.py | 13 +++- .../unit/workflows/python_uv/test_actions.py | 73 +++++++++++++++++++ .../unit/workflows/python_uv/test_packager.py | 29 ++++++++ .../unit/workflows/python_uv/test_workflow.py | 22 ++++-- 8 files changed, 198 insertions(+), 22 deletions(-) create mode 100644 tests/unit/workflows/python_uv/test_actions.py diff --git a/aws_lambda_builders/workflows/python_uv/DESIGN.md b/aws_lambda_builders/workflows/python_uv/DESIGN.md index 06ac2198e..447408386 100644 --- a/aws_lambda_builders/workflows/python_uv/DESIGN.md +++ b/aws_lambda_builders/workflows/python_uv/DESIGN.md @@ -239,7 +239,7 @@ SAM CLI maps them from `Metadata.BuildProperties`. | `uv_extras` | `UvExtras` | list of str | `--extra ` per entry | | `uv_all_extras` | `UvAllExtras` | bool | `--all-extras` | | `uv_prune` | `UvPrune` | list of str | `--prune ` per entry: drops the package and any dependency only it needs | -| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; after install, runs `compileall --invalidation-mode unchecked-hash` with the target Python (also for `requirements.txt` builds) | +| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; runs `compileall --invalidation-mode unchecked-hash` with the target Python over the dependencies and the function's source (also for `requirements.txt` builds) | Default groups stay excluded (`--no-default-groups`); `uv_groups` adds groups on top of that. `uv_prune` is meant for packages the Lambda runtime already ships, such as `boto3`. The function then @@ -258,7 +258,9 @@ timestamps in the deployment package. So the workflow always passes `--no-compil `uv pip install`, removes `UV_COMPILE_BYTECODE` from that command's environment (uv refuses the flag and the variable together), and, when bytecode is enabled, runs the target Python (`uv python find --no-project --no-python-downloads `) as -`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory. +`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory. The function's +own source is compiled the same way after it is copied into the artifacts directory, so it gets bytecode +too, including when the build has no manifest. Bytecode is enabled by `uv_compile_bytecode: true`. When that option isn't given, a truthy `UV_COMPILE_BYTECODE` in the build environment enables it too, and the build logs that at INFO. Values are diff --git a/aws_lambda_builders/workflows/python_uv/actions.py b/aws_lambda_builders/workflows/python_uv/actions.py index 2497b877e..79e7a3fea 100644 --- a/aws_lambda_builders/workflows/python_uv/actions.py +++ b/aws_lambda_builders/workflows/python_uv/actions.py @@ -3,6 +3,8 @@ """ import logging +import os +import shutil from typing import Optional from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose @@ -74,3 +76,42 @@ def execute(self) -> None: ) except (LockFileError, MissingUvError, UvInstallationError, UvBuildError) as ex: raise ActionFailedError(str(ex)) + + +class PythonUvCompileSourceAction(BaseAction): + """Action for compiling the function's own source to bytecode, once it is in the artifacts directory.""" + + NAME = "CompileSource" + DESCRIPTION = "Compiling source to bytecode" + PURPOSE = Purpose.COMPILE_SOURCE + LANGUAGE = "python" + + def __init__(self, source_dir, artifacts_dir, runtime, excludes=None): + self.source_dir = source_dir + self.artifacts_dir = artifacts_dir + self.runtime = runtime + self.excludes = excludes or [] + + self._os_utils = OSUtils() + + def execute(self) -> None: + """Compile what CopySourceAction copied; the dependency build has already compiled the rest.""" + # Match CopySourceAction: a missing source was skipped there, so there is nothing to compile + if not os.path.isdir(self.source_dir): + return + + names = os.listdir(self.source_dir) + ignored = shutil.ignore_patterns(*self.excludes)(self.source_dir, names) + copied = [os.path.join(self.artifacts_dir, name) for name in sorted(names) if name not in ignored] + paths = [path for path in copied if os.path.isdir(path) or (path.endswith(".py") and os.path.isfile(path))] + if not paths: + return + + python_version = self.runtime.replace("python", "") if self.runtime else None + try: + uv_runner = UvRunner(uv_subprocess=SubprocessUv(osutils=self._os_utils), osutils=self._os_utils) + except MissingUvError as ex: + # Only reached when no dependencies were installed. Bytecode is optional, so don't fail the build + LOG.warning("Skipping bytecode compilation of the source: %s", ex) + return + uv_runner.compile_bytecode(python_version, paths) diff --git a/aws_lambda_builders/workflows/python_uv/packager.py b/aws_lambda_builders/workflows/python_uv/packager.py index 55f9344d9..b704acf58 100644 --- a/aws_lambda_builders/workflows/python_uv/packager.py +++ b/aws_lambda_builders/workflows/python_uv/packager.py @@ -179,12 +179,7 @@ def install_requirements( target_dir = os.path.abspath(target_dir) args.extend(["--target", target_dir]) - target_python = None - if config.compile_bytecode: - if python_version: - target_python = self._uv.find_python(python_version) - else: - LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + target_python = self._find_target_python(python_version) if config.compile_bytecode else None # Add configuration arguments that are independent of interpreter selection. args.extend(config.to_uv_args()) @@ -238,7 +233,20 @@ def install_requirements( if target_python: self._compile_bytecode(target_python, target_dir) - def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: + def compile_bytecode(self, python_version: Optional[str], paths: List[str]) -> None: + """Compile unchecked-hash bytecode for the given files and directories with the target Python.""" + target_python = self._find_target_python(python_version) + if target_python: + self._compile_bytecode(target_python, *paths) + + def _find_target_python(self, python_version: Optional[str]) -> Optional[str]: + """Find the interpreter that compiles bytecode for the target runtime, or None if there isn't one.""" + if not python_version: + LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + return None + return self._uv.find_python(python_version) + + def _compile_bytecode(self, python_executable: str, *paths: str) -> None: """Compile reusable bytecode with the interpreter selected for the target runtime.""" command = [ python_executable, @@ -248,7 +256,7 @@ def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: "-q", "--invalidation-mode", "unchecked-hash", - target_dir, + *paths, ] LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command)) rc, stdout, stderr = self._osutils.run_subprocess(command) diff --git a/aws_lambda_builders/workflows/python_uv/workflow.py b/aws_lambda_builders/workflows/python_uv/workflow.py index 6181febae..837d865ab 100644 --- a/aws_lambda_builders/workflows/python_uv/workflow.py +++ b/aws_lambda_builders/workflows/python_uv/workflow.py @@ -9,7 +9,7 @@ from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability -from .actions import PythonUvBuildAction +from .actions import PythonUvBuildAction, PythonUvCompileSourceAction from .utils import OSUtils, UvConfig, detect_uv_manifest, normalize_package_name LOG = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def _compile_bytecode_from_env(): if value is None or value.lower() in UV_ENV_FALSE: return False if value.lower() in UV_ENV_TRUE: - LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode for dependencies", value) + LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode", value) return True LOG.warning("Ignoring UV_COMPILE_BYTECODE=%r: expected one of %s", value, ", ".join(UV_ENV_TRUE + UV_ENV_FALSE)) return False @@ -256,9 +256,15 @@ def _setup_build_actions(self, source_dir, artifacts_dir, scratch_dir, manifest_ ) ) - # Always copy source code (final step) + # Always copy source code self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + # The dependency build compiles what it installs; the source is only in place now + if config and config.compile_bytecode: + self.actions.append( + PythonUvCompileSourceAction(source_dir, artifacts_dir, runtime, excludes=self.EXCLUDED_FILES) + ) + def get_resolvers(self): """ Get path resolvers for the workflow. diff --git a/tests/integration/workflows/python_uv/test_python_uv.py b/tests/integration/workflows/python_uv/test_python_uv.py index df1960b00..0d87c5acd 100644 --- a/tests/integration/workflows/python_uv/test_python_uv.py +++ b/tests/integration/workflows/python_uv/test_python_uv.py @@ -734,6 +734,9 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self): os.environ.pop("UV_COMPILE_BYTECODE", None) requirements = os.path.join(source_dir, "requirements.txt") pathlib.Path(requirements).write_text("six==1.16.0\n") + pathlib.Path(source_dir, "app.py").write_text("import six\n") + pathlib.Path(source_dir, "pkg").mkdir() + pathlib.Path(source_dir, "pkg", "mod.py").write_text("VALUE = 1\n") builder = LambdaBuilder(language="python", dependency_manager="uv", application_framework=None) runtime = f"python{sys.version_info.major}.{sys.version_info.minor}" @@ -757,10 +760,12 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self): experimental_flags=self.experimental_flags, options={"uv_compile_bytecode": True}, ) - pyc_files = list(pathlib.Path(compiled_dir, "__pycache__").glob("six.*.pyc")) - self.assertEqual(len(pyc_files), 1) - # PEP 552 flags word: 1 = hash-based, unchecked - self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1) + # The dependency, and the function's own source (top level and in a package) + for pycache, module in (("__pycache__", "six"), ("__pycache__", "app"), ("pkg/__pycache__", "mod")): + pyc_files = list(pathlib.Path(compiled_dir, pycache).glob(f"{module}.*.pyc")) + self.assertEqual(len(pyc_files), 1, module) + # PEP 552 flags word: 1 = hash-based, unchecked + self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1, module) @skipIf(which("uv") is None, "uv not available") def test_workflow_shows_uv_warnings(self): diff --git a/tests/unit/workflows/python_uv/test_actions.py b/tests/unit/workflows/python_uv/test_actions.py new file mode 100644 index 000000000..92e7f1c3b --- /dev/null +++ b/tests/unit/workflows/python_uv/test_actions.py @@ -0,0 +1,73 @@ +import os +import shutil +import tempfile +from pathlib import Path +from unittest import TestCase +from unittest.mock import patch + +from aws_lambda_builders.workflows.python_uv.actions import PythonUvCompileSourceAction +from aws_lambda_builders.workflows.python_uv.exceptions import MissingUvError + + +class TestPythonUvCompileSourceAction(TestCase): + def setUp(self): + self.source_dir = tempfile.mkdtemp() + self.artifacts_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.source_dir) + self.addCleanup(shutil.rmtree, self.artifacts_dir) + + def _execute(self, runtime="python3.13", excludes=("venv", "*.egg-info")): + with ( + patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv"), + patch("aws_lambda_builders.workflows.python_uv.actions.UvRunner") as runner_class, + ): + PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, runtime, excludes).execute() + return runner_class.return_value.compile_bytecode + + def test_compiles_only_what_was_copied_from_the_source(self): + for root in (self.source_dir, self.artifacts_dir): + Path(root, "app.py").touch() + Path(root, "pkg").mkdir() + Path(root, "requirements.txt").touch() + # Excluded from the copy, so not part of the function's source + Path(self.source_dir, "venv").mkdir() + Path(self.source_dir, "app.egg-info").mkdir() + # Installed dependencies, already compiled by the dependency build + Path(self.artifacts_dir, "six.py").touch() + Path(self.artifacts_dir, "requests").mkdir() + + compile_bytecode = self._execute() + + compile_bytecode.assert_called_once_with( + "3.13", [os.path.join(self.artifacts_dir, "app.py"), os.path.join(self.artifacts_dir, "pkg")] + ) + + def test_does_nothing_without_python_source(self): + Path(self.source_dir, "requirements.txt").touch() + Path(self.artifacts_dir, "requirements.txt").touch() + + self._execute().assert_not_called() + + def test_passes_no_python_version_without_a_runtime(self): + Path(self.source_dir, "app.py").touch() + Path(self.artifacts_dir, "app.py").touch() + + self._execute(runtime=None).assert_called_once_with(None, [os.path.join(self.artifacts_dir, "app.py")]) + + def test_does_nothing_without_a_source_directory(self): + shutil.rmtree(self.source_dir) + self.addCleanup(os.mkdir, self.source_dir) + + self._execute().assert_not_called() + + def test_warns_instead_of_failing_when_uv_is_missing(self): + Path(self.source_dir, "app.py").touch() + Path(self.artifacts_dir, "app.py").touch() + + with ( + patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv", side_effect=MissingUvError()), + self.assertLogs("aws_lambda_builders.workflows.python_uv.actions", level="WARNING") as logs, + ): + PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, "python3.13").execute() + + self.assertIn("Skipping bytecode compilation of the source: uv executable not found", logs.output[0]) diff --git a/tests/unit/workflows/python_uv/test_packager.py b/tests/unit/workflows/python_uv/test_packager.py index 518acd440..3037e1745 100644 --- a/tests/unit/workflows/python_uv/test_packager.py +++ b/tests/unit/workflows/python_uv/test_packager.py @@ -366,6 +366,35 @@ def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fa self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0]) self.assertIn("compile error", logs.output[0]) + def test_compile_bytecode_compiles_the_given_paths_with_the_target_python(self): + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (0, "", "") + + self.uv_runner.compile_bytecode("3.13", ["/artifacts/app.py", "/artifacts/pkg"]) + + self.mock_subprocess_uv.find_python.assert_called_once_with("3.13") + self.mock_osutils.run_subprocess.assert_called_once_with( + [ + "/usr/bin/python3.13", + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + "/artifacts/app.py", + "/artifacts/pkg", + ] + ) + + @parameterized.expand([("no_target_python", "3.13"), ("no_python_version", None)]) + def test_compile_bytecode_skips_when_the_target_python_is_unknown(self, _, python_version): + self.mock_subprocess_uv.find_python.return_value = None + + self.uv_runner.compile_bytecode(python_version, ["/artifacts/app.py"]) + + self.mock_osutils.run_subprocess.assert_not_called() + def test_install_requirements_resolves_relative_target_to_absolute(self): # UV runs from project_dir or workspace_dir, # so a relative --target must be resolved to an absolute path first, diff --git a/tests/unit/workflows/python_uv/test_workflow.py b/tests/unit/workflows/python_uv/test_workflow.py index 3e672155b..06068a133 100644 --- a/tests/unit/workflows/python_uv/test_workflow.py +++ b/tests/unit/workflows/python_uv/test_workflow.py @@ -8,7 +8,7 @@ from aws_lambda_builders.actions import CleanUpAction, CopyDependenciesAction, CopySourceAction from aws_lambda_builders.exceptions import WorkflowFailedError -from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction +from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction, PythonUvCompileSourceAction from aws_lambda_builders.workflows.python_uv.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE, OSUtils from aws_lambda_builders.workflows.python_uv.workflow import PythonUvWorkflow @@ -251,6 +251,20 @@ def test_compile_bytecode_option_is_passed_to_the_build_action(self): config = self._workflow({"uv_compile_bytecode": True}).actions[0].config self.assertTrue(config.compile_bytecode) + def test_compile_bytecode_compiles_the_source_after_copying_it(self): + workflow = self._workflow({"uv_compile_bytecode": True}) + self.assertIsInstance(workflow.actions[-2], CopySourceAction) + compile_action = workflow.actions[-1] + self.assertIsInstance(compile_action, PythonUvCompileSourceAction) + self.assertEqual(compile_action.source_dir, self.source_dir) + self.assertEqual(compile_action.artifacts_dir, "artifacts") + self.assertEqual(compile_action.runtime, "python3.12") + self.assertEqual(compile_action.excludes, PythonUvWorkflow.EXCLUDED_FILES) + + def test_source_is_not_compiled_by_default(self): + workflow = self._workflow({"build_logical_id": "Fn"}) + self.assertIsInstance(workflow.actions[-1], CopySourceAction) + def test_compile_bytecode_on_requirements_build_is_allowed(self): requirements = os.path.join(self.source_dir, "requirements.txt") open(requirements, "w").close() @@ -260,15 +274,13 @@ def test_compile_bytecode_on_requirements_build_is_allowed(self): def test_compile_bytecode_without_manifest_builds_without_dependencies(self): os.remove(self.pyproject) workflow = self._workflow({"uv_compile_bytecode": True}, manifest=os.path.join(self.source_dir, "missing.txt")) - self.assertEqual(len(workflow.actions), 1) - self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction]) def test_truthy_env_without_manifest_builds_without_dependencies(self): os.environ["UV_COMPILE_BYTECODE"] = "1" os.remove(self.pyproject) workflow = self._workflow({"build_logical_id": "Fn"}, manifest=os.path.join(self.source_dir, "missing.txt")) - self.assertEqual(len(workflow.actions), 1) - self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction]) def test_export_option_with_compile_bytecode_on_requirements_build_fails(self): requirements = os.path.join(self.source_dir, "requirements.txt") From e666ced7c48d2f846f6c5d0b21e7e19532c0c01e Mon Sep 17 00:00:00 2001 From: tomaDev Date: Sat, 21 Nov 2026 06:03:39 +0200 Subject: [PATCH 2/2] test(python-uv): pin the exclude filter in the compile-source action test The excluded entries only existed in the source directory, so the artifacts existence check dropped them and the filter was never exercised. Put them in the artifacts directory too. Co-Authored-By: Claude Fable 5.2 --- tests/unit/workflows/python_uv/test_actions.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/unit/workflows/python_uv/test_actions.py b/tests/unit/workflows/python_uv/test_actions.py index 92e7f1c3b..e47355c73 100644 --- a/tests/unit/workflows/python_uv/test_actions.py +++ b/tests/unit/workflows/python_uv/test_actions.py @@ -29,9 +29,11 @@ def test_compiles_only_what_was_copied_from_the_source(self): Path(root, "app.py").touch() Path(root, "pkg").mkdir() Path(root, "requirements.txt").touch() - # Excluded from the copy, so not part of the function's source - Path(self.source_dir, "venv").mkdir() - Path(self.source_dir, "app.egg-info").mkdir() + # Excluded from the copy, so not part of the function's source, even where the + # artifacts directory has an entry of the same name + for root in (self.source_dir, self.artifacts_dir): + Path(root, "venv").mkdir() + Path(root, "app.egg-info").mkdir() # Installed dependencies, already compiled by the dependency build Path(self.artifacts_dir, "six.py").touch() Path(self.artifacts_dir, "requests").mkdir()