diff --git a/aws_lambda_builders/workflows/python_uv/DESIGN.md b/aws_lambda_builders/workflows/python_uv/DESIGN.md index 06ac2198e..447408386 100644 --- a/aws_lambda_builders/workflows/python_uv/DESIGN.md +++ b/aws_lambda_builders/workflows/python_uv/DESIGN.md @@ -239,7 +239,7 @@ SAM CLI maps them from `Metadata.BuildProperties`. | `uv_extras` | `UvExtras` | list of str | `--extra ` per entry | | `uv_all_extras` | `UvAllExtras` | bool | `--all-extras` | | `uv_prune` | `UvPrune` | list of str | `--prune ` per entry: drops the package and any dependency only it needs | -| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; after install, runs `compileall --invalidation-mode unchecked-hash` with the target Python (also for `requirements.txt` builds) | +| `uv_compile_bytecode` | `UvCompileBytecode` | bool | none; runs `compileall --invalidation-mode unchecked-hash` with the target Python over the dependencies and the function's source (also for `requirements.txt` builds) | Default groups stay excluded (`--no-default-groups`); `uv_groups` adds groups on top of that. `uv_prune` is meant for packages the Lambda runtime already ships, such as `boto3`. The function then @@ -258,7 +258,9 @@ timestamps in the deployment package. So the workflow always passes `--no-compil `uv pip install`, removes `UV_COMPILE_BYTECODE` from that command's environment (uv refuses the flag and the variable together), and, when bytecode is enabled, runs the target Python (`uv python find --no-project --no-python-downloads `) as -`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory. +`python -m compileall -f -q --invalidation-mode unchecked-hash` over the install directory. The function's +own source is compiled the same way after it is copied into the artifacts directory, so it gets bytecode +too, including when the build has no manifest. Bytecode is enabled by `uv_compile_bytecode: true`. When that option isn't given, a truthy `UV_COMPILE_BYTECODE` in the build environment enables it too, and the build logs that at INFO. Values are diff --git a/aws_lambda_builders/workflows/python_uv/actions.py b/aws_lambda_builders/workflows/python_uv/actions.py index 2497b877e..79e7a3fea 100644 --- a/aws_lambda_builders/workflows/python_uv/actions.py +++ b/aws_lambda_builders/workflows/python_uv/actions.py @@ -3,6 +3,8 @@ """ import logging +import os +import shutil from typing import Optional from aws_lambda_builders.actions import ActionFailedError, BaseAction, Purpose @@ -74,3 +76,42 @@ def execute(self) -> None: ) except (LockFileError, MissingUvError, UvInstallationError, UvBuildError) as ex: raise ActionFailedError(str(ex)) + + +class PythonUvCompileSourceAction(BaseAction): + """Action for compiling the function's own source to bytecode, once it is in the artifacts directory.""" + + NAME = "CompileSource" + DESCRIPTION = "Compiling source to bytecode" + PURPOSE = Purpose.COMPILE_SOURCE + LANGUAGE = "python" + + def __init__(self, source_dir, artifacts_dir, runtime, excludes=None): + self.source_dir = source_dir + self.artifacts_dir = artifacts_dir + self.runtime = runtime + self.excludes = excludes or [] + + self._os_utils = OSUtils() + + def execute(self) -> None: + """Compile what CopySourceAction copied; the dependency build has already compiled the rest.""" + # Match CopySourceAction: a missing source was skipped there, so there is nothing to compile + if not os.path.isdir(self.source_dir): + return + + names = os.listdir(self.source_dir) + ignored = shutil.ignore_patterns(*self.excludes)(self.source_dir, names) + copied = [os.path.join(self.artifacts_dir, name) for name in sorted(names) if name not in ignored] + paths = [path for path in copied if os.path.isdir(path) or (path.endswith(".py") and os.path.isfile(path))] + if not paths: + return + + python_version = self.runtime.replace("python", "") if self.runtime else None + try: + uv_runner = UvRunner(uv_subprocess=SubprocessUv(osutils=self._os_utils), osutils=self._os_utils) + except MissingUvError as ex: + # Only reached when no dependencies were installed. Bytecode is optional, so don't fail the build + LOG.warning("Skipping bytecode compilation of the source: %s", ex) + return + uv_runner.compile_bytecode(python_version, paths) diff --git a/aws_lambda_builders/workflows/python_uv/packager.py b/aws_lambda_builders/workflows/python_uv/packager.py index 55f9344d9..b704acf58 100644 --- a/aws_lambda_builders/workflows/python_uv/packager.py +++ b/aws_lambda_builders/workflows/python_uv/packager.py @@ -179,12 +179,7 @@ def install_requirements( target_dir = os.path.abspath(target_dir) args.extend(["--target", target_dir]) - target_python = None - if config.compile_bytecode: - if python_version: - target_python = self._uv.find_python(python_version) - else: - LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + target_python = self._find_target_python(python_version) if config.compile_bytecode else None # Add configuration arguments that are independent of interpreter selection. args.extend(config.to_uv_args()) @@ -238,7 +233,20 @@ def install_requirements( if target_python: self._compile_bytecode(target_python, target_dir) - def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: + def compile_bytecode(self, python_version: Optional[str], paths: List[str]) -> None: + """Compile unchecked-hash bytecode for the given files and directories with the target Python.""" + target_python = self._find_target_python(python_version) + if target_python: + self._compile_bytecode(target_python, *paths) + + def _find_target_python(self, python_version: Optional[str]) -> Optional[str]: + """Find the interpreter that compiles bytecode for the target runtime, or None if there isn't one.""" + if not python_version: + LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + return None + return self._uv.find_python(python_version) + + def _compile_bytecode(self, python_executable: str, *paths: str) -> None: """Compile reusable bytecode with the interpreter selected for the target runtime.""" command = [ python_executable, @@ -248,7 +256,7 @@ def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: "-q", "--invalidation-mode", "unchecked-hash", - target_dir, + *paths, ] LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command)) rc, stdout, stderr = self._osutils.run_subprocess(command) diff --git a/aws_lambda_builders/workflows/python_uv/workflow.py b/aws_lambda_builders/workflows/python_uv/workflow.py index 6181febae..837d865ab 100644 --- a/aws_lambda_builders/workflows/python_uv/workflow.py +++ b/aws_lambda_builders/workflows/python_uv/workflow.py @@ -9,7 +9,7 @@ from aws_lambda_builders.exceptions import WorkflowFailedError from aws_lambda_builders.workflow import BaseWorkflow, BuildDirectory, BuildInSourceSupport, Capability -from .actions import PythonUvBuildAction +from .actions import PythonUvBuildAction, PythonUvCompileSourceAction from .utils import OSUtils, UvConfig, detect_uv_manifest, normalize_package_name LOG = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def _compile_bytecode_from_env(): if value is None or value.lower() in UV_ENV_FALSE: return False if value.lower() in UV_ENV_TRUE: - LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode for dependencies", value) + LOG.info("UV_COMPILE_BYTECODE=%s is set, compiling unchecked-hash bytecode", value) return True LOG.warning("Ignoring UV_COMPILE_BYTECODE=%r: expected one of %s", value, ", ".join(UV_ENV_TRUE + UV_ENV_FALSE)) return False @@ -256,9 +256,15 @@ def _setup_build_actions(self, source_dir, artifacts_dir, scratch_dir, manifest_ ) ) - # Always copy source code (final step) + # Always copy source code self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + # The dependency build compiles what it installs; the source is only in place now + if config and config.compile_bytecode: + self.actions.append( + PythonUvCompileSourceAction(source_dir, artifacts_dir, runtime, excludes=self.EXCLUDED_FILES) + ) + def get_resolvers(self): """ Get path resolvers for the workflow. diff --git a/tests/integration/workflows/python_uv/test_python_uv.py b/tests/integration/workflows/python_uv/test_python_uv.py index df1960b00..0d87c5acd 100644 --- a/tests/integration/workflows/python_uv/test_python_uv.py +++ b/tests/integration/workflows/python_uv/test_python_uv.py @@ -734,6 +734,9 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self): os.environ.pop("UV_COMPILE_BYTECODE", None) requirements = os.path.join(source_dir, "requirements.txt") pathlib.Path(requirements).write_text("six==1.16.0\n") + pathlib.Path(source_dir, "app.py").write_text("import six\n") + pathlib.Path(source_dir, "pkg").mkdir() + pathlib.Path(source_dir, "pkg", "mod.py").write_text("VALUE = 1\n") builder = LambdaBuilder(language="python", dependency_manager="uv", application_framework=None) runtime = f"python{sys.version_info.major}.{sys.version_info.minor}" @@ -757,10 +760,12 @@ def test_workflow_compiles_unchecked_hash_bytecode_when_enabled(self): experimental_flags=self.experimental_flags, options={"uv_compile_bytecode": True}, ) - pyc_files = list(pathlib.Path(compiled_dir, "__pycache__").glob("six.*.pyc")) - self.assertEqual(len(pyc_files), 1) - # PEP 552 flags word: 1 = hash-based, unchecked - self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1) + # The dependency, and the function's own source (top level and in a package) + for pycache, module in (("__pycache__", "six"), ("__pycache__", "app"), ("pkg/__pycache__", "mod")): + pyc_files = list(pathlib.Path(compiled_dir, pycache).glob(f"{module}.*.pyc")) + self.assertEqual(len(pyc_files), 1, module) + # PEP 552 flags word: 1 = hash-based, unchecked + self.assertEqual(int.from_bytes(pyc_files[0].read_bytes()[4:8], "little"), 1, module) @skipIf(which("uv") is None, "uv not available") def test_workflow_shows_uv_warnings(self): diff --git a/tests/unit/workflows/python_uv/test_actions.py b/tests/unit/workflows/python_uv/test_actions.py new file mode 100644 index 000000000..e47355c73 --- /dev/null +++ b/tests/unit/workflows/python_uv/test_actions.py @@ -0,0 +1,75 @@ +import os +import shutil +import tempfile +from pathlib import Path +from unittest import TestCase +from unittest.mock import patch + +from aws_lambda_builders.workflows.python_uv.actions import PythonUvCompileSourceAction +from aws_lambda_builders.workflows.python_uv.exceptions import MissingUvError + + +class TestPythonUvCompileSourceAction(TestCase): + def setUp(self): + self.source_dir = tempfile.mkdtemp() + self.artifacts_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.source_dir) + self.addCleanup(shutil.rmtree, self.artifacts_dir) + + def _execute(self, runtime="python3.13", excludes=("venv", "*.egg-info")): + with ( + patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv"), + patch("aws_lambda_builders.workflows.python_uv.actions.UvRunner") as runner_class, + ): + PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, runtime, excludes).execute() + return runner_class.return_value.compile_bytecode + + def test_compiles_only_what_was_copied_from_the_source(self): + for root in (self.source_dir, self.artifacts_dir): + Path(root, "app.py").touch() + Path(root, "pkg").mkdir() + Path(root, "requirements.txt").touch() + # Excluded from the copy, so not part of the function's source, even where the + # artifacts directory has an entry of the same name + for root in (self.source_dir, self.artifacts_dir): + Path(root, "venv").mkdir() + Path(root, "app.egg-info").mkdir() + # Installed dependencies, already compiled by the dependency build + Path(self.artifacts_dir, "six.py").touch() + Path(self.artifacts_dir, "requests").mkdir() + + compile_bytecode = self._execute() + + compile_bytecode.assert_called_once_with( + "3.13", [os.path.join(self.artifacts_dir, "app.py"), os.path.join(self.artifacts_dir, "pkg")] + ) + + def test_does_nothing_without_python_source(self): + Path(self.source_dir, "requirements.txt").touch() + Path(self.artifacts_dir, "requirements.txt").touch() + + self._execute().assert_not_called() + + def test_passes_no_python_version_without_a_runtime(self): + Path(self.source_dir, "app.py").touch() + Path(self.artifacts_dir, "app.py").touch() + + self._execute(runtime=None).assert_called_once_with(None, [os.path.join(self.artifacts_dir, "app.py")]) + + def test_does_nothing_without_a_source_directory(self): + shutil.rmtree(self.source_dir) + self.addCleanup(os.mkdir, self.source_dir) + + self._execute().assert_not_called() + + def test_warns_instead_of_failing_when_uv_is_missing(self): + Path(self.source_dir, "app.py").touch() + Path(self.artifacts_dir, "app.py").touch() + + with ( + patch("aws_lambda_builders.workflows.python_uv.actions.SubprocessUv", side_effect=MissingUvError()), + self.assertLogs("aws_lambda_builders.workflows.python_uv.actions", level="WARNING") as logs, + ): + PythonUvCompileSourceAction(self.source_dir, self.artifacts_dir, "python3.13").execute() + + self.assertIn("Skipping bytecode compilation of the source: uv executable not found", logs.output[0]) diff --git a/tests/unit/workflows/python_uv/test_packager.py b/tests/unit/workflows/python_uv/test_packager.py index 518acd440..3037e1745 100644 --- a/tests/unit/workflows/python_uv/test_packager.py +++ b/tests/unit/workflows/python_uv/test_packager.py @@ -366,6 +366,35 @@ def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fa self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0]) self.assertIn("compile error", logs.output[0]) + def test_compile_bytecode_compiles_the_given_paths_with_the_target_python(self): + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (0, "", "") + + self.uv_runner.compile_bytecode("3.13", ["/artifacts/app.py", "/artifacts/pkg"]) + + self.mock_subprocess_uv.find_python.assert_called_once_with("3.13") + self.mock_osutils.run_subprocess.assert_called_once_with( + [ + "/usr/bin/python3.13", + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + "/artifacts/app.py", + "/artifacts/pkg", + ] + ) + + @parameterized.expand([("no_target_python", "3.13"), ("no_python_version", None)]) + def test_compile_bytecode_skips_when_the_target_python_is_unknown(self, _, python_version): + self.mock_subprocess_uv.find_python.return_value = None + + self.uv_runner.compile_bytecode(python_version, ["/artifacts/app.py"]) + + self.mock_osutils.run_subprocess.assert_not_called() + def test_install_requirements_resolves_relative_target_to_absolute(self): # UV runs from project_dir or workspace_dir, # so a relative --target must be resolved to an absolute path first, diff --git a/tests/unit/workflows/python_uv/test_workflow.py b/tests/unit/workflows/python_uv/test_workflow.py index 3e672155b..06068a133 100644 --- a/tests/unit/workflows/python_uv/test_workflow.py +++ b/tests/unit/workflows/python_uv/test_workflow.py @@ -8,7 +8,7 @@ from aws_lambda_builders.actions import CleanUpAction, CopyDependenciesAction, CopySourceAction from aws_lambda_builders.exceptions import WorkflowFailedError -from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction +from aws_lambda_builders.workflows.python_uv.actions import PythonUvBuildAction, PythonUvCompileSourceAction from aws_lambda_builders.workflows.python_uv.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE, OSUtils from aws_lambda_builders.workflows.python_uv.workflow import PythonUvWorkflow @@ -251,6 +251,20 @@ def test_compile_bytecode_option_is_passed_to_the_build_action(self): config = self._workflow({"uv_compile_bytecode": True}).actions[0].config self.assertTrue(config.compile_bytecode) + def test_compile_bytecode_compiles_the_source_after_copying_it(self): + workflow = self._workflow({"uv_compile_bytecode": True}) + self.assertIsInstance(workflow.actions[-2], CopySourceAction) + compile_action = workflow.actions[-1] + self.assertIsInstance(compile_action, PythonUvCompileSourceAction) + self.assertEqual(compile_action.source_dir, self.source_dir) + self.assertEqual(compile_action.artifacts_dir, "artifacts") + self.assertEqual(compile_action.runtime, "python3.12") + self.assertEqual(compile_action.excludes, PythonUvWorkflow.EXCLUDED_FILES) + + def test_source_is_not_compiled_by_default(self): + workflow = self._workflow({"build_logical_id": "Fn"}) + self.assertIsInstance(workflow.actions[-1], CopySourceAction) + def test_compile_bytecode_on_requirements_build_is_allowed(self): requirements = os.path.join(self.source_dir, "requirements.txt") open(requirements, "w").close() @@ -260,15 +274,13 @@ def test_compile_bytecode_on_requirements_build_is_allowed(self): def test_compile_bytecode_without_manifest_builds_without_dependencies(self): os.remove(self.pyproject) workflow = self._workflow({"uv_compile_bytecode": True}, manifest=os.path.join(self.source_dir, "missing.txt")) - self.assertEqual(len(workflow.actions), 1) - self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction]) def test_truthy_env_without_manifest_builds_without_dependencies(self): os.environ["UV_COMPILE_BYTECODE"] = "1" os.remove(self.pyproject) workflow = self._workflow({"build_logical_id": "Fn"}, manifest=os.path.join(self.source_dir, "missing.txt")) - self.assertEqual(len(workflow.actions), 1) - self.assertIsInstance(workflow.actions[0], CopySourceAction) + self.assertEqual([type(action) for action in workflow.actions], [CopySourceAction, PythonUvCompileSourceAction]) def test_export_option_with_compile_bytecode_on_requirements_build_fails(self): requirements = os.path.join(self.source_dir, "requirements.txt")