From 92f01889ddc01f78adf37982c23e31d5116e4211 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:09:24 -0700 Subject: [PATCH 001/120] feat: add app-centric sdk surface with queue/api/task decorators --- runpod/__init__.py | 28 +++ runpod/apps/__init__.py | 54 ++++++ runpod/apps/app.py | 252 ++++++++++++++++++++++++++ runpod/apps/context.py | 76 ++++++++ runpod/apps/entrypoint.py | 43 +++++ runpod/apps/errors.py | 35 ++++ runpod/apps/gpu.py | 148 +++++++++++++++ runpod/apps/handles.py | 228 +++++++++++++++++++++++ runpod/apps/invoker.py | 27 +++ runpod/apps/markers.py | 89 +++++++++ runpod/apps/schedule.py | 41 +++++ runpod/apps/spec.py | 161 +++++++++++++++++ runpod/apps/stubs.py | 101 +++++++++++ runpod/apps/targets.py | 270 ++++++++++++++++++++++++++++ tests/test_apps/__init__.py | 0 tests/test_apps/test_app.py | 262 +++++++++++++++++++++++++++ tests/test_apps/test_compat_lock.py | 105 +++++++++++ tests/test_apps/test_dispatch.py | 241 +++++++++++++++++++++++++ tests/test_init.py | 4 + 19 files changed, 2165 insertions(+) create mode 100644 runpod/apps/__init__.py create mode 100644 runpod/apps/app.py create mode 100644 runpod/apps/context.py create mode 100644 runpod/apps/entrypoint.py create mode 100644 runpod/apps/errors.py create mode 100644 runpod/apps/gpu.py create mode 100644 runpod/apps/handles.py create mode 100644 runpod/apps/invoker.py create mode 100644 runpod/apps/markers.py create mode 100644 runpod/apps/schedule.py create mode 100644 runpod/apps/spec.py create mode 100644 runpod/apps/stubs.py create mode 100644 runpod/apps/targets.py create mode 100644 tests/test_apps/__init__.py create mode 100644 tests/test_apps/test_app.py create mode 100644 tests/test_apps/test_compat_lock.py create mode 100644 tests/test_apps/test_dispatch.py diff --git a/runpod/__init__.py b/runpod/__init__.py index 6d24180ae..8f4d71d53 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -29,6 +29,17 @@ set_credentials, ) from .endpoint import AsyncioEndpoint, AsyncioJob, Endpoint +from .apps import ( + Api, + App, + EndpointNotFound, + Queue, + is_local, + local_entrypoint, + schedule, +) +from .apps.gpu import CpuInstanceType, GpuGroup, GpuType +from .apps.markers import delete, get, init, patch, post, put from .serverless.modules.rp_logger import RunPodLogger from .version import __version__ @@ -59,6 +70,23 @@ "AsyncioEndpoint", "AsyncioJob", "Endpoint", + # Apps surface + "Api", + "App", + "CpuInstanceType", + "EndpointNotFound", + "GpuGroup", + "GpuType", + "Queue", + "delete", + "get", + "init", + "is_local", + "local_entrypoint", + "patch", + "post", + "put", + "schedule", # Serverless module "serverless", # Logger class diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py new file mode 100644 index 000000000..acb63ce48 --- /dev/null +++ b/runpod/apps/__init__.py @@ -0,0 +1,54 @@ +"""app-centric sdk surface: App, decorated handles, and execution primitives. + +the public entry points here are: + +- App: the registry a project defines its resources on +- @app.queue: queue-based serverless endpoint from a function +- @app.api: load-balanced serverless endpoint from a class (or asgi factory) +- @app.task: ephemeral pod compute from a function +- runpod.local_entrypoint: dev-session entry hook +- runpod.is_local: context check +- runpod.Queue / runpod.Api: stubs for resources deployed from elsewhere +""" + +from .app import App, get_registered_apps +from .context import Context, current_context, is_local +from .entrypoint import local_entrypoint +from .errors import ( + AppError, + EndpointNotFound, + RemoteExecutionError, + ScheduleNotSupported, +) +from .handles import ApiHandle, FunctionHandle +from .markers import delete, get, init, patch, post, put +from .schedule import schedule +from .spec import ResourceKind, ResourceSpec, RouteSpec +from .stubs import Api, Queue + +__all__ = [ + "Api", + "ApiHandle", + "App", + "AppError", + "Context", + "EndpointNotFound", + "FunctionHandle", + "Queue", + "RemoteExecutionError", + "ResourceKind", + "ResourceSpec", + "RouteSpec", + "ScheduleNotSupported", + "current_context", + "delete", + "get", + "get_registered_apps", + "init", + "is_local", + "local_entrypoint", + "patch", + "post", + "put", + "schedule", +] diff --git a/runpod/apps/app.py b/runpod/apps/app.py new file mode 100644 index 000000000..b95f95b36 --- /dev/null +++ b/runpod/apps/app.py @@ -0,0 +1,252 @@ +"""the App registry: where a project declares its resources. + + import runpod + from runpod import App, GpuType + + app = App("my-app") + + @app.queue(name="transcribe", gpu=GpuType.NVIDIA_GEFORCE_RTX_4090) + async def transcribe(audio_url: str): ... + +`rp deploy` imports project modules, collects App instances from the +module-level registry, and ships each app's resources. resolution of +deployed resources is fully server-side (sentinel headers), so apps hold +no persistent local state. +""" + +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from .context import Context, current_context +from .errors import EndpointNotFound, InvalidResourceError +from .gpu import GpuLike +from .handles import ApiHandle, FunctionHandle +from .spec import ( + ResourceKind, + ResourceSpec, + normalize_cpu, + normalize_gpu, + normalize_workers, +) +from .targets import InvocationTarget, PodTarget, SentinelTarget + +DEFAULT_ENV = "default" + +# module-level registry populated as user modules import; `rp deploy` and +# `rp dev` read it after importing the target modules +_REGISTRY: List["App"] = [] + + +def get_registered_apps() -> List["App"]: + """all App instances created in this process.""" + return list(_REGISTRY) + + +def _clear_registry() -> None: + """testing only.""" + _REGISTRY.clear() + + +class App: + """a named collection of deployable resources.""" + + def __init__(self, name: str, *, env: str = DEFAULT_ENV): + if not name or not isinstance(name, str): + raise InvalidResourceError("App name must be a non-empty string") + self.name = name + self.env = env + self._resources: Dict[str, Union[FunctionHandle, ApiHandle]] = {} + # populated by an `rp dev` session: resource name -> live target + self._dev_targets: Dict[str, InvocationTarget] = {} + _REGISTRY.append(self) + + @property + def resources(self) -> Dict[str, Union[FunctionHandle, ApiHandle]]: + return dict(self._resources) + + def _register( + self, name: str, handle: Union[FunctionHandle, ApiHandle] + ) -> None: + if name in self._resources: + raise InvalidResourceError( + f"duplicate resource name '{name}' in app '{self.name}'" + ) + self._resources[name] = handle + + # -- decorators -- + + def queue( + self, + *, + name: Optional[str] = None, + gpu: Optional[Union[GpuLike, List[GpuLike]]] = None, + cpu: Optional[Union[str, List[str]]] = None, + gpu_count: int = 1, + workers: Union[int, Tuple[int, int], None] = None, + idle_timeout: int = 60, + dependencies: Optional[List[str]] = None, + system_dependencies: Optional[List[str]] = None, + volume: Optional[Any] = None, + env: Optional[Dict[str, str]] = None, + ) -> Callable[[Callable], FunctionHandle]: + """declare a queue-based serverless endpoint from a function.""" + + def decorator(fn: Callable) -> FunctionHandle: + spec = ResourceSpec( + kind=ResourceKind.QUEUE, + name=name or fn.__name__, + gpu=normalize_gpu(gpu), + cpu=normalize_cpu(cpu), + gpu_count=gpu_count, + workers=normalize_workers(workers), + idle_timeout=idle_timeout, + dependencies=dependencies, + system_dependencies=system_dependencies, + volume=_volume_ref(volume), + env=env, + ) + handle = FunctionHandle(self, fn, spec) + self._register(spec.name, handle) + return handle + + return decorator + + def task( + self, + *, + name: Optional[str] = None, + gpu: Optional[Union[GpuLike, List[GpuLike]]] = None, + cpu: Optional[Union[str, List[str]]] = None, + gpu_count: int = 1, + dependencies: Optional[List[str]] = None, + system_dependencies: Optional[List[str]] = None, + volume: Optional[Any] = None, + env: Optional[Dict[str, str]] = None, + image: Optional[str] = None, + ) -> Callable[[Callable], FunctionHandle]: + """declare ephemeral pod compute from a function. + + tasks have no standing infrastructure: `.remote()` provisions a + pod, runs the body, returns the result, and terminates the pod. + they never require `rp deploy` (except to register a schedule). + """ + + def decorator(fn: Callable) -> FunctionHandle: + spec = ResourceSpec( + kind=ResourceKind.TASK, + name=name or fn.__name__, + gpu=normalize_gpu(gpu), + cpu=normalize_cpu(cpu), + gpu_count=gpu_count, + dependencies=dependencies, + system_dependencies=system_dependencies, + volume=_volume_ref(volume), + env=env, + image=image, + ) + handle = FunctionHandle(self, fn, spec) + self._register(spec.name, handle) + return handle + + return decorator + + def api( + self, + *, + name: Optional[str] = None, + gpu: Optional[Union[GpuLike, List[GpuLike]]] = None, + cpu: Optional[Union[str, List[str]]] = None, + gpu_count: int = 1, + workers: Union[int, Tuple[int, int], None] = None, + idle_timeout: int = 60, + dependencies: Optional[List[str]] = None, + system_dependencies: Optional[List[str]] = None, + volume: Optional[Any] = None, + env: Optional[Dict[str, str]] = None, + ) -> Callable[[Any], ApiHandle]: + """declare a load-balanced serverless endpoint. + + decorate a class with route markers for the native experience, or + a zero-argument function returning an asgi app to serve an + existing fastapi/starlette application: + + @app.api(name="inference", gpu=GpuType.NVIDIA_L4) + class Inference: + @init + def setup(self): self.model = load() + + @post("/generate") + async def generate(self, body: dict): ... + + @app.api(name="fast", cpu="cpu5c-2-4") + def web(): + from fastapi import FastAPI + server = FastAPI() + ... + return server + """ + + def decorator(target: Any) -> ApiHandle: + spec = ResourceSpec( + kind=ResourceKind.API, + name=name or target.__name__, + gpu=normalize_gpu(gpu), + cpu=normalize_cpu(cpu), + gpu_count=gpu_count, + workers=normalize_workers(workers), + idle_timeout=idle_timeout, + dependencies=dependencies, + system_dependencies=system_dependencies, + volume=_volume_ref(volume), + env=env, + ) + handle = ApiHandle(self, target, spec) + self._register(spec.name, handle) + return handle + + return decorator + + # -- resolution -- + + async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: + """resolve a resource spec to an invocation target. + + stateless by design: deployed resources resolve through the + sentinel (server-side name resolution); dev sessions register + live targets in memory; tasks provision per call. + """ + if spec.kind is ResourceKind.TASK: + handle = self._resources.get(spec.name) + fn = getattr(handle, "_fn", None) + return PodTarget(spec, fn) + + ctx = current_context() + + if ctx is Context.DEV: + target = self._dev_targets.get(spec.name) + if target is not None: + return target + # a dev session should have provisioned every endpoint at + # startup; a miss means the resource was added mid-session + raise EndpointNotFound(self.name, spec.name) + + env_name = os.getenv("FLASH_ENVIRONMENT") or self.env + return SentinelTarget(self.name, env_name, spec.name) + + def __repr__(self) -> str: + return f"" + + +def _volume_ref(volume: Any) -> Optional[str]: + """normalize a volume argument to a name-or-id string reference.""" + if volume is None: + return None + if isinstance(volume, str): + return volume + name = getattr(volume, "name", None) or getattr(volume, "id", None) + if name is None: + raise InvalidResourceError( + f"volume must be a string name/id or an object with .name/.id, " + f"got {type(volume).__name__}" + ) + return str(name) diff --git a/runpod/apps/context.py b/runpod/apps/context.py new file mode 100644 index 000000000..4aeb2919c --- /dev/null +++ b/runpod/apps/context.py @@ -0,0 +1,76 @@ +"""execution context detection and the sync/async bridge.""" + +import asyncio +import os +import threading +from enum import Enum +from typing import Any, Coroutine + + +class Context(Enum): + """where the current process is running.""" + + LOCAL = "local" + """dev machine, plain `python main.py`.""" + + DEV = "dev" + """inside an `rp dev` session (ephemeral live provisioning).""" + + WORKER = "worker" + """inside a runpod serverless endpoint or pod.""" + + +def current_context() -> Context: + """detect the execution context from the environment.""" + if os.getenv("RUNPOD_ENDPOINT_ID") or os.getenv("RUNPOD_POD_ID"): + return Context.WORKER + if os.getenv("RUNPOD_DEV_SESSION"): + return Context.DEV + return Context.LOCAL + + +def is_local() -> bool: + """true when not running inside a runpod container. + + usable as a module-level guard so code only runs on the dev machine: + + if runpod.is_local(): + print("running or imported locally") + """ + return current_context() is not Context.WORKER + + +class _LoopThread: + """a dedicated background event loop for driving async engine code + from synchronous callers. + + this makes `.remote()` safe to call both from plain sync code and from + inside an already-running event loop (where `asyncio.run` would raise). + """ + + _loop: "asyncio.AbstractEventLoop | None" = None + _lock = threading.Lock() + + @classmethod + def _ensure_loop(cls) -> asyncio.AbstractEventLoop: + with cls._lock: + if cls._loop is None or cls._loop.is_closed(): + loop = asyncio.new_event_loop() + thread = threading.Thread( + target=loop.run_forever, + name="runpod-apps-loop", + daemon=True, + ) + thread.start() + cls._loop = loop + return cls._loop + + @classmethod + def run(cls, coro: Coroutine[Any, Any, Any]) -> Any: + loop = cls._ensure_loop() + return asyncio.run_coroutine_threadsafe(coro, loop).result() + + +def block(coro: Coroutine[Any, Any, Any]) -> Any: + """run a coroutine to completion from a synchronous caller.""" + return _LoopThread.run(coro) diff --git a/runpod/apps/entrypoint.py b/runpod/apps/entrypoint.py new file mode 100644 index 000000000..e920308c3 --- /dev/null +++ b/runpod/apps/entrypoint.py @@ -0,0 +1,43 @@ +"""@runpod.local_entrypoint: the dev-session entry hook. + + @runpod.local_entrypoint + async def main(): + result = transcribe.remote("https://...") + +the decorated function is registered so `rp dev ` can find and +run it (sync or async). importing the module never executes it. +""" + +import inspect +from typing import Any, Callable, List, Optional + +_ENTRYPOINTS: List[Callable] = [] + + +def local_entrypoint(fn: Callable) -> Callable: + """register a function as the dev-session entrypoint. + + the function is returned unwrapped so it stays directly callable. + """ + _ENTRYPOINTS.append(fn) + return fn + + +def get_entrypoint() -> Optional[Callable]: + """the most recently registered entrypoint, if any.""" + return _ENTRYPOINTS[-1] if _ENTRYPOINTS else None + + +def run_entrypoint(fn: Callable) -> Any: + """execute an entrypoint, driving the loop for async functions.""" + from .context import block + + result = fn() + if inspect.isawaitable(result): + return block(result) + return result + + +def _clear_entrypoints() -> None: + """testing only.""" + _ENTRYPOINTS.clear() diff --git a/runpod/apps/errors.py b/runpod/apps/errors.py new file mode 100644 index 000000000..acd3dd0a1 --- /dev/null +++ b/runpod/apps/errors.py @@ -0,0 +1,35 @@ +"""errors raised by the apps surface.""" + + +class AppError(Exception): + """base error for the apps sdk.""" + + +class EndpointNotFound(AppError): + """a resource was invoked but is not deployed.""" + + def __init__(self, app_name: str, resource_name: str): + self.app_name = app_name + self.resource_name = resource_name + super().__init__( + f"'{resource_name}' in app '{app_name}' is not deployed. " + f"run `rp deploy` first, or use .local() to run it here." + ) + + +class RemoteExecutionError(AppError): + """the remote worker reported a failure executing the function.""" + + +class ScheduleNotSupported(AppError): + """@schedule requires backend support that is not yet available.""" + + def __init__(self) -> None: + super().__init__( + "recurring schedules are not yet supported by the runpod api. " + "the @schedule decorator records intent but cannot be deployed." + ) + + +class InvalidResourceError(AppError): + """a decorator was applied to an unsupported target or with bad config.""" diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py new file mode 100644 index 000000000..73e026573 --- /dev/null +++ b/runpod/apps/gpu.py @@ -0,0 +1,148 @@ +"""gpu and cpu selection enums. + +GpuGroup values are backend pool ids; GpuType values are exact device +display names. either (or plain strings) are accepted anywhere a gpu is +configured. CpuInstanceType values are backend cpu flavor ids. +""" + +from enum import Enum +from typing import List, Optional, Union + + +class GpuGroup(Enum): + ANY = "any" + """any gpu""" + + ADA_24 = "ADA_24" + """NVIDIA GeForce RTX 4090""" + + ADA_32_PRO = "ADA_32_PRO" + """NVIDIA GeForce RTX 5090""" + + ADA_48_PRO = "ADA_48_PRO" + """NVIDIA RTX 6000 Ada Generation, NVIDIA L40, NVIDIA L40S""" + + ADA_80_PRO = "ADA_80_PRO" + """NVIDIA H100 PCIe, NVIDIA H100 80GB HBM3, NVIDIA H100 NVL""" + + AMPERE_16 = "AMPERE_16" + """NVIDIA RTX A4000, NVIDIA RTX A4500, NVIDIA RTX 4000 Ada Generation, NVIDIA RTX 2000 Ada Generation""" + + AMPERE_24 = "AMPERE_24" + """NVIDIA RTX A5000, NVIDIA L4, NVIDIA GeForce RTX 3090""" + + AMPERE_48 = "AMPERE_48" + """NVIDIA A40, NVIDIA RTX A6000""" + + AMPERE_80 = "AMPERE_80" + """NVIDIA A100 80GB PCIe, NVIDIA A100-SXM4-80GB""" + + HOPPER_141 = "HOPPER_141" + """NVIDIA H200""" + + BLACKWELL_96 = "BLACKWELL_96" + """NVIDIA RTX PRO 6000 Blackwell Server/Workstation/Max-Q editions""" + + BLACKWELL_180 = "BLACKWELL_180" + """NVIDIA B200""" + + @classmethod + def all(cls) -> List["GpuGroup"]: + """all gpu groups except ANY.""" + return [g for g in cls if g is not cls.ANY] + + +class GpuType(Enum): + ANY = "any" + """any gpu""" + + NVIDIA_GEFORCE_RTX_4090 = "NVIDIA GeForce RTX 4090" + NVIDIA_GEFORCE_RTX_5090 = "NVIDIA GeForce RTX 5090" + NVIDIA_RTX_6000_ADA_GENERATION = "NVIDIA RTX 6000 Ada Generation" + NVIDIA_RTX_PRO_6000_BLACKWELL_SERVER_EDITION = ( + "NVIDIA RTX PRO 6000 Blackwell Server Edition" + ) + NVIDIA_RTX_PRO_6000_BLACKWELL_WORKSTATION_EDITION = ( + "NVIDIA RTX PRO 6000 Blackwell Workstation Edition" + ) + NVIDIA_RTX_PRO_6000_BLACKWELL_MAX_Q_WORKSTATION_EDITION = ( + "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition" + ) + NVIDIA_H100_80GB_HBM3 = "NVIDIA H100 80GB HBM3" + NVIDIA_RTX_A4000 = "NVIDIA RTX A4000" + NVIDIA_RTX_A4500 = "NVIDIA RTX A4500" + NVIDIA_RTX_4000_ADA_GENERATION = "NVIDIA RTX 4000 Ada Generation" + NVIDIA_RTX_2000_ADA_GENERATION = "NVIDIA RTX 2000 Ada Generation" + NVIDIA_RTX_A5000 = "NVIDIA RTX A5000" + NVIDIA_L4 = "NVIDIA L4" + NVIDIA_GEFORCE_RTX_3090 = "NVIDIA GeForce RTX 3090" + NVIDIA_A40 = "NVIDIA A40" + NVIDIA_RTX_A6000 = "NVIDIA RTX A6000" + NVIDIA_A100_80GB_PCIe = "NVIDIA A100 80GB PCIe" + NVIDIA_A100_SXM4_80GB = "NVIDIA A100-SXM4-80GB" + NVIDIA_H200 = "NVIDIA H200" + NVIDIA_B200 = "NVIDIA B200" + + @classmethod + def all(cls) -> List["GpuType"]: + """all gpu types except ANY.""" + return [g for g in cls if g is not cls.ANY] + + +POOLS_TO_TYPES = { + GpuGroup.ADA_24: [GpuType.NVIDIA_GEFORCE_RTX_4090], + GpuGroup.ADA_32_PRO: [GpuType.NVIDIA_GEFORCE_RTX_5090], + GpuGroup.ADA_48_PRO: [GpuType.NVIDIA_RTX_6000_ADA_GENERATION], + GpuGroup.ADA_80_PRO: [GpuType.NVIDIA_H100_80GB_HBM3], + GpuGroup.AMPERE_16: [ + GpuType.NVIDIA_RTX_A4000, + GpuType.NVIDIA_RTX_A4500, + GpuType.NVIDIA_RTX_4000_ADA_GENERATION, + GpuType.NVIDIA_RTX_2000_ADA_GENERATION, + ], + GpuGroup.AMPERE_24: [ + GpuType.NVIDIA_RTX_A5000, + GpuType.NVIDIA_L4, + GpuType.NVIDIA_GEFORCE_RTX_3090, + ], + GpuGroup.AMPERE_48: [GpuType.NVIDIA_A40, GpuType.NVIDIA_RTX_A6000], + GpuGroup.AMPERE_80: [ + GpuType.NVIDIA_A100_80GB_PCIe, + GpuType.NVIDIA_A100_SXM4_80GB, + ], + GpuGroup.HOPPER_141: [GpuType.NVIDIA_H200], + GpuGroup.BLACKWELL_96: [ + GpuType.NVIDIA_RTX_PRO_6000_BLACKWELL_SERVER_EDITION, + GpuType.NVIDIA_RTX_PRO_6000_BLACKWELL_WORKSTATION_EDITION, + GpuType.NVIDIA_RTX_PRO_6000_BLACKWELL_MAX_Q_WORKSTATION_EDITION, + ], + GpuGroup.BLACKWELL_180: [GpuType.NVIDIA_B200], +} + + +def pool_for_gpu_type(gpu_type: GpuType) -> Optional[GpuGroup]: + """the pool a specific gpu type belongs to, if any.""" + for group, types in POOLS_TO_TYPES.items(): + if gpu_type in types: + return group + return None + + +class CpuInstanceType(str, Enum): + """cpu instance flavors: {generation}{type}-{vcpu}-{memory_gb}.""" + + CPU3G_1_4 = "cpu3g-1-4" + CPU3G_2_8 = "cpu3g-2-8" + CPU3G_4_16 = "cpu3g-4-16" + CPU3G_8_32 = "cpu3g-8-32" + CPU3C_1_2 = "cpu3c-1-2" + CPU3C_2_4 = "cpu3c-2-4" + CPU3C_4_8 = "cpu3c-4-8" + CPU3C_8_16 = "cpu3c-8-16" + CPU5C_1_2 = "cpu5c-1-2" + CPU5C_2_4 = "cpu5c-2-4" + CPU5C_4_8 = "cpu5c-4-8" + CPU5C_8_16 = "cpu5c-8-16" + + +GpuLike = Union[GpuGroup, GpuType, str] diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py new file mode 100644 index 000000000..d93144bda --- /dev/null +++ b/runpod/apps/handles.py @@ -0,0 +1,228 @@ +"""handles returned by app decorators. + +a handle replaces the decorated object and is the single way to interact +with the resource: + + @app.queue(name="transcribe", gpu=GpuType.NVIDIA_GEFORCE_RTX_4090) + async def transcribe(audio_url: str): ... + + transcribe.remote("https://...") # sync remote call + await transcribe.remote.aio("https://...") + transcribe.spawn("https://...") # fire and forget -> job handle + transcribe.local("https://...") # run the body here + + @transcribe.init + def load_model(): ... # worker startup hook +""" + +import inspect +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type + +from .context import Context, current_context +from .invoker import Invoker +from .markers import INIT_ATTR, RESERVED_PATHS, is_init, route_of +from .schedule import SCHEDULE_ATTR +from .spec import ResourceKind, ResourceSpec, RouteSpec +from .errors import InvalidResourceError, RemoteExecutionError +from .targets import args_to_input + +if TYPE_CHECKING: + from .app import App + + +class Job: + """a submitted queue job, returned by .spawn().""" + + def __init__(self, data: Dict[str, Any], handle: "FunctionHandle"): + self._data = data + self._handle = handle + + @property + def id(self) -> str: + return self._data.get("id", "") + + @property + def status(self) -> str: + return self._data.get("status", "UNKNOWN") + + def __repr__(self) -> str: + return f"Job(id={self.id!r}, status={self.status!r})" + + +class FunctionHandle: + """handle for @app.queue and @app.task functions.""" + + def __init__( + self, + app: "App", + fn: Callable, + spec: ResourceSpec, + ): + self._app = app + self._fn = fn + self.spec = spec + self._init_fn: Optional[Callable] = None + + # adopt a schedule stamped by @schedule below the app decorator + stamped = getattr(fn, SCHEDULE_ATTR, None) + if stamped and not spec.schedule: + spec.schedule = stamped + + self.remote = Invoker(self._remote_async) + self.spawn = Invoker(self._spawn_async) + + self.__name__ = getattr(fn, "__name__", spec.name) + self.__doc__ = getattr(fn, "__doc__", None) + self.__wrapped__ = fn + + # -- lifecycle -- + + def init(self, fn: Callable) -> Callable: + """register a worker-startup hook; runs before the worker is ready, + never locally.""" + setattr(fn, INIT_ATTR, True) + self._init_fn = fn + return fn + + # -- local execution -- + + def local(self, *args: Any, **kwargs: Any) -> Any: + """run the function body here. returns a coroutine iff the + function is async.""" + return self._fn(*args, **kwargs) + + # -- remote execution -- + + async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: + ctx = current_context() + + if ctx is Context.WORKER and self._is_current_worker(): + result = self._fn(*args, **kwargs) + if inspect.isawaitable(result): + result = await result + return result + + target = await self._app._resolve(self.spec) + payload = {"input": args_to_input(self._fn, args, kwargs)} + return await target.invoke(payload) + + async def _spawn_async(self, *args: Any, **kwargs: Any) -> Job: + target = await self._app._resolve(self.spec) + payload = {"input": args_to_input(self._fn, args, kwargs)} + data = await target.submit(payload) + return Job(data, self) + + def _is_current_worker(self) -> bool: + import os + + current = os.getenv("FLASH_RESOURCE_NAME") or os.getenv( + "RUNPOD_RESOURCE_NAME" + ) + return current is not None and current == self.spec.name + + # calling the handle directly is a common mistake; be explicit + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + raise TypeError( + f"'{self.__name__}' is a runpod {self.spec.kind.value} handle. " + f"use .remote(...) to execute remotely, .local(...) to run here, " + f"or .spawn(...) to fire and forget." + ) + + def __repr__(self) -> str: + return ( + f"<{type(self).__name__} {self.spec.kind.value}:{self.spec.name} " + f"app={self._app.name!r}>" + ) + + +class _RouteCaller: + """client-side http verb on an ApiHandle: Inference.post("/x", body).""" + + __slots__ = ("_handle", "_method") + + def __init__(self, handle: "ApiHandle", method: str): + self._handle = handle + self._method = method + + def __call__(self, path: str, body: Any = None, **kwargs: Any) -> Any: + from .context import block + + return block(self.aio(path, body, **kwargs)) + + async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: + target = await self._handle._app._resolve(self._handle.spec) + return await target.request(self._method, path, body, **kwargs) + + +class ApiHandle: + """handle for @app.api classes and asgi factories. + + route registration happens inside the decorated class via markers + (@get/@post/...); the handle only ever makes client calls, so there is + no decorator/client dual mode. + """ + + def __init__(self, app: "App", target: Any, spec: ResourceSpec): + self._app = app + self.spec = spec + self._cls: Optional[Type] = None + self._asgi_factory: Optional[Callable] = None + self._init_name: Optional[str] = None + + if inspect.isclass(target): + self._cls = target + self._collect_routes(target) + elif callable(target): + self._asgi_factory = target + spec.asgi_factory = getattr(target, "__qualname__", target.__name__) + else: + raise InvalidResourceError( + "@app.api must decorate a class with route markers or a " + "zero-argument function returning an asgi app" + ) + + self.get = _RouteCaller(self, "GET") + self.post = _RouteCaller(self, "POST") + self.put = _RouteCaller(self, "PUT") + self.delete = _RouteCaller(self, "DELETE") + self.patch = _RouteCaller(self, "PATCH") + + self.__name__ = getattr(target, "__name__", spec.name) + self.__doc__ = getattr(target, "__doc__", None) + self.__wrapped__ = target + + def _collect_routes(self, cls: Type) -> None: + seen: Dict[tuple, str] = {} + routes: List[RouteSpec] = [] + for name, member in inspect.getmembers(cls, callable): + route = route_of(member) + if route is not None: + method, path = route + if (method, path) in seen: + raise InvalidResourceError( + f"duplicate route {method} {path} on {cls.__name__}: " + f"'{seen[(method, path)]}' and '{name}'" + ) + seen[(method, path)] = name + routes.append(RouteSpec(method=method, path=path, handler_name=name)) + if is_init(member): + if self._init_name is not None and self._init_name != name: + raise InvalidResourceError( + f"multiple @init methods on {cls.__name__}: " + f"'{self._init_name}' and '{name}'" + ) + self._init_name = name + if not routes: + raise InvalidResourceError( + f"@app.api class {cls.__name__} defines no routes; mark " + f"methods with @get/@post/@put/@delete/@patch" + ) + self.spec.routes = routes + + def __repr__(self) -> str: + n_routes = len(self.spec.routes) + return ( + f"" + ) diff --git a/runpod/apps/invoker.py b/runpod/apps/invoker.py new file mode 100644 index 000000000..15251db9e --- /dev/null +++ b/runpod/apps/invoker.py @@ -0,0 +1,27 @@ +"""sync-by-default invocation with an async escape hatch. + +an Invoker wraps a coroutine factory. calling it blocks and returns the +result; calling `.aio(...)` returns the coroutine for the caller to await. + + handle.remote(x) # sync, blocks + await handle.remote.aio(x) # async +""" + +from typing import Any, Callable, Coroutine + +from .context import block + +CoroFactory = Callable[..., Coroutine[Any, Any, Any]] + + +class Invoker: + __slots__ = ("_factory",) + + def __init__(self, factory: CoroFactory): + self._factory = factory + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return block(self._factory(*args, **kwargs)) + + def aio(self, *args: Any, **kwargs: Any) -> Coroutine[Any, Any, Any]: + return self._factory(*args, **kwargs) diff --git a/runpod/apps/markers.py b/runpod/apps/markers.py new file mode 100644 index 000000000..9f8e1e09b --- /dev/null +++ b/runpod/apps/markers.py @@ -0,0 +1,89 @@ +"""route and lifecycle markers for @app.api classes. + +these decorators stamp metadata on methods; the ApiHandle collects them +when the class is registered. they do not wrap or change the function. + + @app.api(name="inference", gpu=GpuType.NVIDIA_GEFORCE_RTX_4090) + class Inference: + @init + def setup(self): + self.model = load_model() + + @post("/generate") + async def generate(self, body: dict): + return {"text": self.model.run(body["prompt"])} +""" + +from typing import Any, Callable + +ROUTE_ATTR = "__runpod_route__" +INIT_ATTR = "__runpod_init__" + +_VALID_METHODS = frozenset({"GET", "POST", "PUT", "DELETE", "PATCH"}) + +# paths used by the worker runtime; user routes must not collide +RESERVED_PATHS = frozenset({"/execute", "/ping"}) + + +def _route_marker(method: str, path: str) -> Callable[[Callable], Callable]: + if method not in _VALID_METHODS: + raise ValueError(f"method must be one of {sorted(_VALID_METHODS)}") + if not path.startswith("/"): + raise ValueError(f"path must start with '/', got: {path!r}") + if path in RESERVED_PATHS: + raise ValueError( + f"path {path!r} is reserved by the worker runtime " + f"(reserved: {', '.join(sorted(RESERVED_PATHS))})" + ) + + def marker(fn: Callable) -> Callable: + setattr(fn, ROUTE_ATTR, (method, path)) + return fn + + return marker + + +def get(path: str) -> Callable[[Callable], Callable]: + """mark a method as a GET route.""" + return _route_marker("GET", path) + + +def post(path: str) -> Callable[[Callable], Callable]: + """mark a method as a POST route.""" + return _route_marker("POST", path) + + +def put(path: str) -> Callable[[Callable], Callable]: + """mark a method as a PUT route.""" + return _route_marker("PUT", path) + + +def delete(path: str) -> Callable[[Callable], Callable]: + """mark a method as a DELETE route.""" + return _route_marker("DELETE", path) + + +def patch(path: str) -> Callable[[Callable], Callable]: + """mark a method as a PATCH route.""" + return _route_marker("PATCH", path) + + +def init(fn: Callable) -> Callable: + """mark a method (or function) as a worker-startup hook. + + on an @app.api class, the marked method runs after instantiation and + before the worker reports healthy. on a queue/task handle, use + `@handle.init` instead. + """ + setattr(fn, INIT_ATTR, True) + return fn + + +def route_of(fn: Any) -> "tuple[str, str] | None": + """return (method, path) if fn is marked as a route.""" + return getattr(fn, ROUTE_ATTR, None) + + +def is_init(fn: Any) -> bool: + """true if fn is marked as an init hook.""" + return getattr(fn, INIT_ATTR, False) is True diff --git a/runpod/apps/schedule.py b/runpod/apps/schedule.py new file mode 100644 index 000000000..c55d4bfc5 --- /dev/null +++ b/runpod/apps/schedule.py @@ -0,0 +1,41 @@ +"""the @schedule decorator: records a cron expression on a handle. + +schedules only take effect through `rp deploy` (the schedule must live +server-side). recurring schedule execution is gated on backend support; +deploying a scheduled resource raises ScheduleNotSupported until it lands. +""" + +from typing import Callable, TypeVar + +T = TypeVar("T") + +SCHEDULE_ATTR = "__runpod_schedule__" + +# flipped when the backend ships recurring schedule triggers +SCHEDULES_ENABLED = False + + +def schedule(*, cron: str) -> Callable[[T], T]: + """attach a cron schedule to a queue or task handle. + + @app.task(name="hourly", gpu=GpuType.NVIDIA_GEFORCE_RTX_4090) + @schedule(cron="0 * * * *") + async def hourly_job(): ... + + decorator order is flexible: @schedule can sit above or below the + app decorator. the cron string is validated at deploy time. + """ + if not cron or not isinstance(cron, str): + raise ValueError("cron must be a non-empty string") + + def decorator(target: T) -> T: + # works on both raw functions (before app decorator) and handles + # (after), since handles expose their spec and raw fns get stamped + spec = getattr(target, "spec", None) + if spec is not None: + spec.schedule = cron + else: + setattr(target, SCHEDULE_ATTR, cron) + return target + + return decorator diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py new file mode 100644 index 000000000..d7294af04 --- /dev/null +++ b/runpod/apps/spec.py @@ -0,0 +1,161 @@ +"""resource specifications produced by app decorators. + +a ResourceSpec is the declarative description of one deployable resource. +it is what the manifest serializes, what `rp deploy` ships to the backend, +and what dev-session provisioning consumes. it holds no live state. +""" + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional, Tuple, Union + +from .errors import InvalidResourceError +from .gpu import GpuGroup, GpuType + +DEFAULT_WORKERS: Tuple[int, int] = (0, 3) + + +class ResourceKind(str, Enum): + QUEUE = "queue" + """queue-based serverless endpoint.""" + + API = "api" + """load-balanced serverless endpoint with http routes.""" + + TASK = "task" + """ephemeral pod compute, provisioned per call.""" + + +@dataclass(frozen=True) +class RouteSpec: + """one http route on an api resource.""" + + method: str + path: str + handler_name: str + + +def normalize_workers(workers: Union[int, Tuple[int, int], None]) -> Tuple[int, int]: + """int n -> (0, n); (min, max) passes through; None -> default.""" + if workers is None: + return DEFAULT_WORKERS + if isinstance(workers, int): + parsed = (DEFAULT_WORKERS[0], workers) + elif isinstance(workers, (tuple, list)) and len(workers) == 2: + parsed = (int(workers[0]), int(workers[1])) + else: + raise InvalidResourceError( + f"workers must be an int or (min, max) tuple, got {workers!r}" + ) + min_w, max_w = parsed + if min_w < 0 or max_w < 1: + raise InvalidResourceError(f"invalid worker range ({min_w}, {max_w})") + if min_w > max_w: + raise InvalidResourceError( + f"workers min ({min_w}) cannot exceed max ({max_w})" + ) + return (min_w, max_w) + + +def normalize_gpu( + gpu: Union[GpuGroup, GpuType, str, List[Union[GpuGroup, GpuType, str]], None], +) -> Optional[List[str]]: + """normalize gpu input to a list of api-facing string values.""" + if gpu is None: + return None + if not isinstance(gpu, list): + gpu = [gpu] + out: List[str] = [] + for g in gpu: + if isinstance(g, (GpuGroup, GpuType)): + out.append(g.value) + elif isinstance(g, str): + out.append(g) + else: + raise InvalidResourceError( + f"gpu must be a GpuGroup, GpuType, or string, got {type(g).__name__}" + ) + return out + + +def normalize_cpu( + cpu: Union[str, List[str], None], +) -> Optional[List[str]]: + """normalize cpu instance ids to a list of strings.""" + if cpu is None: + return None + if isinstance(cpu, str): + return [cpu] + if isinstance(cpu, list): + return [str(c) for c in cpu] + raise InvalidResourceError( + f"cpu must be an instance id string or list, got {type(cpu).__name__}" + ) + + +@dataclass +class ResourceSpec: + """declarative config for one app resource.""" + + kind: ResourceKind + name: str + gpu: Optional[List[str]] = None + cpu: Optional[List[str]] = None + gpu_count: int = 1 + workers: Tuple[int, int] = DEFAULT_WORKERS + idle_timeout: int = 60 + dependencies: Optional[List[str]] = None + system_dependencies: Optional[List[str]] = None + volume: Optional[str] = None + env: Optional[Dict[str, str]] = None + image: Optional[str] = None + schedule: Optional[str] = None + routes: List[RouteSpec] = field(default_factory=list) + asgi_factory: Optional[str] = None + + def __post_init__(self) -> None: + if self.gpu is not None and self.cpu is not None: + raise InvalidResourceError( + f"resource '{self.name}': gpu and cpu are mutually exclusive" + ) + if not self.name: + raise InvalidResourceError("resource name must not be empty") + + @property + def is_cpu(self) -> bool: + return self.cpu is not None + + def to_manifest(self) -> Dict[str, Any]: + """serialize for the deploy manifest.""" + data: Dict[str, Any] = { + "kind": self.kind.value, + "name": self.name, + "gpuCount": self.gpu_count, + "workersMin": self.workers[0], + "workersMax": self.workers[1], + "idleTimeout": self.idle_timeout, + } + if self.gpu is not None: + data["gpus"] = self.gpu + if self.cpu is not None: + data["instanceIds"] = self.cpu + if self.dependencies: + data["dependencies"] = self.dependencies + if self.system_dependencies: + data["systemDependencies"] = self.system_dependencies + if self.volume: + data["networkVolume"] = self.volume + if self.env: + data["env"] = self.env + if self.image: + data["imageName"] = self.image + if self.schedule: + data["schedule"] = self.schedule + if self.routes: + data["routes"] = [ + {"method": r.method, "path": r.path, "handler": r.handler_name} + for r in self.routes + ] + if self.asgi_factory: + data["asgiFactory"] = self.asgi_factory + return data diff --git a/runpod/apps/stubs.py b/runpod/apps/stubs.py new file mode 100644 index 000000000..d2b9fe08b --- /dev/null +++ b/runpod/apps/stubs.py @@ -0,0 +1,101 @@ +"""stubs for resources deployed from other codebases. + +when you know a resource exists but don't have its source, a stub gives +you the same invocation surface as a local handle: + + external_queue = runpod.Queue(app="other-app", name="transcribe") + external_queue.remote(audio_url="https://...") + + external_api = runpod.Api(app="other-app", name="inference") + external_api.post("/generate", {"prompt": "hi"}) + +resolution goes through the same server-side sentinel path as deployed +handles, so a stub is just a handle without a function body. +""" + +import os +from typing import Any, Dict, Optional + +from .errors import EndpointNotFound, RemoteExecutionError +from .invoker import Invoker +from .targets import DEFAULT_TIMEOUT_SECONDS, SentinelTarget + +DEFAULT_ENV = "default" + + +class _StubBase: + def __init__( + self, + *, + app: str, + name: Optional[str] = None, + id: Optional[str] = None, + env: Optional[str] = None, + ): + if (name is None) == (id is None): + raise ValueError("provide exactly one of name= or id=") + self.app_name = app + self.resource_name = name + self.resource_id = id + self.env = env or os.getenv("FLASH_ENVIRONMENT") or DEFAULT_ENV + + def _target(self) -> SentinelTarget: + # id-based stubs still route through the sentinel; the id is + # passed as the resource key and resolved server-side + resource = self.resource_name or self.resource_id or "" + return SentinelTarget(self.app_name, self.env, resource) + + +class Queue(_StubBase): + """client for a queue resource deployed elsewhere.""" + + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + self.remote = Invoker(self._remote_async) + self.spawn = Invoker(self._spawn_async) + + async def _remote_async(self, **kwargs: Any) -> Any: + payload = {"input": kwargs or {"__empty": True}} + return await self._target().invoke(payload) + + async def _spawn_async(self, **kwargs: Any) -> Dict[str, Any]: + payload = {"input": kwargs or {"__empty": True}} + return await self._target().submit(payload) + + def __repr__(self) -> str: + ref = self.resource_name or self.resource_id + return f"" + + +class _StubRouteCaller: + __slots__ = ("_stub", "_method") + + def __init__(self, stub: "Api", method: str): + self._stub = stub + self._method = method + + def __call__(self, path: str, body: Any = None, **kwargs: Any) -> Any: + from .context import block + + return block(self.aio(path, body, **kwargs)) + + async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: + return await self._stub._target().request( + self._method, path, body, **kwargs + ) + + +class Api(_StubBase): + """client for an api resource deployed elsewhere.""" + + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + self.get = _StubRouteCaller(self, "GET") + self.post = _StubRouteCaller(self, "POST") + self.put = _StubRouteCaller(self, "PUT") + self.delete = _StubRouteCaller(self, "DELETE") + self.patch = _StubRouteCaller(self, "PATCH") + + def __repr__(self) -> str: + ref = self.resource_name or self.resource_id + return f"" diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py new file mode 100644 index 000000000..384c18b5d --- /dev/null +++ b/runpod/apps/targets.py @@ -0,0 +1,270 @@ +"""invocation targets: where a `.remote()` call actually goes. + +a target is resolved per-call by App._resolve and knows how to reach one +deployed resource. all resolution is server-side (sentinel headers or the +runpod api); no local state is kept. +""" + +import inspect +import json +import os +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, Optional + +import aiohttp + +from ..user_agent import USER_AGENT +from .errors import EndpointNotFound, RemoteExecutionError +from .spec import ResourceSpec + +# the sentinel pseudo-endpoint id; ai-api resolves the real endpoint from +# the X-Flash-App / X-Flash-Environment / X-Flash-Endpoint headers. +SENTINEL_ID = "flash" + +DEFAULT_TIMEOUT_SECONDS = 300.0 + +# terminal job statuses on the queue data plane +FINAL_STATUSES = frozenset({"COMPLETED", "FAILED", "CANCELLED", "TIMED_OUT"}) + + +def _api_key() -> str: + key = os.getenv("RUNPOD_API_KEY") + if not key: + import runpod + + key = runpod.api_key + if not key: + raise RuntimeError( + "no api key configured. run `rp login` or set RUNPOD_API_KEY." + ) + return key + + +def _endpoint_url_base() -> str: + import runpod + + return runpod.endpoint_url_base.rstrip("/") + + +def _lb_domain() -> str: + """host portion of the data-plane base url, for lb subdomain urls.""" + base = _endpoint_url_base() + host = base.split("://", 1)[-1] + return host.split("/", 1)[0] + + +def _headers(extra: Optional[Dict[str, str]] = None) -> Dict[str, str]: + headers = { + "Authorization": f"Bearer {_api_key()}", + "Content-Type": "application/json", + "User-Agent": USER_AGENT, + } + if extra: + headers.update(extra) + return headers + + +def args_to_input(fn: Callable, args: tuple, kwargs: dict) -> Dict[str, Any]: + """map positional args onto parameter names so the job input is a dict.""" + sig = inspect.signature(fn) + params = [n for n in sig.parameters if n not in ("self", "cls")] + body: Dict[str, Any] = {} + for i, arg in enumerate(args): + if i >= len(params): + raise TypeError( + f"{fn.__name__}() got {len(args)} positional args, " + f"expected at most {len(params)}" + ) + body[params[i]] = arg + body.update(kwargs) + # the platform strips empty input dicts from jobs; keep one field so + # the worker's job polling never sees a missing input + if not body: + body = {"__empty": True} + return body + + +def unwrap_job_output(data: Dict[str, Any]) -> Any: + """extract output from a runsync-style response, raising on failure.""" + if data.get("status") == "FAILED" or data.get("error"): + err = data.get("error") or data.get("output", {}).get("error", "unknown") + raise RemoteExecutionError(f"remote execution failed: {err}") + output = data.get("output", data) + if isinstance(output, dict) and "error" in output: + raise RemoteExecutionError(f"remote execution failed: {output['error']}") + return output + + +class InvocationTarget(ABC): + """one resolved destination for remote calls.""" + + @abstractmethod + async def invoke(self, payload: Dict[str, Any], *, timeout: float) -> Any: + """submit and wait for the result.""" + + @abstractmethod + async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """submit without waiting; returns the raw job data.""" + + async def request( + self, method: str, path: str, body: Any = None, *, timeout: float + ) -> Any: + """plain http call (lb resources only).""" + raise NotImplementedError(f"{type(self).__name__} does not serve http routes") + + +async def _post_json( + url: str, + payload: Any, + headers: Dict[str, str], + timeout: float, + *, + app_name: str = "", + resource_name: str = "", +) -> Dict[str, Any]: + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.post(url, json=payload, headers=headers) as resp: + if resp.status == 404: + raise EndpointNotFound(app_name, resource_name) + resp.raise_for_status() + return await resp.json() + + +class SentinelTarget(InvocationTarget): + """routes through the sentinel endpoint; ai-api resolves the real + endpoint server-side from app/env/resource headers. + + used for deployed resources from every context (worker-to-worker and + local caller alike), which is what makes resolution stateless. + """ + + def __init__(self, app_name: str, env_name: str, resource_name: str): + self.app_name = app_name + self.env_name = env_name + self.resource_name = resource_name + + def _sentinel_headers(self) -> Dict[str, str]: + return _headers( + { + "X-Flash-App": self.app_name, + "X-Flash-Environment": self.env_name, + "X-Flash-Endpoint": self.resource_name, + } + ) + + async def invoke( + self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> Any: + url = f"{_endpoint_url_base()}/{SENTINEL_ID}/runsync" + data = await _post_json( + url, + payload, + self._sentinel_headers(), + timeout, + app_name=self.app_name, + resource_name=self.resource_name, + ) + return unwrap_job_output(data) + + async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: + url = f"{_endpoint_url_base()}/{SENTINEL_ID}/run" + return await _post_json( + url, + payload, + self._sentinel_headers(), + DEFAULT_TIMEOUT_SECONDS, + app_name=self.app_name, + resource_name=self.resource_name, + ) + + async def request( + self, + method: str, + path: str, + body: Any = None, + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Any: + url = f"https://{SENTINEL_ID}.{_lb_domain()}{path}" + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.request( + method, url, json=body, headers=self._sentinel_headers() + ) as resp: + if resp.status == 404: + raise EndpointNotFound(self.app_name, self.resource_name) + resp.raise_for_status() + text = await resp.text() + try: + return json.loads(text) + except json.JSONDecodeError: + return text + + +class LiveTarget(InvocationTarget): + """a dev-session live endpoint: source code ships with every request, + so the worker executes whatever is currently on disk. + + provisioning of the live endpoint itself is owned by the dev session + (see runpod.apps.dev); this target only speaks to an endpoint id. + """ + + def __init__(self, endpoint_id: str): + self.endpoint_id = endpoint_id + + async def invoke( + self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> Any: + url = f"{_endpoint_url_base()}/{self.endpoint_id}/runsync" + data = await _post_json(url, payload, _headers(), timeout) + return unwrap_job_output(data) + + async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: + url = f"{_endpoint_url_base()}/{self.endpoint_id}/run" + return await _post_json(url, payload, _headers(), DEFAULT_TIMEOUT_SECONDS) + + async def request( + self, + method: str, + path: str, + body: Any = None, + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Any: + url = f"https://{self.endpoint_id}.{_lb_domain()}{path}" + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.request( + method, url, json=body, headers=_headers() + ) as resp: + resp.raise_for_status() + text = await resp.text() + try: + return json.loads(text) + except json.JSONDecodeError: + return text + + +class PodTarget(InvocationTarget): + """ephemeral pod execution for @app.task: provision a pod, run the + function body, collect the result, terminate. + + transport lands with the task execution engine. + """ + + def __init__(self, spec: ResourceSpec, fn: Callable): + self.spec = spec + self.fn = fn + + async def invoke( + self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> Any: + raise NotImplementedError( + "task pod execution is not wired up yet; use .local() meanwhile" + ) + + async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: + raise NotImplementedError( + "task pod execution is not wired up yet; use .local() meanwhile" + ) diff --git a/tests/test_apps/__init__.py b/tests/test_apps/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py new file mode 100644 index 000000000..236cd72b7 --- /dev/null +++ b/tests/test_apps/test_app.py @@ -0,0 +1,262 @@ +"""tests for the App registry and decorators.""" + +import pytest + +import runpod +from runpod.apps import App, ApiHandle, FunctionHandle, ResourceKind +from runpod.apps.app import _REGISTRY, _clear_registry +from runpod.apps.errors import InvalidResourceError +from runpod.apps.markers import get, init, post + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +def test_app_registers_itself(): + app = App("my-app") + assert app in _REGISTRY + assert runpod.apps.get_registered_apps() == [app] + + +def test_app_requires_name(): + with pytest.raises(InvalidResourceError): + App("") + + +def test_queue_decorator_returns_handle(): + app = App("a") + + @app.queue(name="q", gpu=runpod.GpuType.NVIDIA_L4, workers=3) + async def q(x: int): + return x + + assert isinstance(q, FunctionHandle) + assert q.spec.kind is ResourceKind.QUEUE + assert q.spec.name == "q" + assert q.spec.gpu == ["NVIDIA L4"] + assert q.spec.workers == (0, 3) + assert app.resources["q"] is q + + +def test_queue_name_defaults_to_function_name(): + app = App("a") + + @app.queue() + def my_fn(): + return 1 + + assert my_fn.spec.name == "my_fn" + + +def test_task_decorator(): + app = App("a") + + @app.task(name="t", cpu="cpu5c-2-4") + def t(): + return "ok" + + assert t.spec.kind is ResourceKind.TASK + assert t.spec.cpu == ["cpu5c-2-4"] + + +def test_gpu_cpu_mutually_exclusive(): + app = App("a") + with pytest.raises(InvalidResourceError): + + @app.queue(name="bad", gpu=runpod.GpuType.NVIDIA_L4, cpu="cpu5c-2-4") + def bad(): + pass + + +def test_duplicate_resource_name_rejected(): + app = App("a") + + @app.queue(name="dupe") + def one(): + pass + + with pytest.raises(InvalidResourceError): + + @app.queue(name="dupe") + def two(): + pass + + +def test_workers_int_shorthand(): + app = App("a") + + @app.queue(name="q", workers=5) + def q(): + pass + + assert q.spec.workers == (0, 5) + + +def test_workers_tuple(): + app = App("a") + + @app.queue(name="q", workers=(1, 4)) + def q(): + pass + + assert q.spec.workers == (1, 4) + + +def test_workers_invalid(): + app = App("a") + with pytest.raises(InvalidResourceError): + + @app.queue(name="q", workers=(3, 1)) + def q(): + pass + + +def test_api_class_collects_routes(): + app = App("a") + + @app.api(name="api", cpu="cpu5c-2-4") + class Inference: + @init + def setup(self): + self.model = object() + + @get("/health") + def health(self): + return {"ok": True} + + @post("/generate") + async def generate(self, body: dict): + return body + + assert isinstance(Inference, ApiHandle) + routes = {(r.method, r.path) for r in Inference.spec.routes} + assert routes == {("GET", "/health"), ("POST", "/generate")} + assert Inference._init_name == "setup" + + +def test_api_class_without_routes_rejected(): + app = App("a") + with pytest.raises(InvalidResourceError): + + @app.api(name="api") + class Empty: + def nothing(self): + pass + + +def test_api_duplicate_route_rejected(): + app = App("a") + with pytest.raises(InvalidResourceError): + + @app.api(name="api") + class Dupe: + @post("/x") + def one(self): + pass + + @post("/x") + def two(self): + pass + + +def test_api_asgi_factory(): + app = App("a") + + @app.api(name="web", cpu="cpu5c-2-4") + def web(): + return object() + + assert isinstance(web, ApiHandle) + assert web.spec.asgi_factory is not None + + +def test_api_reserved_path_rejected(): + with pytest.raises(ValueError): + post("/execute") + + +def test_schedule_below_app_decorator(): + app = App("a") + + @app.task(name="cron") + @runpod.schedule(cron="0 * * * *") + async def cron(): + pass + + assert cron.spec.schedule == "0 * * * *" + + +def test_schedule_above_app_decorator(): + app = App("a") + + @runpod.schedule(cron="*/5 * * * *") + @app.task(name="cron") + async def cron(): + pass + + assert cron.spec.schedule == "*/5 * * * *" + + +def test_handle_direct_call_raises(): + app = App("a") + + @app.queue(name="q") + def q(): + pass + + with pytest.raises(TypeError, match=r"\.remote\("): + q() + + +def test_handle_local_runs_body(): + app = App("a") + + @app.queue(name="q") + def q(x): + return x * 2 + + assert q.local(21) == 42 + + +async def test_handle_local_async_follows_signature(): + app = App("a") + + @app.queue(name="q") + async def q(x): + return x + 1 + + assert await q.local(1) == 2 + + +def test_manifest_serialization(): + app = App("a") + volume = "my-volume" + + @app.queue( + name="q", + gpu=[runpod.GpuGroup.ADA_24], + workers=(1, 3), + dependencies=["torch"], + volume=volume, + env={"KEY": "val"}, + ) + def q(): + pass + + manifest = q.spec.to_manifest() + assert manifest == { + "kind": "queue", + "name": "q", + "gpuCount": 1, + "workersMin": 1, + "workersMax": 3, + "idleTimeout": 60, + "gpus": ["ADA_24"], + "dependencies": ["torch"], + "networkVolume": "my-volume", + "env": {"KEY": "val"}, + } diff --git a/tests/test_apps/test_compat_lock.py b/tests/test_apps/test_compat_lock.py new file mode 100644 index 000000000..d810aa5c1 --- /dev/null +++ b/tests/test_apps/test_compat_lock.py @@ -0,0 +1,105 @@ +"""compatibility locks: the legacy surface must not change. + +these tests intentionally assert exact signatures and behaviors of the +pre-apps sdk. if one fails, the apps work has leaked into the legacy +surface, which is a release blocker. +""" + +import inspect +import unittest +from unittest.mock import patch + +import runpod +from runpod.endpoint.runner import Endpoint, Job, RunPodClient + + +class TestEndpointSignatureLock(unittest.TestCase): + """runpod.Endpoint(id).run_sync() is byte-for-byte the legacy client.""" + + def test_endpoint_is_runner_endpoint(self): + self.assertIs(runpod.Endpoint, Endpoint) + + def test_constructor_signature(self): + params = list(inspect.signature(Endpoint.__init__).parameters) + self.assertEqual(params, ["self", "endpoint_id", "api_key"]) + + def test_run_sync_signature(self): + params = list(inspect.signature(Endpoint.run_sync).parameters) + self.assertEqual(params, ["self", "request_input", "timeout"]) + default = inspect.signature(Endpoint.run_sync).parameters["timeout"].default + self.assertEqual(default, 86400) + + def test_run_signature(self): + params = list(inspect.signature(Endpoint.run).parameters) + self.assertEqual(params, ["self", "request_input"]) + + def test_endpoint_methods_present(self): + for method in ("run", "run_sync", "health", "purge_queue"): + self.assertTrue(callable(getattr(Endpoint, method))) + + def test_job_methods_present(self): + for method in ("status", "output", "stream", "cancel"): + self.assertTrue(callable(getattr(Job, method))) + + @patch.object(RunPodClient, "post") + @patch("runpod.api_key", "test-key") + def test_run_sync_behavior(self, mock_post): + mock_post.return_value = { + "id": "job-1", + "status": "COMPLETED", + "output": {"result": 42}, + } + endpoint = Endpoint("ep-abc", api_key="test-key") + result = endpoint.run_sync({"prompt": "hi"}) + self.assertEqual(result, {"result": 42}) + mock_post.assert_called_once_with( + "ep-abc/runsync", {"input": {"prompt": "hi"}}, timeout=86400 + ) + + @patch.object(RunPodClient, "post") + @patch("runpod.api_key", "test-key") + def test_run_returns_job(self, mock_post): + mock_post.return_value = {"id": "job-2"} + endpoint = Endpoint("ep-abc", api_key="test-key") + job = endpoint.run({"prompt": "hi"}) + self.assertIsInstance(job, Job) + self.assertEqual(job.job_id, "job-2") + + +class TestLegacyFlatVerbsPresent(unittest.TestCase): + """the flat api verbs stay importable from the package root.""" + + def test_flat_verbs(self): + for verb in ( + "create_pod", + "get_pods", + "get_pod", + "stop_pod", + "resume_pod", + "terminate_pod", + "create_endpoint", + "get_endpoints", + "create_template", + "get_gpus", + "get_gpu", + "get_user", + "update_user_settings", + "update_endpoint_template", + "create_container_registry_auth", + "update_container_registry_auth", + "delete_container_registry_auth", + ): + self.assertTrue( + callable(getattr(runpod, verb)), f"runpod.{verb} missing" + ) + + def test_asyncio_endpoint_present(self): + self.assertTrue(hasattr(runpod, "AsyncioEndpoint")) + self.assertTrue(hasattr(runpod, "AsyncioJob")) + + def test_serverless_start_present(self): + self.assertTrue(callable(runpod.serverless.start)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py new file mode 100644 index 000000000..889c8128c --- /dev/null +++ b/tests/test_apps/test_dispatch.py @@ -0,0 +1,241 @@ +"""tests for context detection and remote dispatch.""" + +from unittest.mock import AsyncMock, patch + +import pytest + +import runpod +from runpod.apps import App, Context, current_context, is_local +from runpod.apps.app import _clear_registry +from runpod.apps.errors import EndpointNotFound, RemoteExecutionError +from runpod.apps.targets import ( + SentinelTarget, + args_to_input, + unwrap_job_output, +) + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +class TestContextDetection: + def test_local_by_default(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + assert current_context() is Context.LOCAL + assert is_local() is True + + def test_worker_via_endpoint_id(self, monkeypatch): + monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") + assert current_context() is Context.WORKER + assert is_local() is False + + def test_worker_via_pod_id(self, monkeypatch): + monkeypatch.delenv("RUNPOD_ENDPOINT_ID", raising=False) + monkeypatch.setenv("RUNPOD_POD_ID", "pod-1") + assert current_context() is Context.WORKER + + def test_dev_session(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("RUNPOD_DEV_SESSION", "1") + assert current_context() is Context.DEV + assert is_local() is True + + +class TestArgsToInput: + def test_positional_mapped_to_names(self): + def fn(prompt, temp): + pass + + assert args_to_input(fn, ("hi", 0.5), {}) == {"prompt": "hi", "temp": 0.5} + + def test_kwargs_merged(self): + def fn(a, b): + pass + + assert args_to_input(fn, (1,), {"b": 2}) == {"a": 1, "b": 2} + + def test_empty_gets_placeholder(self): + def fn(): + pass + + assert args_to_input(fn, (), {}) == {"__empty": True} + + def test_too_many_positional(self): + def fn(a): + pass + + with pytest.raises(TypeError): + args_to_input(fn, (1, 2), {}) + + +class TestUnwrapJobOutput: + def test_output_extracted(self): + assert unwrap_job_output({"status": "COMPLETED", "output": {"x": 1}}) == { + "x": 1 + } + + def test_failed_raises(self): + with pytest.raises(RemoteExecutionError): + unwrap_job_output({"status": "FAILED", "error": "boom"}) + + def test_error_in_output_raises(self): + with pytest.raises(RemoteExecutionError): + unwrap_job_output({"status": "COMPLETED", "output": {"error": "bad"}}) + + +class TestRemoteDispatch: + def test_remote_sync_via_sentinel(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = {"doubled": 4} + result = q.remote(2) + + assert result == {"doubled": 4} + mock_invoke.assert_awaited_once_with({"input": {"x": 2}}) + + async def test_remote_aio(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + async def q(x): + return x + + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = {"ok": True} + result = await q.remote.aio(1) + + assert result == {"ok": True} + + def test_worker_executes_own_body(self, monkeypatch): + monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x * 10 + + assert q.remote(4) == 40 + + def test_worker_calls_sibling_via_sentinel(self, monkeypatch): + monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") + monkeypatch.setenv("FLASH_RESOURCE_NAME", "other") + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = "remote-result" + result = q.remote(1) + + assert result == "remote-result" + + def test_dev_without_provisioned_target_raises(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("RUNPOD_DEV_SESSION", "1") + app = App("my-app") + + @app.queue(name="q") + def q(): + pass + + with pytest.raises(EndpointNotFound): + q.remote() + + def test_spawn_returns_job(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "submit", new_callable=AsyncMock + ) as mock_submit: + mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} + job = q.spawn(1) + + assert job.id == "job-1" + assert job.status == "IN_QUEUE" + + +class TestStubs: + def test_queue_stub_requires_name_or_id(self): + with pytest.raises(ValueError): + runpod.Queue(app="a") + with pytest.raises(ValueError): + runpod.Queue(app="a", name="x", id="y") + + def test_queue_stub_remote(self): + stub = runpod.Queue(app="other-app", name="q") + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = {"ok": 1} + result = stub.remote(prompt="hi") + + assert result == {"ok": 1} + mock_invoke.assert_awaited_once_with({"input": {"prompt": "hi"}}) + + def test_api_stub_http(self): + stub = runpod.Api(app="other-app", name="api") + with patch.object( + SentinelTarget, "request", new_callable=AsyncMock + ) as mock_request: + mock_request.return_value = {"status": "healthy"} + result = stub.get("/health") + + assert result == {"status": "healthy"} + mock_request.assert_awaited_once_with("GET", "/health", None) + + +class TestSyncBridge: + def test_remote_inside_running_loop(self, monkeypatch): + """calling sync .remote() from inside an event loop must not raise.""" + import asyncio + + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = "bridged" + + async def caller(): + return q.remote(1) + + result = asyncio.run(caller()) + + assert result == "bridged" diff --git a/tests/test_init.py b/tests/test_init.py index 912423288..a5ca488a2 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -98,6 +98,10 @@ def test_all_covers_expected_public_api(self): 'check_credentials', 'get_credentials', 'set_credentials', # Endpoint classes 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', + # Apps surface + 'Api', 'App', 'CpuInstanceType', 'EndpointNotFound', 'GpuGroup', + 'GpuType', 'Queue', 'delete', 'get', 'init', 'is_local', + 'local_entrypoint', 'patch', 'post', 'put', 'schedule', # Serverless module 'serverless', # Logger class From a51a633da968833b1a9673ee6ade3971df7b90f2 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:36:04 -0700 Subject: [PATCH 002/120] feat: add deploy pipeline, dev sessions, and rp cli --- pyproject.toml | 1 + requirements.txt | 3 +- runpod/api/graphql.py | 102 ++++- runpod/apps/api.py | 179 ++++++++ runpod/apps/deploy.py | 170 +++++++ runpod/apps/dev.py | 144 ++++++ runpod/apps/discovery.py | 100 ++++ runpod/apps/handles.py | 5 +- runpod/apps/protocol.py | 60 +++ runpod/apps/serialization.py | 57 +++ runpod/apps/targets.py | 52 ++- runpod/cli/entry.py | 2 - runpod/cli/groups/project/__init__.py | 1 - runpod/cli/groups/project/commands.py | 157 ------- runpod/cli/groups/project/functions.py | 429 ------------------ runpod/cli/groups/project/helpers.py | 90 ---- .../starter_templates/default/.runpodignore | 2 - .../default/builder/requirements.txt | 8 - .../starter_templates/default/src/handler.py | 15 - .../llama2/builder/requirements.txt | 4 - .../starter_templates/llama2/src/handler.py | 42 -- runpod/cli/utils/rp_sync.py | 67 --- runpod/rp_cli/__init__.py | 1 + runpod/rp_cli/main.py | 173 +++++++ setup.py | 7 +- tests/test_apps/test_deploy.py | 190 ++++++++ .../test_cli_groups/test_project_commands.py | 162 ------- .../test_cli_groups/test_project_functions.py | 338 -------------- .../test_cli_groups/test_project_helpers.py | 114 ----- tests/test_cli/test_cli_utils/test_sync.py | 130 ------ 30 files changed, 1217 insertions(+), 1588 deletions(-) create mode 100644 runpod/apps/api.py create mode 100644 runpod/apps/deploy.py create mode 100644 runpod/apps/dev.py create mode 100644 runpod/apps/discovery.py create mode 100644 runpod/apps/protocol.py create mode 100644 runpod/apps/serialization.py delete mode 100644 runpod/cli/groups/project/__init__.py delete mode 100644 runpod/cli/groups/project/commands.py delete mode 100644 runpod/cli/groups/project/functions.py delete mode 100644 runpod/cli/groups/project/helpers.py delete mode 100644 runpod/cli/groups/project/starter_templates/default/.runpodignore delete mode 100644 runpod/cli/groups/project/starter_templates/default/builder/requirements.txt delete mode 100644 runpod/cli/groups/project/starter_templates/default/src/handler.py delete mode 100644 runpod/cli/groups/project/starter_templates/llama2/builder/requirements.txt delete mode 100644 runpod/cli/groups/project/starter_templates/llama2/src/handler.py delete mode 100644 runpod/cli/utils/rp_sync.py create mode 100644 runpod/rp_cli/__init__.py create mode 100644 runpod/rp_cli/main.py create mode 100644 tests/test_apps/test_deploy.py delete mode 100644 tests/test_cli/test_cli_groups/test_project_commands.py delete mode 100644 tests/test_cli/test_cli_groups/test_project_functions.py delete mode 100644 tests/test_cli/test_cli_groups/test_project_helpers.py delete mode 100644 tests/test_cli/test_cli_utils/test_sync.py diff --git a/pyproject.toml b/pyproject.toml index d4639a153..6441f49b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ local_scheme = "no-local-version" [project.scripts] runpod = "runpod.cli.entry:runpod_cli" +rp = "runpod.rp_cli.main:run" [dependency-groups] diff --git a/requirements.txt b/requirements.txt index 4c09742b3..23500ff14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ aiohttp-retry >= 2.8.3 backoff >= 2.2.1 boto3 >= 1.26.165 click >= 8.1.7 +cloudpickle >= 3.0.0 colorama >= 0.4.6, < 0.4.7 cryptography >= 48.0.1 fastapi[all] >= 0.138.1 @@ -16,5 +17,5 @@ requests >= 2.31.0 tomli >= 2.4.1 tomlkit >= 0.15.0 tqdm-loggable >= 0.2 +typer >= 0.12.0 urllib3 >= 1.26.6 -watchdog >= 3.0.0 diff --git a/runpod/api/graphql.py b/runpod/api/graphql.py index c7c3ccc36..8074163b3 100644 --- a/runpod/api/graphql.py +++ b/runpod/api/graphql.py @@ -14,40 +14,102 @@ HTTP_STATUS_UNAUTHORIZED = 401 -def run_graphql_query(query: str, api_key: Optional[str] = None) -> Dict[str, Any]: - """ - Run a GraphQL query with optional API key override. - - Args: - query: The GraphQL query to execute. - api_key: Optional API key to use for this query. - """ +def _graphql_url() -> str: + api_url_base = os.environ.get("RUNPOD_API_BASE_URL", "https://api.runpod.io") + return f"{api_url_base}/graphql" + + +def _resolve_api_key(api_key: Optional[str]) -> str: from runpod import api_key as global_api_key # pylint: disable=import-outside-toplevel, cyclic-import - - # Use provided API key or fall back to global + effective_api_key = api_key or global_api_key - if not effective_api_key: raise error.AuthenticationError("No API key provided") + return effective_api_key - api_url_base = os.environ.get("RUNPOD_API_BASE_URL", "https://api.runpod.io") - url = f"{api_url_base}/graphql" - headers = { +def _build_headers(api_key: str) -> Dict[str, str]: + return { "Content-Type": "application/json", "User-Agent": USER_AGENT, - "Authorization": f"Bearer {effective_api_key}", + "Authorization": f"Bearer {api_key}", } - data = json.dumps({"query": query}) - response = requests.post(url, headers=headers, data=data, timeout=30) + +def _build_payload(query: str, variables: Optional[Dict[str, Any]]) -> Dict[str, Any]: + payload: Dict[str, Any] = {"query": query} + if variables: + payload["variables"] = variables + return payload + + +def _check_response(response_json: Dict[str, Any], query: str) -> Dict[str, Any]: + if "errors" in response_json: + raise error.QueryError(response_json["errors"][0]["message"], query) + return response_json + + +def run_graphql_query( + query: str, + api_key: Optional[str] = None, + variables: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Run a GraphQL query with optional API key override. + + Args: + query: The GraphQL query to execute. + api_key: Optional API key to use for this query. + variables: Optional GraphQL variables to send with the query. + """ + effective_api_key = _resolve_api_key(api_key) + + response = requests.post( + _graphql_url(), + headers=_build_headers(effective_api_key), + data=json.dumps(_build_payload(query, variables)), + timeout=30, + ) if response.status_code == HTTP_STATUS_UNAUTHORIZED: raise error.AuthenticationError( "Unauthorized request, please check your API key." ) - if "errors" in response.json(): - raise error.QueryError(response.json()["errors"][0]["message"], query) + return _check_response(response.json(), query) + + +async def run_graphql_query_async( + query: str, + api_key: Optional[str] = None, + variables: Optional[Dict[str, Any]] = None, + timeout: float = 60.0, +) -> Dict[str, Any]: + """ + Async variant of run_graphql_query, sharing the same url, headers, + auth resolution, and error handling. + + Args: + query: The GraphQL query to execute. + api_key: Optional API key to use for this query. + variables: Optional GraphQL variables to send with the query. + timeout: Total request timeout in seconds. + """ + import aiohttp # pylint: disable=import-outside-toplevel + + effective_api_key = _resolve_api_key(api_key) + + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.post( + _graphql_url(), + headers=_build_headers(effective_api_key), + json=_build_payload(query, variables), + ) as response: + if response.status == HTTP_STATUS_UNAUTHORIZED: + raise error.AuthenticationError( + "Unauthorized request, please check your API key." + ) + response_json = await response.json() - return response.json() + return _check_response(response_json, query) diff --git a/runpod/apps/api.py b/runpod/apps/api.py new file mode 100644 index 000000000..913facd9e --- /dev/null +++ b/runpod/apps/api.py @@ -0,0 +1,179 @@ +"""control-plane calls for app provisioning. + +a thin wrapper over runpod.api.graphql's shared async transport, +scoped to what the apps surface needs: endpoint save/delete for dev +sessions, and the app / build / environment lifecycle for deploys. +management verbs for the wider sdk stay in runpod.api.ctl_commands. +""" + +from typing import Any, Dict, List, Optional + +import aiohttp + +from ..api.graphql import run_graphql_query_async +from ..error import QueryError + + +class AppsApiClient: + """async control-plane client scoped to apps provisioning.""" + + def __init__(self, api_key: Optional[str] = None): + self._api_key = api_key + + async def _execute( + self, query: str, variables: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + response = await run_graphql_query_async( + query, api_key=self._api_key, variables=variables + ) + return response["data"] + + # -- endpoints (dev session provisioning) -- + + async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: + """create or update a serverless endpoint. include id to update.""" + mutation = """ + mutation saveEndpoint($input: EndpointInput!) { + saveEndpoint(input: $input) { + id + name + templateId + gpuIds + instanceIds + workersMin + workersMax + idleTimeout + } + } + """ + data = await self._execute(mutation, {"input": endpoint_input}) + return data["saveEndpoint"] + + async def delete_endpoint(self, endpoint_id: str) -> bool: + mutation = """ + mutation deleteEndpoint($id: String!) { + deleteEndpoint(id: $id) + } + """ + data = await self._execute(mutation, {"id": endpoint_id}) + return "deleteEndpoint" in data + + async def list_my_endpoints(self) -> List[Dict[str, Any]]: + query = """ + query myEndpoints { + myself { + endpoints { id name templateId workersMin workersMax } + } + } + """ + data = await self._execute(query) + return data["myself"]["endpoints"] + + # -- app lifecycle (deploy) -- + + async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: + query = """ + query getFlashAppByName($flashAppName: String!) { + flashAppByName(flashAppName: $flashAppName) { + id + name + flashEnvironments { id name state activeBuildId } + } + } + """ + try: + data = await self._execute(query, {"flashAppName": name}) + except QueryError as exc: + if "not found" in str(exc).lower(): + return None + raise + return data["flashAppByName"] + + async def create_app(self, name: str) -> Dict[str, Any]: + mutation = """ + mutation createFlashApp($input: CreateFlashAppInput!) { + createFlashApp(input: $input) { id name } + } + """ + data = await self._execute(mutation, {"input": {"name": name}}) + return data["createFlashApp"] + + async def create_environment(self, app_id: str, name: str) -> Dict[str, Any]: + mutation = """ + mutation createFlashEnvironment($input: CreateFlashEnvironmentInput!) { + createFlashEnvironment(input: $input) { id name } + } + """ + data = await self._execute( + mutation, {"input": {"flashAppId": app_id, "name": name}} + ) + return data["createFlashEnvironment"] + + async def prepare_artifact_upload( + self, app_id: str, tarball_size: int + ) -> Dict[str, Any]: + mutation = """ + mutation PrepareArtifactUpload($input: PrepareFlashArtifactUploadInput!) { + prepareFlashArtifactUpload(input: $input) { + uploadUrl + objectKey + expiresAt + } + } + """ + data = await self._execute( + mutation, {"input": {"flashAppId": app_id, "tarballSize": tarball_size}} + ) + return data["prepareFlashArtifactUpload"] + + async def finalize_artifact_upload( + self, app_id: str, object_key: str, manifest: Dict[str, Any] + ) -> Dict[str, Any]: + mutation = """ + mutation FinalizeArtifactUpload($input: FinalizeFlashArtifactUploadInput!) { + finalizeFlashArtifactUpload(input: $input) { id manifest } + } + """ + data = await self._execute( + mutation, + { + "input": { + "flashAppId": app_id, + "objectKey": object_key, + "manifest": manifest, + } + }, + ) + return data["finalizeFlashArtifactUpload"] + + async def deploy_build( + self, environment_id: str, build_id: str + ) -> Dict[str, Any]: + mutation = """ + mutation deployBuildToEnvironment($input: DeployBuildToEnvironmentInput!) { + deployBuildToEnvironment(input: $input) { id name } + } + """ + data = await self._execute( + mutation, + { + "input": { + "flashEnvironmentId": environment_id, + "flashBuildId": build_id, + } + }, + ) + return data["deployBuildToEnvironment"] + + async def upload_tarball(self, upload_url: str, tar_path: str) -> None: + """put the build tarball to the presigned url.""" + timeout = aiohttp.ClientTimeout(total=600) + with open(tar_path, "rb") as f: + body = f.read() + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.put( + upload_url, + data=body, + headers={"Content-Type": "application/gzip"}, + ) as resp: + resp.raise_for_status() diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py new file mode 100644 index 000000000..289d0e42f --- /dev/null +++ b/runpod/apps/deploy.py @@ -0,0 +1,170 @@ +"""deploy pipeline: discovered apps -> manifest -> tarball -> activated build. + +per app: + 1. validate resources + 2. build the manifest from the app registry + 3. package the project into a tarball (honoring .runpodignore) + 4. get-or-create the flash app + environment + 5. upload via presigned url, finalize the build + 6. activate the build on the environment (coordinator provisions) +""" + +import fnmatch +import json +import logging +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .api import AppsApiClient +from .app import App +from .errors import ScheduleNotSupported +from .schedule import SCHEDULES_ENABLED +from .spec import ResourceKind + +log = logging.getLogger(__name__) + +MANIFEST_VERSION = 1 + +DEFAULT_IGNORES = [ + ".git", + ".venv", + "venv", + "__pycache__", + "*.pyc", + ".runpod", + ".flash", + "node_modules", + ".DS_Store", + "*.tar.gz", +] + + +@dataclass +class DeployResult: + app_name: str + build_id: str + environment_id: str + resources: List[str] + + +def build_manifest(app: App, project_root: Path) -> Dict[str, Any]: + """serialize an app's resources for the coordinator.""" + resources = [] + for handle in app.resources.values(): + spec = handle.spec + if spec.schedule and not SCHEDULES_ENABLED: + raise ScheduleNotSupported() + entry = spec.to_manifest() + fn = getattr(handle, "_fn", None) or getattr(handle, "__wrapped__", None) + if fn is not None: + module = getattr(fn, "__module__", "") + entry["module"] = module + entry["qualname"] = getattr(fn, "__qualname__", fn.__name__) + resources.append(entry) + return { + "version": MANIFEST_VERSION, + "app": app.name, + "resources": resources, + } + + +def _load_ignores(project_root: Path) -> List[str]: + patterns = list(DEFAULT_IGNORES) + ignore_file = project_root / ".runpodignore" + if ignore_file.exists(): + for line in ignore_file.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + patterns.append(line) + return patterns + + +def _is_ignored(rel_path: str, patterns: List[str]) -> bool: + parts = rel_path.split("/") + for pattern in patterns: + if fnmatch.fnmatch(rel_path, pattern): + return True + if any(fnmatch.fnmatch(part, pattern) for part in parts): + return True + return False + + +def package_project( + project_root: Path, + manifest: Dict[str, Any], + output: Optional[Path] = None, +) -> Path: + """tar the project plus the manifest into a build artifact.""" + if output is None: + output = Path(tempfile.mkdtemp()) / "artifact.tar.gz" + + patterns = _load_ignores(project_root) + + with tarfile.open(output, "w:gz") as tar: + for path in sorted(project_root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(project_root).as_posix() + if _is_ignored(rel, patterns): + continue + tar.add(path, arcname=rel) + + manifest_bytes = json.dumps(manifest, indent=2).encode() + info = tarfile.TarInfo(name="runpod_manifest.json") + info.size = len(manifest_bytes) + import io + + tar.addfile(info, io.BytesIO(manifest_bytes)) + + return output + + +async def deploy_app( + app: App, + project_root: Path, + *, + env_name: Optional[str] = None, + api: Optional[AppsApiClient] = None, +) -> DeployResult: + """run the full deploy pipeline for one app.""" + client = api or AppsApiClient() + env_name = env_name or app.env + + manifest = build_manifest(app, project_root) + tar_path = package_project(project_root, manifest) + tar_size = tar_path.stat().st_size + log.info("packaged %s (%d bytes)", app.name, tar_size) + + remote_app = await client.get_app_by_name(app.name) + if remote_app is None: + remote_app = await client.create_app(app.name) + log.info("created app %s (%s)", app.name, remote_app["id"]) + app_id = remote_app["id"] + + environments = { + e["name"]: e for e in remote_app.get("flashEnvironments") or [] + } + environment = environments.get(env_name) + if environment is None: + environment = await client.create_environment(app_id, env_name) + log.info("created environment %s (%s)", env_name, environment["id"]) + + upload = await client.prepare_artifact_upload(app_id, tar_size) + await client.upload_tarball(upload["uploadUrl"], str(tar_path)) + build = await client.finalize_artifact_upload( + app_id, upload["objectKey"], manifest + ) + log.info("uploaded build %s", build["id"]) + + await client.deploy_build(environment["id"], build["id"]) + log.info("activated build %s on %s/%s", build["id"], app.name, env_name) + + return DeployResult( + app_name=app.name, + build_id=build["id"], + environment_id=environment["id"], + resources=[r["name"] for r in manifest["resources"]], + ) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py new file mode 100644 index 000000000..5fa13c501 --- /dev/null +++ b/runpod/apps/dev.py @@ -0,0 +1,144 @@ +"""dev session: ephemeral live endpoints for `rp dev`. + +lifecycle: get-or-create endpoints named dev-{app}-{resource} on the +generic worker images (adopting leftovers from a killed session), run +the local entrypoint, and delete every session endpoint on exit. the +api is the only source of truth; nothing is persisted locally. +""" + +import asyncio +import logging +import sys +from typing import Dict, List, Optional, Union + +from .api import AppsApiClient +from .app import App +from .handles import ApiHandle, FunctionHandle +from .spec import ResourceKind, ResourceSpec +from .targets import LiveTarget + +log = logging.getLogger(__name__) + +DEV_PREFIX = "dev" + +# worker runtime images by (kind, is_cpu); tag pinned via env override +DEFAULT_IMAGES = { + ("queue", False): "runpod/flash:py3.12-latest", + ("queue", True): "runpod/flash-cpu:py3.12-latest", + ("api", False): "runpod/flash-lb:py3.12-latest", + ("api", True): "runpod/flash-lb-cpu:py3.12-latest", +} + + +def dev_endpoint_name(app_name: str, resource_name: str) -> str: + return f"{DEV_PREFIX}-{app_name}-{resource_name}" + + +def _image_for(spec: ResourceSpec) -> str: + import os + + override = os.getenv("RUNPOD_DEV_IMAGE") + if override: + return override + key = (spec.kind.value, spec.is_cpu) + image = DEFAULT_IMAGES.get(key) + if image is None: + raise ValueError(f"no dev image for resource kind {spec.kind.value}") + return image + + +def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: + """build the saveEndpoint payload for a live dev endpoint.""" + payload: Dict = { + "name": dev_endpoint_name(app.name, spec.name), + "workersMin": spec.workers[0], + "workersMax": spec.workers[1], + "idleTimeout": spec.idle_timeout, + "scalerType": ( + "REQUEST_COUNT" if spec.kind is ResourceKind.API else "QUEUE_DELAY" + ), + "scalerValue": 4, + "template": { + "name": f"{dev_endpoint_name(app.name, spec.name)}-template", + "imageName": _image_for(spec), + "containerDiskInGb": 10, + "isServerless": True, + }, + } + if spec.kind is ResourceKind.API: + payload["type"] = "LB" + if spec.is_cpu: + payload["instanceIds"] = spec.cpu + else: + payload["gpuIds"] = ",".join(spec.gpu or ["any"]) + payload["gpuCount"] = spec.gpu_count + if spec.env: + payload["template"]["env"] = [ + {"key": k, "value": v} for k, v in spec.env.items() + ] + return payload + + +class DevSession: + """owns the live endpoints for one `rp dev` invocation.""" + + def __init__(self, apps: List[App], api: Optional[AppsApiClient] = None): + self.apps = apps + self.api = api or AppsApiClient() + # endpoint ids created or adopted by this session + self._endpoint_ids: List[str] = [] + + def _provisionable( + self, app: App + ) -> List[Union[FunctionHandle, ApiHandle]]: + """endpoints only; tasks have no standing infra to manage.""" + return [ + h + for h in app.resources.values() + if h.spec.kind in (ResourceKind.QUEUE, ResourceKind.API) + ] + + async def start(self) -> None: + """provision (or adopt) a live endpoint per queue/api resource and + register the targets on each app.""" + existing = {e["name"]: e for e in await self.api.list_my_endpoints()} + + for app in self.apps: + for handle in self._provisionable(app): + spec = handle.spec + name = dev_endpoint_name(app.name, spec.name) + + found = existing.get(name) + if found: + endpoint_id = found["id"] + log.info("adopted existing dev endpoint %s (%s)", name, endpoint_id) + else: + result = await self.api.save_endpoint( + _endpoint_input(app, spec) + ) + endpoint_id = result["id"] + log.info("provisioned dev endpoint %s (%s)", name, endpoint_id) + + self._endpoint_ids.append(endpoint_id) + app._dev_targets[spec.name] = LiveTarget(endpoint_id) + + async def stop(self) -> None: + """delete every endpoint this session created or adopted.""" + for endpoint_id in self._endpoint_ids: + try: + await self.api.delete_endpoint(endpoint_id) + log.info("deleted dev endpoint %s", endpoint_id) + except Exception as exc: + log.warning( + "failed to delete dev endpoint %s: %s", endpoint_id, exc + ) + self._endpoint_ids.clear() + for app in self.apps: + app._dev_targets.clear() + + async def __aenter__(self) -> "DevSession": + await self.start() + return self + + async def __aexit__(self, *exc_info) -> None: + await self.stop() diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py new file mode 100644 index 000000000..080e91a19 --- /dev/null +++ b/runpod/apps/discovery.py @@ -0,0 +1,100 @@ +"""module discovery for `rp deploy` and `rp dev`. + +imports target modules under __name__ != "__main__" (so main guards +never run) and collects App instances from the registry. a per-module +import timeout guards against module-level code that blocks. +""" + +import importlib.util +import logging +import sys +import threading +from pathlib import Path +from typing import List + +from .app import App, get_registered_apps + +log = logging.getLogger(__name__) + +IMPORT_TIMEOUT_SECONDS = 30 + +_SKIP_DIRS = { + ".git", + ".venv", + "venv", + "node_modules", + "__pycache__", + ".runpod", + ".flash", + "build", + "dist", +} + + +class DiscoveryError(Exception): + """a module failed to import during discovery.""" + + +def _python_files(target: Path) -> List[Path]: + if target.is_file(): + if target.suffix != ".py": + raise DiscoveryError(f"{target} is not a python file") + return [target] + files = [] + for path in sorted(target.rglob("*.py")): + if any(part in _SKIP_DIRS for part in path.parts): + continue + files.append(path) + return files + + +def _import_module(path: Path) -> None: + """import one file as a uniquely-named module, never as __main__.""" + module_name = f"_runpod_discovered_{path.stem}_{abs(hash(str(path)))}" + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise DiscoveryError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + + error: List[BaseException] = [] + + def run() -> None: + try: + spec.loader.exec_module(module) + except BaseException as exc: # noqa: BLE001 - reported to caller + error.append(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(IMPORT_TIMEOUT_SECONDS) + + if thread.is_alive(): + raise DiscoveryError( + f"importing {path} timed out after {IMPORT_TIMEOUT_SECONDS}s; " + f"module-level code must not block (guard it with " + f'`if __name__ == "__main__":` or runpod.is_local())' + ) + if error: + raise DiscoveryError(f"importing {path} failed: {error[0]}") from error[0] + + +def discover_apps(target: Path) -> List[App]: + """import all python files under target and return the apps they define.""" + before = set(id(a) for a in get_registered_apps()) + + sys_path_added = False + root = target if target.is_dir() else target.parent + root_str = str(root.resolve()) + if root_str not in sys.path: + sys.path.insert(0, root_str) + sys_path_added = True + + try: + for path in _python_files(target): + _import_module(path) + finally: + if sys_path_added: + sys.path.remove(root_str) + + return [a for a in get_registered_apps() if id(a) not in before] diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index d93144bda..1702f3fdd 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -24,7 +24,6 @@ def load_model(): ... # worker startup hook from .schedule import SCHEDULE_ATTR from .spec import ResourceKind, ResourceSpec, RouteSpec from .errors import InvalidResourceError, RemoteExecutionError -from .targets import args_to_input if TYPE_CHECKING: from .app import App @@ -103,12 +102,12 @@ async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: return result target = await self._app._resolve(self.spec) - payload = {"input": args_to_input(self._fn, args, kwargs)} + payload = target.build_payload(self._fn, self.spec, args, kwargs) return await target.invoke(payload) async def _spawn_async(self, *args: Any, **kwargs: Any) -> Job: target = await self._app._resolve(self.spec) - payload = {"input": args_to_input(self._fn, args, kwargs)} + payload = target.build_payload(self._fn, self.spec, args, kwargs) data = await target.submit(payload) return Job(data, self) diff --git a/runpod/apps/protocol.py b/runpod/apps/protocol.py new file mode 100644 index 000000000..bc9f34bbf --- /dev/null +++ b/runpod/apps/protocol.py @@ -0,0 +1,60 @@ +"""wire protocol for remote function execution. + +this is the contract between the sdk and the worker runtime images. +the worker repo imports these shapes from the runpod package so client +and worker can never drift. + +live mode ships function source with every request; deployed mode omits +the code (the worker unpacked the build and resolves the function by +name). args/kwargs are base64 cloudpickle unless serialization_format +is "json". +""" + +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List, Optional + +EXECUTION_FUNCTION = "function" + +FORMAT_CLOUDPICKLE = "cloudpickle" +FORMAT_JSON = "json" + + +@dataclass +class FunctionRequest: + """a request to execute one function on a worker.""" + + function_name: str + function_code: Optional[str] = None + args: List[Any] = field(default_factory=list) + kwargs: Dict[str, Any] = field(default_factory=dict) + dependencies: Optional[List[str]] = None + system_dependencies: Optional[List[str]] = None + execution_type: str = EXECUTION_FUNCTION + accelerate_downloads: bool = True + serialization_format: str = FORMAT_CLOUDPICKLE + + def to_input(self) -> Dict[str, Any]: + """serialize as the job input dict.""" + data = asdict(self) + return {k: v for k, v in data.items() if v is not None} + + +@dataclass +class FunctionResponse: + """the worker's result for one function execution.""" + + success: bool + result: Optional[str] = None + json_result: Any = None + error: Optional[str] = None + stdout: Optional[str] = None + + @classmethod + def from_output(cls, output: Dict[str, Any]) -> "FunctionResponse": + return cls( + success=output.get("success", False), + result=output.get("result"), + json_result=output.get("json_result"), + error=output.get("error"), + stdout=output.get("stdout"), + ) diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py new file mode 100644 index 000000000..216bb1189 --- /dev/null +++ b/runpod/apps/serialization.py @@ -0,0 +1,57 @@ +"""argument and result serialization for remote execution. + +args/kwargs cross the wire as base64-encoded cloudpickle strings, the +format the worker runtime images expect. function source is extracted +with decorators stripped so the worker can exec it standalone. +""" + +import ast +import base64 +import inspect +import textwrap +from typing import Any, Callable, Dict, List + +import cloudpickle + + +def serialize_arg(arg: Any) -> str: + return base64.b64encode(cloudpickle.dumps(arg)).decode("utf-8") + + +def serialize_args(args: tuple) -> List[str]: + return [serialize_arg(a) for a in args] + + +def serialize_kwargs(kwargs: dict) -> Dict[str, str]: + return {k: serialize_arg(v) for k, v in kwargs.items()} + + +def deserialize_result(result_b64: str) -> Any: + return cloudpickle.loads(base64.b64decode(result_b64)) + + +def get_function_source(fn: Callable) -> str: + """extract a function's source with decorators stripped. + + the worker execs this source into a fresh namespace and calls the + function by name, so the emitted code must be a bare def. + """ + fn = inspect.unwrap(fn) + source = textwrap.dedent(inspect.getsource(fn)) + + module = ast.parse(source) + fn_def = None + for node in ast.walk(module): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == fn.__name__ + ): + fn_def = node + break + if fn_def is None: + raise ValueError(f"could not find function definition for {fn.__name__}") + + # slice from the def line, dropping decorator lines above it + lines = source.split("\n") + fn_lines = lines[fn_def.lineno - 1 :] + return textwrap.dedent("\n".join(fn_lines)) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 384c18b5d..a6113df07 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -15,6 +15,7 @@ from ..user_agent import USER_AGENT from .errors import EndpointNotFound, RemoteExecutionError +from .protocol import FunctionRequest, FunctionResponse from .spec import ResourceSpec # the sentinel pseudo-endpoint id; ai-api resolves the real endpoint from @@ -96,7 +97,20 @@ def unwrap_job_output(data: Dict[str, Any]) -> Any: class InvocationTarget(ABC): - """one resolved destination for remote calls.""" + """one resolved destination for remote calls. + + each target owns its payload shape: sentinel targets send plain + kwargs (the deployed worker resolves functions from the unpacked + build), live targets send the full FunctionRequest with source. + """ + + def build_payload( + self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict + ) -> Dict[str, Any]: + return {"input": args_to_input(fn, args, kwargs)} + + def unwrap(self, data: Dict[str, Any]) -> Any: + return unwrap_job_output(data) @abstractmethod async def invoke(self, payload: Dict[str, Any], *, timeout: float) -> Any: @@ -213,12 +227,46 @@ class LiveTarget(InvocationTarget): def __init__(self, endpoint_id: str): self.endpoint_id = endpoint_id + def build_payload( + self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict + ) -> Dict[str, Any]: + from .serialization import ( + get_function_source, + serialize_args, + serialize_kwargs, + ) + + request = FunctionRequest( + function_name=fn.__name__, + function_code=get_function_source(fn), + args=serialize_args(args), + kwargs=serialize_kwargs(kwargs), + dependencies=spec.dependencies, + system_dependencies=spec.system_dependencies, + ) + return {"input": request.to_input()} + + def unwrap(self, data: Dict[str, Any]) -> Any: + output = unwrap_job_output(data) + if isinstance(output, dict) and "success" in output: + response = FunctionResponse.from_output(output) + if not response.success: + raise RemoteExecutionError( + f"remote execution failed: {response.error or 'unknown'}" + ) + if response.result is not None: + from .serialization import deserialize_result + + return deserialize_result(response.result) + return response.json_result + return output + async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: url = f"{_endpoint_url_base()}/{self.endpoint_id}/runsync" data = await _post_json(url, payload, _headers(), timeout) - return unwrap_job_output(data) + return self.unwrap(data) async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: url = f"{_endpoint_url_base()}/{self.endpoint_id}/run" diff --git a/runpod/cli/entry.py b/runpod/cli/entry.py index 6fefa2ddf..4794b05e8 100644 --- a/runpod/cli/entry.py +++ b/runpod/cli/entry.py @@ -9,7 +9,6 @@ from .groups.config.commands import config_wizard from .groups.exec.commands import exec_cli from .groups.pod.commands import pod_cli -from .groups.project.commands import project_cli from .groups.ssh.commands import ssh_cli @@ -23,4 +22,3 @@ def runpod_cli(): runpod_cli.add_command(ssh_cli) # runpod ssh runpod_cli.add_command(pod_cli) # runpod pod runpod_cli.add_command(exec_cli) # runpod exec -runpod_cli.add_command(project_cli) # runpod project diff --git a/runpod/cli/groups/project/__init__.py b/runpod/cli/groups/project/__init__.py deleted file mode 100644 index 0f41264ed..000000000 --- a/runpod/cli/groups/project/__init__.py +++ /dev/null @@ -1 +0,0 @@ -""" Allows project import """ diff --git a/runpod/cli/groups/project/commands.py b/runpod/cli/groups/project/commands.py deleted file mode 100644 index b5df4a3f1..000000000 --- a/runpod/cli/groups/project/commands.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -Runpod | CLI | Project | Commands -""" - -import os -import sys - -import click -from InquirerPy import prompt as cli_select - -from runpod import get_user - -from .functions import create_new_project, create_project_endpoint, start_project -from .helpers import validate_project_name - - -@click.group("project", help="Create, deploy, and manage Runpod projects.") -def project_cli(): - """Create and deploy projects on Runpod.""" - - -# -------------------------------- New Project ------------------------------- # -@project_cli.command("new") -@click.option( - "--name", "-n", "project_name", type=str, default=None, help="The project name." -) -@click.option( - "--type", - "-t", - "model_type", - type=click.Choice(["llama2"], case_sensitive=False), - default=None, - help="The type of Hugging Face model.", -) -@click.option( - "--model", - "-m", - "model_name", - type=str, - default=None, - help="The name of the Hugging Face model. (e.g. meta-llama/Llama-2-7b)", -) -@click.option("--init", "-i", "init_current_dir", is_flag=True, default=False) -def new_project_wizard(project_name, model_type, model_name, init_current_dir): - """Create a new project.""" - network_volumes = get_user()["networkVolumes"] - if len(network_volumes) == 0: - click.echo("You do not have any network volumes.") - click.echo( - "Please create a network volume (https://console.runpod.io/user/storage) and try again." - ) # pylint: disable=line-too-long - sys.exit(1) - - click.echo("Creating a new project...") - - if init_current_dir: - project_name = os.path.basename(os.getcwd()) - - if project_name is None: - project_name = click.prompt(" > Enter the project name", type=str) - - validate_project_name(project_name) - - def print_net_vol(vol): - return { - "name": f"{vol['id']}: {vol['name']} ({vol['size']} GB, {vol['dataCenterId']})", - "value": vol["id"], - } - - network_volumes = list(map(print_net_vol, network_volumes)) - questions = [ - { - "type": "rawlist", - "name": "volume-id", - "qmark": "", - "amark": "", - "message": " > Select a Network Volume:", - "choices": network_volumes, - } - ] - runpod_volume_id = cli_select(questions)["volume-id"] - - cuda_version = click.prompt( - " > Select a CUDA version, or press enter to use the default", - type=click.Choice(["11.1.1", "11.8.0", "12.1.0"], case_sensitive=False), - default="11.8.0", - ) - - python_version = click.prompt( - " > Select a Python version, or press enter to use the default", - type=click.Choice(["3.8", "3.9", "3.10", "3.11"], case_sensitive=False), - default="3.10", - ) - - click.echo("") - click.echo("Project Summary:") - - click.echo(f" - Project Name: {project_name}") - click.echo(f" - Runpod Network Storage ID: {runpod_volume_id}") - click.echo(f" - CUDA Version: {cuda_version}") - click.echo(f" - Python Version: {python_version}") - - click.echo("") - click.echo("The project will be created in the current directory.") - click.confirm("Do you want to continue?", abort=True) - - create_new_project( - project_name, - runpod_volume_id, - cuda_version, - python_version, - model_type, - model_name, - init_current_dir, - ) # pylint: disable=too-many-arguments - - click.echo(f"Project {project_name} created successfully!") - click.echo("") - click.echo( - "From your project root run `runpod project start` to start a development pod." - ) - - -# ------------------------------- Start Project ------------------------------ # -@project_cli.command("start") -def start_project_pod(): - """ - Start the project development pod from runpod.toml - """ - click.echo( - "Starting the project will create a development pod on Runpod, if one does not already exist." - ) # pylint: disable=line-too-long - click.echo( - " - You will be charged based on the GPU type specified in runpod.toml." - ) - click.echo(" - When you are finished close the connection with CTRL+C.") - click.echo("") - click.confirm("Do you want to continue?", abort=True) - - click.echo("Starting project development pod...") - - start_project() - - -# ------------------------------ Deploy Project ------------------------------ # -@project_cli.command("deploy") -def deploy_project(): - """Deploy the project to Runpod.""" - click.echo("Deploying project...") - - endpoint_id = create_project_endpoint() - - click.echo(f"Project deployed successfully! Endpoint ID: {endpoint_id}") - click.echo("The following urls are available:") - click.echo(f" - https://api.runpod.ai/v2/{endpoint_id}/runsync") - click.echo(f" - https://api.runpod.ai/v2/{endpoint_id}/run") - click.echo(f" - https://api.runpod.ai/v2/{endpoint_id}/health") diff --git a/runpod/cli/groups/project/functions.py b/runpod/cli/groups/project/functions.py deleted file mode 100644 index 4fe011573..000000000 --- a/runpod/cli/groups/project/functions.py +++ /dev/null @@ -1,429 +0,0 @@ -""" -Runpod | CLI | Project | Functions -""" - -import os -import sys -import uuid -from datetime import datetime - -import tomlkit -from tomlkit import comment, document, nl, table - -from runpod import ( - __version__, - create_endpoint, - create_template, - get_pod, - update_endpoint_template, -) -from runpod.cli import BASE_DOCKER_IMAGE, ENV_VARS, GPU_TYPES -from runpod.cli.utils.ssh_cmd import SSHConnection - -from ...utils.rp_sync import sync_directory -from .helpers import ( - attempt_pod_launch, - copy_template_files, - get_project_endpoint, - get_project_pod, - load_project_config, -) - -STARTER_TEMPLATES = os.path.join(os.path.dirname(__file__), "starter_templates") - - -def _launch_dev_pod(): - """Launch a development pod.""" - config = load_project_config() # Load runpod.toml - - print("Deploying development pod on Runpod...") - - # Prepare the environment variables - environment_variables = {"RUNPOD_PROJECT_ID": config["project"]["uuid"]} - for variable in config["project"].get("env_vars", {}): - environment_variables[variable] = config["project"]["env_vars"][variable] - - # Prepare the GPU types - selected_gpu_types = config["project"].get("gpu_types", []) - if config["project"].get("gpu", None): - selected_gpu_types.append(config["project"]["gpu"]) - - # Attempt to launch a pod with the given configuration - new_pod = attempt_pod_launch(config, environment_variables) - if new_pod is None: - print( - "Selected GPU types unavailable, try again later or use a different type." - ) - return None - - print("Waiting for pod to come online... ", end="") - sys.stdout.flush() - - # Wait for the pod to come online - while ( - new_pod.get("desiredStatus", None) != "RUNNING" - or new_pod.get("runtime") is None - ): - new_pod = get_pod(new_pod["id"]) - - project_pod_id = new_pod["id"] - - print( - f"Project {config['project']['name']} pod ({project_pod_id}) created.", - end="\n\n", - ) - return project_pod_id - - -# -------------------------------- New Project ------------------------------- # -def create_new_project( - project_name, - runpod_volume_id, - cuda_version, - python_version, # pylint: disable=too-many-locals, too-many-arguments, too-many-statements - model_type=None, - model_name=None, - init_current_dir=False, -): - """Create a new project.""" - if not init_current_dir: - project_folder = os.path.join(os.getcwd(), project_name) - if not os.path.exists(project_folder): - os.makedirs(project_folder) - - if model_type is None: - model_type = "default" - - template_dir = os.path.join(STARTER_TEMPLATES, model_type) - - copy_template_files(template_dir, project_folder) - - # Replace placeholders in requirements.txt - requirements_path = os.path.join(project_folder, "builder/requirements.txt") - with open(requirements_path, "r", encoding="utf-8") as requirements_file: - requirements_content = requirements_file.read() - - if "dev" in __version__: - requirements_content = requirements_content.replace( - "<>", "git+https://github.com/runpod/runpod-python.git" - ) - else: - requirements_content = requirements_content.replace( - "<>", f"runpod=={__version__}" - ) - - with open(requirements_path, "w", encoding="utf-8") as requirements_file: - requirements_file.write(requirements_content) - - # If there's a model_name, replace placeholders in handler.py - if model_name: - handler_path = os.path.join(project_name, "src/handler.py") - with open(handler_path, "r", encoding="utf-8") as file: - handler_content = file.read() - handler_content = handler_content.replace("<>", model_name) - - with open(handler_path, "w", encoding="utf-8") as file: - file.write(handler_content) - else: - project_folder = os.getcwd() - - project_uuid = str(uuid.uuid4())[:8] - - toml_config = document() - toml_config.add(comment("Runpod Project Configuration")) - toml_config.add(nl()) - toml_config.add("title", project_name) - - project_table = table() - project_table.add("uuid", project_uuid) - project_table.add("name", project_name) - project_table.add("base_image", BASE_DOCKER_IMAGE.format(cuda_version=cuda_version)) - project_table.add("gpu_types", GPU_TYPES) - project_table.add("gpu_count", 1) - project_table.add("storage_id", runpod_volume_id) - project_table.add("volume_mount_path", "/runpod-volume") - project_table.add("ports", "8080/http, 22/tcp") - project_table.add("container_disk_size_gb", 10) - project_table.add("env_vars", ENV_VARS) - toml_config.add("project", project_table) - - template_table = table() - template_table.add("model_type", str(model_type)) - template_table.add("model_name", str(model_name)) - toml_config.add("template", template_table) - - runtime_table = table() - runtime_table.add("python_version", python_version) - runtime_table.add("handler_path", "src/handler.py") - runtime_table.add("requirements_path", "builder/requirements.txt") - toml_config.add("runtime", runtime_table) - - with open( - os.path.join(project_folder, "runpod.toml"), "w", encoding="UTF-8" - ) as config_file: - tomlkit.dump(toml_config, config_file) - - -# ------------------------------- Start Project ------------------------------ # -def start_project(): # pylint: disable=too-many-locals, too-many-branches - """ - Start the project development environment from runpod.toml - - Check if the project pod already exists. - - - If the project pod does not exist: - # SSH into the pod and create a project folder within the volume - # Create a folder in the volume for the project that matches the project uuid - # Create a folder in the project folder that matches the project name - # Copy the project files into the project folder - # crate a virtual environment using the python version specified in the project config - # install the requirements.txt file - - - If the project pod does exist: - - """ - config = load_project_config() # Load runpod.toml - - for config_item in config["project"]: - print(f' - {config_item}: {config["project"][config_item]}') - print("") - - project_pod_id = get_project_pod(config["project"]["uuid"]) - - # Check if the project pod already exists, if not create it. - if not project_pod_id: - project_pod_id = _launch_dev_pod() - - if project_pod_id is None: - return - - with SSHConnection(project_pod_id) as ssh_conn: - - project_path_uuid = ( - f'{config["project"]["volume_mount_path"]}/{config["project"]["uuid"]}' - ) - project_path_uuid_dev = os.path.join(project_path_uuid, "dev") - project_path_uuid_prod = os.path.join(project_path_uuid, "prod") - remote_project_path = os.path.join( - project_path_uuid_dev, config["project"]["name"] - ) - - # Create the project folder on the pod - print( - f"Checking pod project folder: {remote_project_path} on pod {project_pod_id}" - ) - ssh_conn.run_commands( - [f"mkdir -p {remote_project_path} {project_path_uuid_prod}"] - ) - - # Copy local files to the pod project folder - print(f"Syncing files to pod {project_pod_id}") - ssh_conn.rsync(os.getcwd(), project_path_uuid_dev) - - # Create the virtual environment - venv_path = os.path.join(project_path_uuid_dev, "venv") - print( - f"Activating Python virtual environment: {venv_path} on pod {project_pod_id}" - ) - commands = [ - f'python{config["runtime"]["python_version"]} -m virtualenv {venv_path}', - f"source {venv_path}/bin/activate &&" - f"cd {remote_project_path} &&" - "python -m pip install --upgrade pip &&" - f'python -m pip install -v --requirement {config["runtime"]["requirements_path"]}', - ] - ssh_conn.run_commands(commands) - - # Start the watcher and then start the API development server - sync_directory(ssh_conn, os.getcwd(), project_path_uuid_dev) - - project_name = config["project"]["name"] - pip_req_path = os.path.join( - remote_project_path, config["runtime"]["requirements_path"] - ) - handler_path = os.path.join( - remote_project_path, config["runtime"]["handler_path"] - ) - - launch_api_server = [ - f""" - pkill inotify - - function force_kill {{ - kill $1 2>/dev/null - sleep 1 - - if ps -p $1 > /dev/null; then - echo "Graceful kill failed, attempting SIGKILL..." - kill -9 $1 2>/dev/null - sleep 1 - - if ps -p $1 > /dev/null; then - echo "Failed to kill process with PID: $1" - exit 1 - else - echo "Killed process with PID: $1 using SIGKILL" - fi - - else - echo "Killed process with PID: $1" - fi - }} - - function cleanup {{ - echo "Cleaning up..." - force_kill $last_pid - }} - - trap cleanup EXIT SIGINT - - if source {project_path_uuid_dev}/venv/bin/activate; then - echo -e "- Activated virtual environment." - else - echo "Failed to activate virtual environment." - exit 1 - fi - - if cd {project_path_uuid_dev}/{project_name}; then - echo -e "- Changed to project directory." - else - echo "Failed to change directory." - exit 1 - fi - - exclude_pattern='(__pycache__|\\.pyc$)' - function update_exclude_pattern {{ - exclude_pattern='(__pycache__|\\.pyc$)' - if [[ -f .runpodignore ]]; then - while IFS= read -r line; do - line=$(echo "$line" | tr -d '[:space:]') - [[ "$line" =~ ^#.*$ || -z "$line" ]] && continue # Skip comments and empty lines - exclude_pattern="${{exclude_pattern}}|(${{line}})" - done < .runpodignore - echo -e "- Ignoring files matching pattern: $exclude_pattern" - fi - }} - update_exclude_pattern - - # Start the API server in the background, and save the PID - python {handler_path} --rp_serve_api --rp_api_host="0.0.0.0" --rp_api_port=8080 --rp_api_concurrency=1 & - last_pid=$! - - echo -e "- Started API server with PID: $last_pid" && echo "" - echo "Connect to the API server at:" - echo "> https://$RUNPOD_POD_ID-8080.proxy.runpod.net" && echo "" - - while true; do - if changed_file=$(inotifywait -q -r -e modify,create,delete --exclude "$exclude_pattern" {remote_project_path} --format '%w%f'); then - echo "Detected changes in: $changed_file" - else - echo "Failed to detect changes." - exit 1 - fi - - force_kill $last_pid - - if [[ $changed_file == *"requirements"* ]]; then - echo "Installing new requirements..." - python -m pip install --upgrade pip && python -m pip install -r {pip_req_path} - fi - - if [[ $changed_file == *".runpodignore"* ]]; then - update_exclude_pattern - fi - - python {handler_path} --rp_serve_api --rp_api_host="0.0.0.0" --rp_api_port=8080 --rp_api_concurrency=1 & - last_pid=$! - - echo "Restarted API server with PID: $last_pid" - done - """ - ] - - print("") - print("Starting project development endpoint...") - ssh_conn.run_commands(launch_api_server) - - -# ------------------------------ Deploy Project ------------------------------ # -def create_project_endpoint(): - """Create a project endpoint. - - Move code in dev to prod folder - - TODO: git commit the diff from current state to new state - - Create a serverless template for the project - - Create a new endpoint using the template - """ - config = load_project_config() - project_pod_id = get_project_pod(config["project"]["uuid"]) - - # Check if the project pod already exists, if not create it. - if not project_pod_id: - project_pod_id = _launch_dev_pod() - - if project_pod_id is None: - return None - - with SSHConnection(project_pod_id) as ssh_conn: - project_path_uuid = ( - f'{config["project"]["volume_mount_path"]}/{config["project"]["uuid"]}' - ) - project_path_uuid_prod = os.path.join(project_path_uuid, "prod") - remote_project_path = os.path.join( - project_path_uuid_prod, config["project"]["name"] - ) - - # Copy local files to the pod project folder - ssh_conn.run_commands([f"mkdir -p {remote_project_path}"]) - print(f"Syncing files to pod {project_pod_id} prod") - ssh_conn.rsync(os.getcwd(), project_path_uuid_prod) - - # Create the virtual environment - venv_path = os.path.join(project_path_uuid_prod, "venv") - print( - f"Activating Python virtual environment: {venv_path} on pod {project_pod_id}" - ) - commands = [ - f'python{config["runtime"]["python_version"]} -m venv {venv_path}', - f"source {venv_path}/bin/activate &&" - f"cd {remote_project_path} &&" - "python -m pip install --upgrade pip &&" - f'python -m pip install -v --requirement {config["runtime"]["requirements_path"]}', - ] - ssh_conn.run_commands(commands) - ssh_conn.close() - - environment_variables = {} - for variable in config["project"]["env_vars"]: - environment_variables[variable] = config["project"]["env_vars"][variable] - - # Construct the docker start command - activate_cmd = ( - f'. /runpod-volume/{config["project"]["uuid"]}/prod/venv/bin/activate' - ) - python_cmd = f'python -u /runpod-volume/{config["project"]["uuid"]}/prod/{config["project"]["name"]}/{config["runtime"]["handler_path"]}' # pylint: disable=line-too-long - docker_start_cmd = 'bash -c "' + activate_cmd + " && " + python_cmd + '"' - - project_endpoint_template = create_template( - name=f'{config["project"]["name"]}-endpoint | {config["project"]["uuid"]} | {datetime.now()}', # pylint: disable=line-too-long - image_name=config["project"]["base_image"], - container_disk_in_gb=config["project"]["container_disk_size_gb"], - docker_start_cmd=docker_start_cmd, - env=environment_variables, - is_serverless=True, - ) - - deployed_endpoint = get_project_endpoint(config["project"]["uuid"]) - if not deployed_endpoint: - deployed_endpoint = create_endpoint( - name=f'{config["project"]["name"]}-endpoint | {config["project"]["uuid"]}', - template_id=project_endpoint_template["id"], - network_volume_id=config["project"]["storage_id"], - ) - else: - deployed_endpoint = update_endpoint_template( - endpoint_id=deployed_endpoint["id"], - template_id=project_endpoint_template["id"], - ) - - # does user want to tear down and recreate workers immediately? - - return deployed_endpoint["id"] diff --git a/runpod/cli/groups/project/helpers.py b/runpod/cli/groups/project/helpers.py deleted file mode 100644 index be7c7d98d..000000000 --- a/runpod/cli/groups/project/helpers.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Helper functions for project group commands.""" - -import os -import re -import shutil - -import click -import tomlkit - -from runpod import create_pod -from runpod import error as rp_error -from runpod import get_endpoints, get_pods - - -def validate_project_name(name): - """ - Validate the project name. - """ - match = re.search(r"[<>:\"/\\|?*\s]", name) - if match: - raise click.BadParameter( - f"Project name contains an invalid character: '{match.group()}'." - ) - return name - - -def get_project_pod(project_id: str): - """Check if a project pod exists. - Return the pod_id if it exists, else return None. - """ - for pod in get_pods(): - if project_id in pod["name"]: - return pod["id"] - - return None - - -def get_project_endpoint(project_id: str): - """Check if a project endpoint exists. - Return the endpoint if it exists, else return None. - """ - for endpoint in get_endpoints(): - if project_id in endpoint["name"]: - return endpoint - - return None - - -def copy_template_files(template_dir, destination): - """Copy the template files to the destination directory.""" - for item in os.listdir(template_dir): - source_item = os.path.join(template_dir, item) - destination_item = os.path.join(destination, item) - if os.path.isdir(source_item): - shutil.copytree(source_item, destination_item) - else: - shutil.copy2(source_item, destination_item) - - -def attempt_pod_launch(config, environment_variables): - """Attempt to launch a pod with the given configuration.""" - for gpu_type in config["project"].get("gpu_types", []): - print(f"Trying to get a pod with {gpu_type}... ", end="") - try: - created_pod = create_pod( - f'{config["project"]["name"]}-dev ({config["project"]["uuid"]})', - config["project"]["base_image"], - gpu_type, - gpu_count=int(config["project"]["gpu_count"]), - support_public_ip=True, - ports=f'{config["project"]["ports"]}', - network_volume_id=f'{config["project"]["storage_id"]}', - volume_mount_path=f'{config["project"]["volume_mount_path"]}', - container_disk_in_gb=int(config["project"]["container_disk_size_gb"]), - env=environment_variables, - ) - print("Success!") - return created_pod - except rp_error.QueryError: - print("Unavailable.") - return None - - -def load_project_config(): - """Load the project config file.""" - project_config_file = os.path.join(os.getcwd(), "runpod.toml") - if not os.path.exists(project_config_file): - raise FileNotFoundError("runpod.toml not found in the current directory.") - with open(project_config_file, "r", encoding="UTF-8") as config_file: - return tomlkit.load(config_file) diff --git a/runpod/cli/groups/project/starter_templates/default/.runpodignore b/runpod/cli/groups/project/starter_templates/default/.runpodignore deleted file mode 100644 index fef4dcc03..000000000 --- a/runpod/cli/groups/project/starter_templates/default/.runpodignore +++ /dev/null @@ -1,2 +0,0 @@ -# Similar to .gitignore -# Matches will not be synced to the development pod or cause the development pod to reload. diff --git a/runpod/cli/groups/project/starter_templates/default/builder/requirements.txt b/runpod/cli/groups/project/starter_templates/default/builder/requirements.txt deleted file mode 100644 index 8fa962d6b..000000000 --- a/runpod/cli/groups/project/starter_templates/default/builder/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Required Python packages get listed here, one per line. -# Reccomended to lock the version number to avoid unexpected changes. - -# You can also install packages from a git repository, e.g.: -# git+https://github.com/runpod/runpod-python.git -# To learn more, see https://pip.pypa.io/en/stable/reference/requirements-file-format/ - -<> diff --git a/runpod/cli/groups/project/starter_templates/default/src/handler.py b/runpod/cli/groups/project/starter_templates/default/src/handler.py deleted file mode 100644 index bd26d7ec4..000000000 --- a/runpod/cli/groups/project/starter_templates/default/src/handler.py +++ /dev/null @@ -1,15 +0,0 @@ -""" A template for a handler file. """ - -import runpod - - -def handler(job): - """ - This is the handler function for the job. - """ - job_input = job["input"] - name = job_input.get("name", "World") - return f"Hello, {name}!" - - -runpod.serverless.start({"handler": handler}) diff --git a/runpod/cli/groups/project/starter_templates/llama2/builder/requirements.txt b/runpod/cli/groups/project/starter_templates/llama2/builder/requirements.txt deleted file mode 100644 index 5a94046a1..000000000 --- a/runpod/cli/groups/project/starter_templates/llama2/builder/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# List your python dependencies here. See https://pip.pypa.io/en/stable/user_guide/#requirements-files - -runpod==1.2.0 -transformers>=4.33.0 diff --git a/runpod/cli/groups/project/starter_templates/llama2/src/handler.py b/runpod/cli/groups/project/starter_templates/llama2/src/handler.py deleted file mode 100644 index 00048abc6..000000000 --- a/runpod/cli/groups/project/starter_templates/llama2/src/handler.py +++ /dev/null @@ -1,42 +0,0 @@ -""" A template for a Llama2 handler file. """ - -# pylint: skip-file -# ruff: noqa - - -from transformers import HfApi - -import runpod - -SELECTED_MODEL = "<>" - - -def get_model_framework(model_name): - api = HfApi() - model_files = api.model_info(model_name).files - - # Check the files associated with the model - if "pytorch_model.bin" in model_files: - return "PyTorch" - elif "tf_model.h5" in model_files: - return "TensorFlow" - else: - return "Unknown" - - -def prepare_inputs(text, **kwargs): - # Filter kwargs based on what the tokenizer accepts - filtered_args = {k: v for k, v in kwargs.items() if k in valid_args} - - inputs = tokenizer(text, return_tensors="pt", **filtered_args) - return inputs - - -def handle_request(text, **input_args): - inputs = prepare_inputs(text, **input_args) - with torch.no_grad(): - outputs = model(**inputs) - return process_outputs(outputs) - - -runpod.serverless.start({"handler": handle_request}) diff --git a/runpod/cli/utils/rp_sync.py b/runpod/cli/utils/rp_sync.py deleted file mode 100644 index fa7fe2909..000000000 --- a/runpod/cli/utils/rp_sync.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Watches a directory for changes and syncs them to a remote directory. -""" - -import threading -import time - -from watchdog.events import FileSystemEventHandler -from watchdog.observers.polling import PollingObserver as Observer - -from runpod.cli import STOP_EVENT - -from .rp_runpodignore import get_ignore_list, should_ignore - - -class WatcherHandler(FileSystemEventHandler): - """Watches a directory for changes and syncs them to a remote directory.""" - - def __init__(self, action_function, local_path): - self.action_function = action_function - self.local_path = local_path - - self.ignore_list = get_ignore_list() - self.debouncer = None - - def on_any_event(self, event): - """Called on any event.""" - if event.is_directory or should_ignore(event.src_path, self.ignore_list): - return - - if self.debouncer is not None: - self.debouncer.cancel() # Cancel any existing timer - - # Start a new timer that will call the action function after 1 second - self.debouncer = threading.Timer(0.5, self.action_function) - self.debouncer.start() - - -def start_watcher(action_function, local_path): - """ - Starts the watcher. - """ - event_handler = WatcherHandler(action_function, local_path) - observer = Observer() - observer.schedule(event_handler, local_path, recursive=True) - observer.start() - - try: - while not STOP_EVENT.is_set(): - time.sleep(0.5) - finally: - observer.stop() - observer.join() - - -def sync_directory(ssh_client, local_path, remote_path): - """ - Syncs a local directory to a remote directory. - """ - - def sync(): - print("Syncing files...") - ssh_client.rsync(local_path, remote_path, quiet=True) - - threading.Thread(target=start_watcher, daemon=True, args=(sync, local_path)).start() - - return sync # For testing purposes diff --git a/runpod/rp_cli/__init__.py b/runpod/rp_cli/__init__.py new file mode 100644 index 000000000..de9fc580d --- /dev/null +++ b/runpod/rp_cli/__init__.py @@ -0,0 +1 @@ +"""the `rp` command line interface.""" diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py new file mode 100644 index 000000000..94cde5dfe --- /dev/null +++ b/runpod/rp_cli/main.py @@ -0,0 +1,173 @@ +"""the `rp` cli: deploy, dev, login, plus mounted legacy groups. + +typer is the root; the existing click groups from runpod.cli mount +underneath so `rp pod ...`, `rp ssh ...`, etc keep working. +""" + +import asyncio +import os +import sys +from pathlib import Path +from typing import Optional + +import typer + +app = typer.Typer( + name="rp", + help="Runpod CLI: deploy and manage apps, endpoints, and pods.", + no_args_is_help=True, + pretty_exceptions_show_locals=False, +) + + +def _echo(message: str) -> None: + typer.echo(message) + + +def _fail(message: str) -> None: + typer.secho(message, fg=typer.colors.RED, err=True) + raise typer.Exit(1) + + +@app.command() +def deploy( + target: Optional[Path] = typer.Argument( + None, + help="Directory or file to deploy (defaults to current directory).", + ), + env: Optional[str] = typer.Option( + None, "--env", "-e", help="Target environment name." + ), +): + """package and deploy all apps found in TARGET.""" + from runpod.apps.deploy import deploy_app + from runpod.apps.discovery import DiscoveryError, discover_apps + + target = (target or Path.cwd()).resolve() + if not target.exists(): + _fail(f"{target} does not exist") + + project_root = target if target.is_dir() else target.parent + + _echo(f"discovering apps in {target} ...") + try: + apps = discover_apps(target) + except DiscoveryError as exc: + _fail(str(exc)) + + if not apps: + _fail( + "no runpod.App found. define one with:\n\n" + " from runpod import App\n" + ' app = App("my-app")\n' + ) + + for found in apps: + names = ", ".join(found.resources) or "(no resources)" + _echo(f"deploying app '{found.name}': {names}") + result = asyncio.run( + deploy_app(found, project_root, env_name=env) + ) + _echo( + f" build {result.build_id} active on " + f"{result.app_name}/{env or found.env}" + ) + _echo("done.") + + +@app.command() +def dev( + module: Path = typer.Argument(..., help="Python module to run."), +): + """start an interactive dev session for MODULE. + + provisions temporary live endpoints, runs the module's + @runpod.local_entrypoint, and deletes the endpoints on exit. + """ + from runpod.apps.dev import DevSession + from runpod.apps.discovery import DiscoveryError, discover_apps + from runpod.apps.entrypoint import get_entrypoint, run_entrypoint + + module = module.resolve() + if not module.is_file(): + _fail(f"{module} is not a file") + + os.environ["RUNPOD_DEV_SESSION"] = "1" + + try: + apps = discover_apps(module) + except DiscoveryError as exc: + _fail(str(exc)) + + if not apps: + _fail("no runpod.App found in module") + + entrypoint = get_entrypoint() + if entrypoint is None: + _fail( + "no @runpod.local_entrypoint found. define one:\n\n" + " @runpod.local_entrypoint\n" + " async def main(): ...\n" + ) + + async def _session() -> None: + session = DevSession(apps) + _echo("provisioning dev endpoints ...") + await session.start() + try: + while True: + _echo("--- running entrypoint ---") + try: + run_entrypoint(entrypoint) + except Exception as exc: # noqa: BLE001 - dev loop survives user errors + typer.secho(f"entrypoint error: {exc}", fg=typer.colors.RED) + _echo("--- press enter to re-run, ctrl-c to exit ---") + await asyncio.get_event_loop().run_in_executor(None, input) + finally: + _echo("cleaning up dev endpoints ...") + await session.stop() + + try: + asyncio.run(_session()) + except (KeyboardInterrupt, EOFError): + _echo("\nsession ended.") + + +@app.command() +def login(): + """authenticate with runpod and store the api key.""" + from runpod.cli.groups.config.functions import set_credentials + + api_key = typer.prompt("Runpod API key", hide_input=True) + try: + set_credentials(api_key, overwrite=True) + except ValueError as exc: + _fail(str(exc)) + _echo("credentials saved to ~/.runpod/config.toml") + + +def _mount_legacy_groups() -> None: + """mount the existing click groups so `rp pod ...` etc work.""" + import typer.main as typer_main + + from runpod.cli.groups.config.commands import config_wizard + from runpod.cli.groups.exec.commands import exec_cli + from runpod.cli.groups.pod.commands import pod_cli + from runpod.cli.groups.ssh.commands import ssh_cli + + click_app = typer_main.get_command(app) + for command in (config_wizard, ssh_cli, pod_cli, exec_cli): + click_app.add_command(command) + globals()["cli"] = click_app + + +_mount_legacy_groups() + + +def run() -> None: + """console-script entry point.""" + cli() # noqa: F821 - bound by _mount_legacy_groups + + +if __name__ == "__main__": + run() diff --git a/setup.py b/setup.py index 6fbfa7768..a98c375eb 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,12 @@ "serverless/binaries/README.md", ] }, - entry_points={"console_scripts": ["runpod = runpod.cli.entry:runpod_cli"]}, + entry_points={ + "console_scripts": [ + "runpod = runpod.cli.entry:runpod_cli", + "rp = runpod.rp_cli.main:run", + ] + }, keywords=[ "runpod", "ai", diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py new file mode 100644 index 000000000..ed1a3b21d --- /dev/null +++ b/tests/test_apps/test_deploy.py @@ -0,0 +1,190 @@ +"""tests for discovery, manifest building, and packaging.""" + +import io +import json +import tarfile +import textwrap +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +import runpod +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.deploy import ( + DeployResult, + build_manifest, + deploy_app, + package_project, +) +from runpod.apps.discovery import DiscoveryError, discover_apps +from runpod.apps.errors import ScheduleNotSupported + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +def _write_project(tmp_path: Path) -> Path: + (tmp_path / "main.py").write_text( + textwrap.dedent( + """ + import runpod + from runpod import App + + app = App("demo-app") + + @app.queue(name="q1", cpu="cpu5c-2-4") + def q1(x: int): + return x * 2 + + if __name__ == "__main__": + raise SystemExit("main guard must not run during discovery") + """ + ) + ) + return tmp_path + + +class TestDiscovery: + def test_discovers_app(self, tmp_path): + _write_project(tmp_path) + apps = discover_apps(tmp_path) + assert len(apps) == 1 + assert apps[0].name == "demo-app" + assert "q1" in apps[0].resources + + def test_single_file_target(self, tmp_path): + _write_project(tmp_path) + apps = discover_apps(tmp_path / "main.py") + assert len(apps) == 1 + + def test_main_guard_not_executed(self, tmp_path): + _write_project(tmp_path) + # would raise SystemExit if __main__ ran + discover_apps(tmp_path) + + def test_import_error_reported(self, tmp_path): + (tmp_path / "broken.py").write_text("import nonexistent_module_xyz") + with pytest.raises(DiscoveryError, match="broken.py"): + discover_apps(tmp_path) + + def test_skips_venv_dirs(self, tmp_path): + _write_project(tmp_path) + venv = tmp_path / ".venv" / "lib" + venv.mkdir(parents=True) + (venv / "bad.py").write_text("import nonexistent_module_xyz") + apps = discover_apps(tmp_path) + assert len(apps) == 1 + + def test_non_python_file_rejected(self, tmp_path): + f = tmp_path / "notes.txt" + f.write_text("hi") + with pytest.raises(DiscoveryError): + discover_apps(f) + + +class TestManifest: + def test_manifest_shape(self, tmp_path): + app = App("m-app") + + @app.queue(name="q", cpu="cpu5c-2-4", dependencies=["numpy"]) + def q(x): + return x + + manifest = build_manifest(app, tmp_path) + assert manifest["app"] == "m-app" + assert manifest["version"] == 1 + (resource,) = manifest["resources"] + assert resource["kind"] == "queue" + assert resource["name"] == "q" + assert resource["dependencies"] == ["numpy"] + assert resource["qualname"] + + def test_schedule_blocked_until_backend_support(self, tmp_path): + app = App("s-app") + + @app.task(name="t") + @runpod.schedule(cron="0 * * * *") + def t(): + pass + + with pytest.raises(ScheduleNotSupported): + build_manifest(app, tmp_path) + + +class TestPackaging: + def test_tarball_contains_source_and_manifest(self, tmp_path): + _write_project(tmp_path) + manifest = {"version": 1, "app": "demo-app", "resources": []} + tar_path = package_project(tmp_path, manifest) + + with tarfile.open(tar_path) as tar: + names = tar.getnames() + assert "main.py" in names + assert "runpod_manifest.json" in names + extracted = json.load(tar.extractfile("runpod_manifest.json")) + assert extracted["app"] == "demo-app" + + def test_ignores_applied(self, tmp_path): + _write_project(tmp_path) + (tmp_path / "secret.env").write_text("KEY=1") + (tmp_path / ".runpodignore").write_text("secret.env\n") + pycache = tmp_path / "__pycache__" + pycache.mkdir() + (pycache / "x.pyc").write_text("junk") + + tar_path = package_project(tmp_path, {"version": 1, "resources": []}) + with tarfile.open(tar_path) as tar: + names = tar.getnames() + assert "secret.env" not in names + assert not any("__pycache__" in n for n in names) + + +class TestDeployPipeline: + async def test_deploy_app_full_flow(self, tmp_path): + _write_project(tmp_path) + (app,) = discover_apps(tmp_path) + + api = AsyncMock() + api.get_app_by_name.return_value = None + api.create_app.return_value = {"id": "app-1", "flashEnvironments": []} + api.create_environment.return_value = {"id": "env-1", "name": "default"} + api.prepare_artifact_upload.return_value = { + "uploadUrl": "https://upload", + "objectKey": "key-1", + } + api.finalize_artifact_upload.return_value = {"id": "build-1"} + api.deploy_build.return_value = {"id": "env-1"} + + result = await deploy_app(app, tmp_path, api=api) + + assert isinstance(result, DeployResult) + assert result.build_id == "build-1" + assert result.resources == ["q1"] + api.upload_tarball.assert_awaited_once() + api.deploy_build.assert_awaited_once_with("env-1", "build-1") + + async def test_deploy_reuses_existing_app_and_env(self, tmp_path): + _write_project(tmp_path) + (app,) = discover_apps(tmp_path) + + api = AsyncMock() + api.get_app_by_name.return_value = { + "id": "app-1", + "flashEnvironments": [{"id": "env-1", "name": "default"}], + } + api.prepare_artifact_upload.return_value = { + "uploadUrl": "https://upload", + "objectKey": "key-1", + } + api.finalize_artifact_upload.return_value = {"id": "build-2"} + + await deploy_app(app, tmp_path, api=api) + + api.create_app.assert_not_awaited() + api.create_environment.assert_not_awaited() diff --git a/tests/test_cli/test_cli_groups/test_project_commands.py b/tests/test_cli/test_cli_groups/test_project_commands.py deleted file mode 100644 index 54a9fab09..000000000 --- a/tests/test_cli/test_cli_groups/test_project_commands.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Runpod | CLI | Groups | Project | Commands | Tests -""" - -import unittest -from unittest.mock import patch - -from click.testing import CliRunner - -from runpod.cli.groups.project.commands import ( - deploy_project, - new_project_wizard, - start_project_pod, -) - - -class TestProjectCLI(unittest.TestCase): - """A collection of tests for the Project CLI commands.""" - - def setUp(self): - self.runner = CliRunner() - - def test_new_project_wizard_no_network_volumes(self): - """ - Tests the new_project_wizard command with no network volumes. - """ - with patch("runpod.cli.groups.project.commands.get_user") as mock_get_user: - mock_get_user.return_value = {"networkVolumes": []} - - result = self.runner.invoke(new_project_wizard) - - self.assertEqual(result.exit_code, 1) - self.assertIn("You do not have any network volumes.", result.output) - - def test_new_project_wizard_success(self): - """ - Tests the new_project_wizard command. - """ - with patch("click.prompt") as mock_prompt, patch( - "click.confirm", return_value=True - ) as mock_confirm, patch( - "runpod.cli.groups.project.commands.create_new_project" - ) as mock_create, patch( - "runpod.cli.groups.project.commands.get_user" - ) as mock_get_user, patch( - "runpod.cli.groups.project.commands.cli_select" - ) as mock_select: - mock_get_user.return_value = { - "networkVolumes": [ - { - "id": "XYZ_VOLUME", - "name": "XYZ_VOLUME", - "size": 100, - "dataCenterId": "XYZ", - } - ] - } # pylint: disable=line-too-long - mock_prompt.side_effect = ["TestProject", "11.8.0", "3.10"] - mock_select.return_value = {"volume-id": "XYZ_VOLUME"} - - result = self.runner.invoke( - new_project_wizard, - ["--type", "llama2", "--model", "meta-llama/Llama-2-7b"], - ) # pylint: disable=line-too-long - - self.assertEqual(result.exit_code, 0) - mock_confirm.assert_called_with("Do you want to continue?", abort=True) - mock_create.assert_called() - mock_prompt.assert_called() - mock_create.assert_called_with( - "TestProject", - "XYZ_VOLUME", - "11.8.0", - "3.10", - "llama2", - "meta-llama/Llama-2-7b", - False, - ) # pylint: disable=line-too-long - self.assertIn("Project TestProject created successfully!", result.output) - - def test_new_project_wizard_success_init_current_dir(self): - """ - Tests the new_project_wizard command with the --init flag. - """ - with patch("click.prompt") as mock_prompt, patch( - "click.confirm", return_value=True - ) as mock_confirm, patch( - "runpod.cli.groups.project.commands.create_new_project" - ) as mock_create, patch( - "runpod.cli.groups.project.commands.get_user" - ) as mock_get_user, patch( - "runpod.cli.groups.project.commands.cli_select" - ) as mock_select, patch( - "os.getcwd" - ) as mock_getcwd: - mock_get_user.return_value = { - "networkVolumes": [ - { - "id": "XYZ_VOLUME", - "name": "XYZ_VOLUME", - "size": 100, - "dataCenterId": "XYZ", - } - ] - } # pylint: disable=line-too-long - mock_select.return_value = {"volume-id": "XYZ_VOLUME"} - mock_prompt.side_effect = ["XYZ_VOLUME", "11.8.0", "3.10"] - - self.runner.invoke(new_project_wizard, ["--init"]) - assert mock_confirm.called - assert mock_create.called - assert mock_getcwd.called - - def test_new_project_wizard_invalid_name(self): - """ - Tests the new_project_wizard command with an invalid project name. - """ - with patch("runpod.cli.groups.project.commands.get_user") as mock_get_user: - mock_get_user.return_value = {"networkVolumes": ["XYZ_VOLUME"]} - - result = self.runner.invoke(new_project_wizard, ["--name", "Invalid/Name"]) - - self.assertEqual(result.exit_code, 2) - self.assertIn("Project name contains an invalid character", result.output) - - def test_start_project_pod(self): - """ - Tests the start_project_pod command. - """ - with patch("click.confirm", return_value=True) as mock_confirm, patch( - "runpod.cli.groups.project.commands.start_project" - ) as mock_start: - mock_start.return_value = None - result = self.runner.invoke(start_project_pod) - - mock_confirm.assert_called_with("Do you want to continue?", abort=True) - self.assertEqual(result.exit_code, 0) - self.assertIn("Starting project development pod...", result.output) - - @patch("runpod.cli.groups.project.commands.click.echo") - @patch("runpod.cli.groups.project.commands.create_project_endpoint") - def test_deploy_project(self, mock_create_project_endpoint, mock_click_echo): - """Test the deploy_project function.""" - mock_create_project_endpoint.return_value = "test_endpoint_id" - - result = self.runner.invoke(deploy_project) - - mock_create_project_endpoint.assert_called_once() - - mock_click_echo.assert_any_call("Deploying project...") - mock_click_echo.assert_any_call("The following urls are available:") - mock_click_echo.assert_any_call( - " - https://api.runpod.ai/v2/test_endpoint_id/runsync" - ) - mock_click_echo.assert_any_call( - " - https://api.runpod.ai/v2/test_endpoint_id/run" - ) - mock_click_echo.assert_any_call( - " - https://api.runpod.ai/v2/test_endpoint_id/health" - ) - - self.assertEqual(result.exit_code, 0) diff --git a/tests/test_cli/test_cli_groups/test_project_functions.py b/tests/test_cli/test_cli_groups/test_project_functions.py deleted file mode 100644 index 667d71e60..000000000 --- a/tests/test_cli/test_cli_groups/test_project_functions.py +++ /dev/null @@ -1,338 +0,0 @@ -""" Test functions in runpod.cli.groups.project.functions module. """ - -import os -import shutil -import unittest -from unittest.mock import mock_open, patch - -from runpod.cli.groups.project.functions import ( - STARTER_TEMPLATES, - create_new_project, - create_project_endpoint, - start_project, -) - - -class TestCreateNewProject(unittest.TestCase): - """Test the create_new_project function.""" - - def tearDown(self): - toml_file_location = os.path.join(os.getcwd(), "test_project") - if os.path.exists(toml_file_location): - shutil.rmtree(toml_file_location) - - @patch("os.makedirs") - @patch("os.path.exists", return_value=False) - @patch("os.getcwd", return_value="/current/path") - @patch("runpod.cli.groups.project.functions.copy_template_files") - def test_create_project_folder( - self, mock_copy_template_files, mock_getcwd, mock_exists, mock_makedirs - ): # pylint: disable=line-too-long - """Test that a new project folder is created if init_current_dir is False.""" - with patch("builtins.open", new_callable=mock_open): - create_new_project("test_project", "volume_id", "11.1.1", "3.8") - mock_makedirs.assert_called_once_with("/current/path/test_project") - assert mock_copy_template_files.called - assert mock_getcwd.called - assert mock_exists.called - - @patch("os.makedirs") - @patch("os.path.exists", return_value=False) - @patch("os.getcwd", return_value="/tmp/testdir") - @patch("builtins.open", new_callable=mock_open) - def test_create_new_project_init_current_dir( - self, mock_file_open, mock_getcwd, mock_path_exists, mock_makedirs - ): # pylint: disable=line-too-long - """Test that a new project folder is not created if init_current_dir is True.""" - project_name = "test_project" - runpod_volume_id = "12345" - cuda_version = "11.1.1" - python_version = "3.9" - - create_new_project( - project_name, - runpod_volume_id, - cuda_version, - python_version, - init_current_dir=True, - ) - mock_makedirs.assert_not_called() - mock_file_open.assert_called_with( - "/tmp/testdir/runpod.toml", "w", encoding="UTF-8" - ) - assert mock_getcwd.called - assert mock_path_exists.called is False - - @patch("os.makedirs") - @patch("os.path.exists", return_value=False) - @patch("os.getcwd", return_value="/current/path") - @patch("runpod.cli.groups.project.functions.copy_template_files") - def test_copy_template_files( - self, mock_copy_template_files, mock_getcwd, mock_exists, mock_makedirs - ): # pylint: disable=line-too-long - """Test that template files are copied to the new project folder.""" - with patch("builtins.open", new_callable=mock_open): - create_new_project("test_project", "volume_id", "11.1.1", "3.8") - mock_copy_template_files.assert_called_once_with( - STARTER_TEMPLATES + "/default", "/current/path/test_project" - ) # pylint: disable=line-too-long - assert mock_getcwd.called - assert mock_exists.called - assert mock_makedirs.called - - @patch("os.path.exists", return_value=True) - @patch( - "builtins.open", - new_callable=mock_open, - read_data="data with <> placeholder", - ) # pylint: disable=line-too-long - def test_replace_placeholders_in_handler( - self, mock_open_file, mock_exists - ): # pylint: disable=line-too-long - """Test that placeholders in handler.py are replaced if model_name is given.""" - with patch("runpod.cli.groups.project.functions.copy_template_files"): - create_new_project( - "test_project", "volume_id", "11.8.0", "3.8", model_name="my_model" - ) - assert mock_open_file.called - assert mock_exists.called - - @patch("os.path.exists", return_value=False) - @patch("builtins.open", new_callable=mock_open) - def test_create_runpod_toml(self, mock_open_file, mock_exists): - """Test that runpod.toml configuration file is created.""" - with patch("runpod.cli.groups.project.functions.copy_template_files"): - create_new_project("test_project", "volume_id", "11.8.0", "3.8") - toml_file_location = os.path.join(os.getcwd(), "test_project", "runpod.toml") - mock_open_file.assert_called_with( - toml_file_location, "w", encoding="UTF-8" - ) # pylint: disable=line-too-long - assert mock_exists.called - - @patch("os.path.exists", return_value=True) - @patch("builtins.open", new_callable=mock_open, read_data="<> placeholder") - def test_update_requirements_file(self, mock_open_file, mock_exists): - """Test that placeholders in requirements.txt are replaced correctly.""" - with patch("runpod.cli.groups.project.functions.__version__", "dev"), patch( - "runpod.cli.groups.project.functions.copy_template_files" - ): - create_new_project("test_project", "volume_id", "11.8.0", "3.8") - assert mock_open_file.called - assert mock_exists.called - - @patch("os.path.exists", return_value=True) - @patch("builtins.open", new_callable=mock_open, read_data="<> placeholder") - def test_update_requirements_file_non_dev(self, mock_open_file, mock_exists): - """Test that placeholders in requirements.txt are replaced for non-dev versions.""" - with patch("runpod.cli.groups.project.functions.__version__", "1.0.0"), patch( - "runpod.cli.groups.project.functions.copy_template_files" - ): - create_new_project("test_project", "volume_id", "11.8.0", "3.8") - assert mock_open_file.called - assert mock_exists.called - - -class TestStartProject(unittest.TestCase): - """Test the start_project function.""" - - @patch("runpod.cli.groups.project.functions.load_project_config") - @patch("runpod.cli.groups.project.functions.get_project_pod") - @patch("runpod.cli.groups.project.functions.attempt_pod_launch") - @patch("runpod.cli.groups.project.functions.get_pod") - @patch("runpod.cli.groups.project.functions.SSHConnection") - @patch("os.getcwd", return_value="/current/path") - def test_start_nonexistent_successfully( - self, - mock_getcwd, - mock_ssh_connection, - mock_get_pod, - mock_attempt_pod_launch, - mock_get_project_pod, - mock_load_project_config, - ): # pylint: disable=line-too-long, too-many-arguments - """Test that a project is launched successfully.""" - mock_load_project_config.return_value = { - "project": { - "uuid": "123456", - "name": "test_project", - "volume_mount_path": "/mount/path", - "env_vars": {"ENV_VAR": "value"}, - "gpu": "NVIDIA GPU", - }, - "runtime": { - "python_version": "3.8", - "handler_path": "handler.py", - "requirements_path": "requirements.txt", - }, - } - - mock_get_project_pod.return_value = False - - mock_attempt_pod_launch.return_value = { - "id": "new_pod_id", - "desiredStatus": "PENDING", - "runtime": None, - } - - mock_get_pod.return_value = { - "id": "new_pod_id", - "desiredStatus": "RUNNING", - "runtime": "ONLINE", - } - - mock_ssh_instance = mock_ssh_connection.return_value - mock_ssh_instance.__enter__.return_value = mock_ssh_instance - mock_ssh_instance.run_commands.return_value = None - - with patch( - "runpod.cli.groups.project.functions.sync_directory" - ) as mock_sync_directory: - start_project() - - mock_attempt_pod_launch.assert_called() - mock_get_pod.assert_called_with("new_pod_id") - mock_ssh_connection.assert_called_with("new_pod_id") - mock_ssh_instance.run_commands.assert_called() - assert mock_getcwd.called - assert mock_sync_directory.called - - @patch("runpod.cli.groups.project.functions.get_project_pod") - @patch("runpod.cli.groups.project.functions.attempt_pod_launch") - def test_failed_pod_launch(self, mock_attempt_pod, mock_get_pod): - """Test that a project is not launched if pod launch fails.""" - mock_attempt_pod.return_value = None - mock_get_pod.return_value = None - - with patch("builtins.print") as mock_print, patch( - "runpod.cli.groups.project.functions.load_project_config" - ): - - start_project() - mock_print.assert_called_with( - "Selected GPU types unavailable, try again later or use a different type." - ) # pylint: disable=line-too-long - - -class TestStartProjectAPI(unittest.TestCase): - """Test the start_project_api function.""" - - @patch("runpod.cli.groups.project.functions.load_project_config") - @patch("runpod.cli.groups.project.functions.get_project_pod") - @patch("runpod.cli.groups.project.functions.SSHConnection") - @patch("os.getcwd", return_value="/current/path") - @patch("runpod.cli.groups.project.functions.sync_directory") - def test_start_project_api_successfully( - self, - mock_sync_directory, - mock_getcwd, - mock_ssh_connection, - mock_get_project_pod, - mock_load_project_config, - ): # pylint: disable=line-too-long, too-many-arguments - """Test that a project API is started successfully.""" - mock_load_project_config.return_value = { - "project": { - "uuid": "123456", - "name": "test_project", - "volume_mount_path": "/mount/path", - }, - "runtime": { - "python_version": "3.8", - "handler_path": "handler.py", - "requirements_path": "requirements.txt", - }, - } - - mock_get_project_pod.return_value = {"id": "pod_id"} - - mock_ssh_instance = mock_ssh_connection.return_value - mock_ssh_instance.__enter__.return_value = mock_ssh_instance - mock_ssh_instance.run_commands.return_value = None - - start_project() - - mock_get_project_pod.assert_called_with("123456") - mock_ssh_connection.assert_called_with({"id": "pod_id"}) - mock_sync_directory.assert_called_with( - mock_ssh_instance, "/current/path", "/mount/path/123456/dev" - ) - mock_ssh_instance.run_commands.assert_called() - assert mock_getcwd.called - - -class TestCreateProjectEndpoint(unittest.TestCase): - """Test the create_project_endpoint function.""" - - @patch("runpod.cli.groups.project.functions.SSHConnection") - @patch("runpod.cli.groups.project.functions.load_project_config") - @patch("runpod.cli.groups.project.functions.create_template") - @patch("runpod.cli.groups.project.functions.create_endpoint") - @patch("runpod.cli.groups.project.functions.update_endpoint_template") - @patch("runpod.cli.groups.project.functions.get_project_pod") - @patch("runpod.cli.groups.project.functions.get_project_endpoint") - def test_create_project_endpoint( - self, - mock_get_project_endpoint, - mock_get_project_pod, - mock_update_endpoint, - mock_create_endpoint, # pylint: disable=too-many-arguments,line-too-long - mock_create_template, - mock_load_project_config, - mock_ssh_connection, - ): # pylint: disable=line-too-long - """Test that a project endpoint is created successfully.""" - mock_get_project_endpoint.return_value = False - - mock_get_project_pod.return_value = None - with patch( - "runpod.cli.groups.project.functions._launch_dev_pod" - ) as mock_launch_dev_pod: - mock_launch_dev_pod.return_value = None - assert create_project_endpoint() is None - - mock_get_project_pod.return_value = {"id": "test_pod_id"} - mock_load_project_config.return_value = { - "project": { - "name": "test_project", - "volume_mount_path": "/runpod-volume/123456", - "uuid": "123456", - "env_vars": {"TEST_VAR": "value"}, - "base_image": "test_image", - "container_disk_size_gb": 10, - "storage_id": "test_storage_id", - }, - "runtime": { - "python_version": "3.8", - "handler_path": "handler.py", - "requirements_path": "requirements.txt", - }, - } - mock_create_template.return_value = {"id": "test_template_id"} - mock_create_endpoint.return_value = {"id": "test_endpoint_id"} - - mock_ssh_instance = mock_ssh_connection.return_value - mock_ssh_instance.__enter__.return_value = mock_ssh_instance - mock_ssh_instance.run_commands.return_value = None - - with patch("runpod.cli.groups.project.functions.datetime") as mock_datetime: - mock_datetime.now.return_value = "123456" - result = create_project_endpoint() - - self.assertEqual(result, "test_endpoint_id") - mock_create_template.assert_called_with( - name="test_project-endpoint | 123456 | 123456", - image_name="test_image", - container_disk_in_gb=10, - docker_start_cmd='bash -c ". /runpod-volume/123456/prod/venv/bin/activate && python -u /runpod-volume/123456/prod/test_project/handler.py"', # pylint: disable=line-too-long - env={"TEST_VAR": "value"}, - is_serverless=True, - ) - mock_create_endpoint.assert_called_with( - name="test_project-endpoint | 123456", - template_id="test_template_id", - network_volume_id="test_storage_id", - ) - - mock_update_endpoint.return_value = {"id": "test_endpoint_id"} - mock_get_project_endpoint.return_value = {"id": "test_endpoint_id"} - self.assertEqual(create_project_endpoint(), "test_endpoint_id") diff --git a/tests/test_cli/test_cli_groups/test_project_helpers.py b/tests/test_cli/test_cli_groups/test_project_helpers.py deleted file mode 100644 index 9b3437c86..000000000 --- a/tests/test_cli/test_cli_groups/test_project_helpers.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Tests for the project helpers.""" - -import unittest -from unittest.mock import mock_open, patch - -import click - -from runpod import error as rp_error -from runpod.cli.groups.project.helpers import ( - attempt_pod_launch, - copy_template_files, - get_project_endpoint, - get_project_pod, - load_project_config, - validate_project_name, -) - - -class TestHelpers(unittest.TestCase): - """Test the project helpers.""" - - def test_validate_project_name_valid(self): - """Test the validate_project_name function with valid input.""" - name = "validProjectName" - result = validate_project_name(name) - self.assertEqual(result, name) - - def test_validate_project_name_invalid(self): - """Test the validate_project_name function with invalid input.""" - name = "invalid:name" - with self.assertRaises(click.BadParameter): - validate_project_name(name) - - @patch("runpod.cli.groups.project.helpers.get_pods") - def test_get_project_pod_exists(self, mock_get_pods): - """Test the get_project_pod function when the project pod exists.""" - mock_get_pods.return_value = [{"name": "test-1234", "id": "pod_id"}] - result = get_project_pod("1234") - self.assertEqual(result, "pod_id") - - @patch("runpod.cli.groups.project.helpers.get_pods") - def test_get_project_pod_not_exists(self, mock_get_pods): - """Test the get_project_pod function when the project pod doesn't exist.""" - mock_get_pods.return_value = [{"name": "another-5678", "id": "another_pod_id"}] - result = get_project_pod("1234") - self.assertIsNone(result) - - @patch("runpod.cli.groups.project.helpers.get_endpoints") - def test_get_project_endpoint_exists(self, mock_get_endpoints): - """Test the get_project_endpoint function when the project endpoint exists.""" - mock_get_endpoints.return_value = [] - assert get_project_endpoint("1234") is None - - mock_get_endpoints.return_value = [{"name": "test-1234", "id": "endpoint_id"}] - result = get_project_endpoint("1234") - self.assertEqual(result, {"name": "test-1234", "id": "endpoint_id"}) - - @patch("os.listdir") - @patch("os.path.isdir", return_value=False) - @patch("shutil.copy2") - def test_copy_template_files(self, mock_copy, mock_isdir, mock_listdir): - """Test the copy_template_files function.""" - mock_listdir.return_value = ["file1.txt", "file2.txt"] - copy_template_files("/template", "/destination") - self.assertEqual(mock_copy.call_count, 2) - assert mock_isdir.called - - @patch("os.listdir") - @patch("os.path.isdir", return_value=True) - @patch("shutil.copytree") - def test_copy_template_files_dir(self, mock_copy, mock_isdir, mock_listdir): - """Test the copy_template_files function.""" - mock_listdir.return_value = ["file1.txt", "file2.txt"] - copy_template_files("/template", "/destination") - self.assertEqual(mock_copy.call_count, 2) - assert mock_isdir.called - - @patch("runpod.cli.groups.project.helpers.create_pod") - def test_attempt_pod_launch_success(self, mock_create_pod): - """Test the attempt_pod_launch function when it succeeds.""" - mock_create_pod.return_value = "pod_id" - config = { - "project": { - "name": "test", - "uuid": "1234", - "base_image": "base_image", - "gpu_types": ["gpu_type"], - "gpu_count": "1", - "ports": "ports", - "storage_id": "storage_id", - "volume_mount_path": "volume_mount_path", - "container_disk_size_gb": "1", - } - } - environment_variables = {"key": "value"} - result = attempt_pod_launch(config, environment_variables) - self.assertEqual(result, "pod_id") - - mock_create_pod.side_effect = rp_error.QueryError("error") - assert attempt_pod_launch(config, environment_variables) is None - - @patch("os.path.exists", return_value=True) - @patch("builtins.open", new_callable=mock_open, read_data="[project]\nname='test'") - def test_load_project_config(self, mock_file, mock_exists): - """Test the load_project_config function.""" - config = load_project_config() - self.assertEqual(config["project"]["name"], "test") - assert mock_exists.called - assert mock_file.called - - with patch("os.path.exists", return_value=False), self.assertRaises( - FileNotFoundError - ): - load_project_config() diff --git a/tests/test_cli/test_cli_utils/test_sync.py b/tests/test_cli/test_cli_utils/test_sync.py deleted file mode 100644 index adfc998f4..000000000 --- a/tests/test_cli/test_cli_utils/test_sync.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Tests for runpod.cli.utils.rp_sync module.""" - -import time -import unittest -from unittest.mock import ANY, MagicMock, patch - -from runpod.cli import STOP_EVENT -from runpod.cli.utils.rp_sync import WatcherHandler, start_watcher, sync_directory - - -class TestWatcherHandler(unittest.TestCase): - """Tests for the WatcherHandler class.""" - - @patch("runpod.cli.utils.rp_sync.should_ignore") - def test_on_any_event_with_ignored_file(self, mock_should_ignore): - """Test that the action function is not called when the file is ignored.""" - mock_should_ignore.return_value = True - mock_action_function = MagicMock() - handler = WatcherHandler(mock_action_function, "some_path") - - event_mock = MagicMock() - event_mock.is_directory = False - event_mock.src_path = "some_path/ignored_file.txt" - - handler.on_any_event(event_mock) - mock_action_function.assert_not_called() - - @patch("runpod.cli.utils.rp_sync.should_ignore") - def test_on_any_event_with_not_ignored_file(self, mock_should_ignore): - """Test that the action function is called when the file is not ignored.""" - mock_should_ignore.return_value = False - mock_action_function = MagicMock() - handler = WatcherHandler(mock_action_function, "some_path") - - event_mock = MagicMock() - event_mock.is_directory = False - event_mock.src_path = "some_path/not_ignored_file.txt" - - handler.on_any_event(event_mock) - handler.on_any_event(event_mock) # Call it twice to test the debouncer - time.sleep(2) - mock_action_function.assert_called_once() - - def test_on_any_event_with_directory(self): - """Test that the action function is not called when the event is a directory.""" - mock_action_function = MagicMock() - handler = WatcherHandler(mock_action_function, "some_path") - - event_mock = MagicMock() - event_mock.is_directory = True - - handler.on_any_event(event_mock) - time.sleep(2) - mock_action_function.assert_not_called() - - -class TestSyncDirectory(unittest.TestCase): - """Tests for the sync_directory function.""" - - @patch("runpod.cli.utils.rp_sync.threading.Thread") - @patch("runpod.cli.utils.rp_sync.start_watcher") - def test_sync_directory(self, mock_start_watcher, mock_thread_class): - """Test that the sync_directory function calls the start_watcher function.""" - mock_ssh_client = MagicMock() - - local_path = "local_path" - remote_path = "remote_path" - - sync_directory(mock_ssh_client, local_path, remote_path) - - target_function = mock_thread_class.call_args[1]["target"] - target_function() - - mock_start_watcher.assert_called_once() - - @patch("runpod.cli.utils.rp_sync.threading.Thread") - @patch("runpod.cli.utils.rp_sync.start_watcher") - def test_sync_directory_sync_function(self, mock_start_watcher, mock_thread_class): - """Test that the sync_directory function calls the start_watcher function.""" - mock_ssh_client = MagicMock() - - local_path = "local_path" - remote_path = "remote_path" - - sync_function = sync_directory(mock_ssh_client, local_path, remote_path) - sync_function() - - mock_ssh_client.rsync.assert_called_once_with( - local_path, remote_path, quiet=True - ) - - mock_thread_class.assert_called_once() - mock_thread_class.assert_called_with( - target=mock_start_watcher, daemon=True, args=(ANY, local_path) - ) - - assert mock_start_watcher.called is False - - -class TestStartWatcher(unittest.TestCase): - """Tests for the start_watcher function.""" - - @patch("runpod.cli.utils.rp_sync.Observer") - @patch("runpod.cli.utils.rp_sync.WatcherHandler") - def test_start_watcher(self, mock_watch_handler, mock_observer_class): - """Test that the start_watcher function starts the watcher correctly.""" - fake_action = MagicMock() - local_path = "/path/to/watch" - - mock_observer_instance = mock_observer_class.return_value - - STOP_EVENT.clear() - with patch("runpod.cli.utils.rp_sync.time.sleep") as mock_sleep: - - def side_effect(*args, **kwargs): - del args, kwargs - STOP_EVENT.set() - - mock_sleep.side_effect = side_effect - start_watcher(fake_action, local_path) - - mock_watch_handler.assert_called_once_with(fake_action, local_path) - - mock_observer_instance.schedule.assert_called_once_with( - mock_watch_handler.return_value, local_path, recursive=True - ) - - mock_observer_instance.start.assert_called_once() - mock_observer_instance.stop.assert_called_once() - mock_observer_instance.join.assert_called_once() From 7f0a45a553e3bf3389bf91ea796f527f20a2e0c9 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 15:57:24 -0700 Subject: [PATCH 003/120] feat: bind dev templates to endpoints and verify live execution e2e --- runpod/apps/app.py | 17 ++++ runpod/apps/dev.py | 23 ++++-- runpod/apps/spec.py | 3 + runpod/apps/targets.py | 55 +++++++++++-- tests/e2e/apps_live_smoke.py | 45 ++++++++++ tests/test_apps/test_dev.py | 154 +++++++++++++++++++++++++++++++++++ 6 files changed, 286 insertions(+), 11 deletions(-) create mode 100644 tests/e2e/apps_live_smoke.py create mode 100644 tests/test_apps/test_dev.py diff --git a/runpod/apps/app.py b/runpod/apps/app.py index b95f95b36..e45598c4c 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -88,6 +88,7 @@ def queue( system_dependencies: Optional[List[str]] = None, volume: Optional[Any] = None, env: Optional[Dict[str, str]] = None, + datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Callable], FunctionHandle]: """declare a queue-based serverless endpoint from a function.""" @@ -104,6 +105,7 @@ def decorator(fn: Callable) -> FunctionHandle: system_dependencies=system_dependencies, volume=_volume_ref(volume), env=env, + datacenter=_datacenter_list(datacenter), ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -123,6 +125,7 @@ def task( volume: Optional[Any] = None, env: Optional[Dict[str, str]] = None, image: Optional[str] = None, + datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Callable], FunctionHandle]: """declare ephemeral pod compute from a function. @@ -143,6 +146,7 @@ def decorator(fn: Callable) -> FunctionHandle: volume=_volume_ref(volume), env=env, image=image, + datacenter=_datacenter_list(datacenter), ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -163,6 +167,7 @@ def api( system_dependencies: Optional[List[str]] = None, volume: Optional[Any] = None, env: Optional[Dict[str, str]] = None, + datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Any], ApiHandle]: """declare a load-balanced serverless endpoint. @@ -199,6 +204,7 @@ def decorator(target: Any) -> ApiHandle: system_dependencies=system_dependencies, volume=_volume_ref(volume), env=env, + datacenter=_datacenter_list(datacenter), ) handle = ApiHandle(self, target, spec) self._register(spec.name, handle) @@ -237,6 +243,17 @@ def __repr__(self) -> str: return f"" +def _datacenter_list( + datacenter: Optional[Union[str, List[str]]], +) -> Optional[List[str]]: + """normalize datacenter input to a list of location strings.""" + if datacenter is None: + return None + if isinstance(datacenter, str): + return [datacenter] + return [str(d) for d in datacenter] + + def _volume_ref(volume: Any) -> Optional[str]: """normalize a volume argument to a name-or-id string reference.""" if volume is None: diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 5fa13c501..93b9b4c2b 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -47,8 +47,16 @@ def _image_for(spec: ResourceSpec) -> str: return image +# cpu serverless is only stocked in EU-RO-1 +CPU_LOCATIONS = "EU-RO-1" + + def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: - """build the saveEndpoint payload for a live dev endpoint.""" + """build the saveEndpoint payload for a live dev endpoint. + + the template is nested in the saveEndpoint input so it is bound to + the endpoint and cascades on deleteEndpoint. + """ payload: Dict = { "name": dev_endpoint_name(app.name, spec.name), "workersMin": spec.workers[0], @@ -62,20 +70,23 @@ def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), "containerDiskInGb": 10, - "isServerless": True, + "dockerArgs": "", + "env": [ + {"key": k, "value": v} for k, v in (spec.env or {}).items() + ], }, } if spec.kind is ResourceKind.API: payload["type"] = "LB" + if spec.datacenter: + payload["locations"] = ",".join(spec.datacenter) if spec.is_cpu: payload["instanceIds"] = spec.cpu + if not spec.datacenter: + payload["locations"] = CPU_LOCATIONS else: payload["gpuIds"] = ",".join(spec.gpu or ["any"]) payload["gpuCount"] = spec.gpu_count - if spec.env: - payload["template"]["env"] = [ - {"key": k, "value": v} for k, v in spec.env.items() - ] return payload diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index d7294af04..947b9275f 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -108,6 +108,7 @@ class ResourceSpec: system_dependencies: Optional[List[str]] = None volume: Optional[str] = None env: Optional[Dict[str, str]] = None + datacenter: Optional[List[str]] = None image: Optional[str] = None schedule: Optional[str] = None routes: List[RouteSpec] = field(default_factory=list) @@ -147,6 +148,8 @@ def to_manifest(self) -> Dict[str, Any]: data["networkVolume"] = self.volume if self.env: data["env"] = self.env + if self.datacenter: + data["locations"] = ",".join(self.datacenter) if self.image: data["imageName"] = self.image if self.schedule: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index a6113df07..96a3ee76b 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -145,6 +145,47 @@ async def _post_json( return await resp.json() +async def _get_json( + url: str, headers: Dict[str, str], timeout: float +) -> Dict[str, Any]: + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.get(url, headers=headers) as resp: + resp.raise_for_status() + return await resp.json() + + +async def _wait_terminal( + base_url: str, + data: Dict[str, Any], + headers: Dict[str, str], + timeout: float, +) -> Dict[str, Any]: + """poll /status until the job reaches a terminal state. + + runsync returns early (e.g. IN_QUEUE) when the job outlives the sync + window, typically on cold starts; polling covers the rest. + """ + import asyncio + import time + + deadline = time.monotonic() + timeout + interval = 0.5 + while data.get("status") not in FINAL_STATUSES: + job_id = data.get("id") + if not job_id: + return data + if time.monotonic() >= deadline: + raise TimeoutError( + f"job {job_id} did not complete within {timeout}s " + f"(last status: {data.get('status', 'UNKNOWN')})" + ) + await asyncio.sleep(interval) + interval = min(interval * 1.5, 5.0) + data = await _get_json(f"{base_url}/status/{job_id}", headers, 30.0) + return data + + class SentinelTarget(InvocationTarget): """routes through the sentinel endpoint; ai-api resolves the real endpoint server-side from app/env/resource headers. @@ -170,15 +211,17 @@ def _sentinel_headers(self) -> Dict[str, str]: async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: - url = f"{_endpoint_url_base()}/{SENTINEL_ID}/runsync" + base = f"{_endpoint_url_base()}/{SENTINEL_ID}" + headers = self._sentinel_headers() data = await _post_json( - url, + f"{base}/runsync", payload, - self._sentinel_headers(), + headers, timeout, app_name=self.app_name, resource_name=self.resource_name, ) + data = await _wait_terminal(base, data, headers, timeout) return unwrap_job_output(data) async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: @@ -264,8 +307,10 @@ def unwrap(self, data: Dict[str, Any]) -> Any: async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: - url = f"{_endpoint_url_base()}/{self.endpoint_id}/runsync" - data = await _post_json(url, payload, _headers(), timeout) + base = f"{_endpoint_url_base()}/{self.endpoint_id}" + headers = _headers() + data = await _post_json(f"{base}/runsync", payload, headers, timeout) + data = await _wait_terminal(base, data, headers, timeout) return self.unwrap(data) async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/e2e/apps_live_smoke.py b/tests/e2e/apps_live_smoke.py new file mode 100644 index 000000000..43ac64d34 --- /dev/null +++ b/tests/e2e/apps_live_smoke.py @@ -0,0 +1,45 @@ +"""manual e2e smoke test for the apps live path. + +provisions a real cpu dev endpoint, executes a function remotely via +the live source-per-request protocol, and tears the endpoint down. +requires credentials in ~/.runpod/config.toml or RUNPOD_API_KEY. + +run directly, not via pytest: + python tests/e2e/apps_live_smoke.py +""" + +import asyncio +import os +import sys + +os.environ["RUNPOD_DEV_SESSION"] = "1" + +from runpod.apps import App # noqa: E402 +from runpod.apps.dev import DevSession # noqa: E402 + +app = App("apps-e2e-v2") + + +@app.queue(name="smoke", cpu="cpu3c-1-2", workers=(0, 1)) +def smoke(x: int, y: int): + return {"sum": x + y, "python": True} + + +async def main() -> int: + session = DevSession([app]) + print("provisioning dev endpoint ...") + await session.start() + try: + print("invoking smoke.remote(2, 3) ...") + result = await smoke.remote.aio(2, 3) + print(f"result: {result}") + assert result == {"sum": 5, "python": True}, f"unexpected result: {result}" + print("PASS") + return 0 + finally: + print("cleaning up ...") + await session.stop() + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py new file mode 100644 index 000000000..188ad2a27 --- /dev/null +++ b/tests/test_apps/test_dev.py @@ -0,0 +1,154 @@ +"""tests for dev session provisioning.""" + +from unittest.mock import AsyncMock + +import pytest + +import runpod +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.dev import DevSession, _endpoint_input, dev_endpoint_name +from runpod.apps.targets import LiveTarget + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +def _mock_api(): + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.return_value = {"id": "ep-new"} + api.delete_endpoint.return_value = True + return api + + +class TestEndpointInput: + def test_cpu_queue_payload(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", workers=(0, 1)) + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["name"] == "dev-a-q" + assert payload["instanceIds"] == ["cpu3c-1-2"] + assert payload["locations"] == "EU-RO-1" + assert payload["scalerType"] == "QUEUE_DELAY" + template = payload["template"] + assert template["dockerArgs"] == "" + assert template["env"] == [] + assert "gpuIds" not in payload + + def test_gpu_queue_payload(self): + app = App("a") + + @app.queue(name="q", gpu=runpod.GpuGroup.ADA_24) + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["gpuIds"] == "ADA_24" + assert payload["gpuCount"] == 1 + assert "instanceIds" not in payload + assert "locations" not in payload + + def test_api_payload_is_lb(self): + app = App("a") + + @app.api(name="api", cpu="cpu3c-1-2") + class Api: + @runpod.post("/x") + def x(self, body: dict): + return body + + payload = _endpoint_input(app, Api.spec) + assert payload["type"] == "LB" + assert payload["scalerType"] == "REQUEST_COUNT" + + def test_datacenter_pins_locations(self): + app = App("a") + + @app.queue(name="q", gpu=runpod.GpuGroup.ADA_24, datacenter="US-KS-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["locations"] == "US-KS-2" + + def test_env_forwarded_to_template(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", env={"K": "v"}) + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["template"]["env"] == [{"key": "K", "value": "v"}] + + +class TestDevSession: + async def test_start_provisions_and_registers_targets(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = _mock_api() + session = DevSession([app], api=api) + await session.start() + + api.save_endpoint.assert_awaited_once() + target = app._dev_targets["q"] + assert isinstance(target, LiveTarget) + assert target.endpoint_id == "ep-new" + + async def test_start_adopts_existing_endpoint(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = _mock_api() + api.list_my_endpoints.return_value = [ + {"id": "ep-old", "name": dev_endpoint_name("a", "q")} + ] + session = DevSession([app], api=api) + await session.start() + + api.save_endpoint.assert_not_awaited() + assert app._dev_targets["q"].endpoint_id == "ep-old" + + async def test_stop_deletes_all_session_endpoints(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = _mock_api() + session = DevSession([app], api=api) + await session.start() + await session.stop() + + api.delete_endpoint.assert_awaited_once_with("ep-new") + assert app._dev_targets == {} + + async def test_tasks_not_provisioned(self): + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2") + def t(): + pass + + api = _mock_api() + session = DevSession([app], api=api) + await session.start() + + api.save_endpoint.assert_not_awaited() From f97280b845b51d80dc2f6e7cc1ed2d4fc3764fde Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 16:21:17 -0700 Subject: [PATCH 004/120] feat: add dev session refresh with file watching and storage-supported datacenter selection --- runpod/__init__.py | 2 + runpod/apps/__init__.py | 2 + runpod/apps/api.py | 9 ++- runpod/apps/datacenter.py | 61 +++++++++++++++ runpod/apps/dev.py | 107 +++++++++++++++++++++----- runpod/apps/watch.py | 59 +++++++++++++++ runpod/rp_cli/main.py | 61 ++++++++++----- tests/e2e/apps_live_smoke.py | 9 +++ tests/test_apps/test_dev.py | 137 ++++++++++++++++++++++++++++++++-- tests/test_apps/test_watch.py | 62 +++++++++++++++ tests/test_init.py | 2 +- 11 files changed, 468 insertions(+), 43 deletions(-) create mode 100644 runpod/apps/datacenter.py create mode 100644 runpod/apps/watch.py create mode 100644 tests/test_apps/test_watch.py diff --git a/runpod/__init__.py b/runpod/__init__.py index 8f4d71d53..7c410d496 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -32,6 +32,7 @@ from .apps import ( Api, App, + DataCenter, EndpointNotFound, Queue, is_local, @@ -74,6 +75,7 @@ "Api", "App", "CpuInstanceType", + "DataCenter", "EndpointNotFound", "GpuGroup", "GpuType", diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index acb63ce48..f0284adac 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -13,6 +13,7 @@ from .app import App, get_registered_apps from .context import Context, current_context, is_local +from .datacenter import DataCenter from .entrypoint import local_entrypoint from .errors import ( AppError, @@ -32,6 +33,7 @@ "App", "AppError", "Context", + "DataCenter", "EndpointNotFound", "FunctionHandle", "Queue", diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 913facd9e..b24ec7b19 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -62,7 +62,14 @@ async def list_my_endpoints(self) -> List[Dict[str, Any]]: query = """ query myEndpoints { myself { - endpoints { id name templateId workersMin workersMax } + endpoints { + id + name + templateId + workersMin + workersMax + template { env { key value } } + } } } """ diff --git a/runpod/apps/datacenter.py b/runpod/apps/datacenter.py new file mode 100644 index 000000000..3a71007b8 --- /dev/null +++ b/runpod/apps/datacenter.py @@ -0,0 +1,61 @@ +"""datacenter selection for app resources. + +only datacenters with storage support and S3 API support are listed; +resources may attach network volumes at any time, so every deployable +location must be able to host them. +""" + +from enum import Enum +from typing import List + + +class DataCenter(str, Enum): + """runpod datacenters with storage and S3 API support.""" + + # north america + US_CA_2 = "US-CA-2" + US_IL_1 = "US-IL-1" + US_KS_2 = "US-KS-2" + US_MO_1 = "US-MO-1" + US_MO_2 = "US-MO-2" + US_NC_2 = "US-NC-2" + US_NE_1 = "US-NE-1" + US_WA_1 = "US-WA-1" + + # europe + EU_CZ_1 = "EU-CZ-1" + EU_RO_1 = "EU-RO-1" + EUR_NO_1 = "EUR-NO-1" + + @classmethod + def from_string(cls, value: str) -> "DataCenter": + """parse a datacenter id, accepting lowercase and underscores.""" + normalized = value.strip().upper().replace("_", "-") + try: + return cls(normalized) + except ValueError: + valid = ", ".join(dc.value for dc in cls) + raise ValueError( + f"unknown datacenter '{value}'. valid datacenters: {valid}" + ) + + @classmethod + def all(cls) -> List["DataCenter"]: + return list(cls) + + +# datacenters with high cpu serverless stock, restricted to the +# storage+S3 set above. cpu5c/cpu5g are only stocked in EU-RO-1. +CPU3_DATACENTERS: List[DataCenter] = [ + DataCenter.EU_CZ_1, + DataCenter.EU_RO_1, + DataCenter.US_CA_2, + DataCenter.US_IL_1, + DataCenter.US_KS_2, + DataCenter.US_MO_1, + DataCenter.US_MO_2, + DataCenter.US_NC_2, + DataCenter.US_NE_1, + DataCenter.US_WA_1, +] +CPU5_DATACENTERS: List[DataCenter] = [DataCenter.EU_RO_1] diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 93b9b4c2b..9ff01fcc2 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -47,11 +47,23 @@ def _image_for(spec: ResourceSpec) -> str: return image -# cpu serverless is only stocked in EU-RO-1 -CPU_LOCATIONS = "EU-RO-1" +def _cpu_locations(instance_ids: List[str]) -> str: + """locations covering every requested cpu flavor's stock, limited to + datacenters with storage and S3 support. the flash images are + autocached on every host (host imagecache daemon, no DC filter), so + the widest supported spread maximizes provisioning odds.""" + from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS + if any(i.startswith("cpu5") for i in instance_ids): + return ",".join(dc.value for dc in CPU5_DATACENTERS) + return ",".join(dc.value for dc in CPU3_DATACENTERS) -def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: +# template env var bumped on refresh; env is a version-triggering template +# property, so changing it recreates all workers server-side +GENERATION_ENV = "RUNPOD_DEV_GENERATION" + + +def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: """build the saveEndpoint payload for a live dev endpoint. the template is nested in the saveEndpoint input so it is bound to @@ -66,13 +78,15 @@ def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: "REQUEST_COUNT" if spec.kind is ResourceKind.API else "QUEUE_DELAY" ), "scalerValue": 4, + "flashBootType": "FLASHBOOT", "template": { "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), "containerDiskInGb": 10, "dockerArgs": "", "env": [ - {"key": k, "value": v} for k, v in (spec.env or {}).items() + {"key": GENERATION_ENV, "value": str(generation)}, + *({"key": k, "value": v} for k, v in (spec.env or {}).items()), ], }, } @@ -83,7 +97,7 @@ def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: if spec.is_cpu: payload["instanceIds"] = spec.cpu if not spec.datacenter: - payload["locations"] = CPU_LOCATIONS + payload["locations"] = _cpu_locations(spec.cpu or []) else: payload["gpuIds"] = ",".join(spec.gpu or ["any"]) payload["gpuCount"] = spec.gpu_count @@ -91,13 +105,23 @@ def _endpoint_input(app: App, spec: ResourceSpec) -> Dict: class DevSession: - """owns the live endpoints for one `rp dev` invocation.""" + """owns the live endpoints for one `rp dev` invocation. + + lifecycle: adopt-or-create by name on start, reconcile on refresh + (config updates + a generation bump that recreates workers so every + request after a code change runs fully fresh), delete on stop. + """ def __init__(self, apps: List[App], api: Optional[AppsApiClient] = None): self.apps = apps self.api = api or AppsApiClient() - # endpoint ids created or adopted by this session - self._endpoint_ids: List[str] = [] + self.generation = 1 + # endpoint name -> id for everything this session owns + self._endpoints: Dict[str, str] = {} + + @property + def _endpoint_ids(self) -> List[str]: + return list(self._endpoints.values()) def _provisionable( self, app: App @@ -118,32 +142,79 @@ async def start(self) -> None: for handle in self._provisionable(app): spec = handle.spec name = dev_endpoint_name(app.name, spec.name) + payload = _endpoint_input(app, spec, self.generation) found = existing.get(name) if found: - endpoint_id = found["id"] - log.info("adopted existing dev endpoint %s (%s)", name, endpoint_id) + # adopt: reconcile the leftover endpoint to the + # current spec instead of creating a duplicate + payload["id"] = found["id"] + result = await self.api.save_endpoint(payload) + endpoint_id = result["id"] + log.info("adopted dev endpoint %s (%s)", name, endpoint_id) else: - result = await self.api.save_endpoint( - _endpoint_input(app, spec) - ) + result = await self.api.save_endpoint(payload) endpoint_id = result["id"] log.info("provisioned dev endpoint %s (%s)", name, endpoint_id) - self._endpoint_ids.append(endpoint_id) + self._endpoints[name] = endpoint_id app._dev_targets[spec.name] = LiveTarget(endpoint_id) + async def refresh(self, apps: List[App]) -> None: + """reconcile endpoints against a re-scanned set of apps. + + every surviving endpoint gets the new config plus a bumped + generation env var; env is a version-triggering template + property, so the platform recreates all workers and subsequent + requests execute in fresh processes. added resources are + provisioned, removed ones deleted.""" + self.generation += 1 + self.apps = apps + + desired: Dict[str, tuple] = {} + for app in apps: + for handle in self._provisionable(app): + name = dev_endpoint_name(app.name, handle.spec.name) + desired[name] = (app, handle) + + # delete endpoints whose resources disappeared + for name in list(self._endpoints): + if name not in desired: + endpoint_id = self._endpoints.pop(name) + try: + await self.api.delete_endpoint(endpoint_id) + log.info("deleted removed dev endpoint %s", name) + except Exception as exc: + log.warning("failed to delete %s: %s", name, exc) + + # update survivors (config + generation bump) and create additions + for name, (app, handle) in desired.items(): + payload = _endpoint_input(app, handle.spec, self.generation) + existing_id = self._endpoints.get(name) + if existing_id: + payload["id"] = existing_id + result = await self.api.save_endpoint(payload) + endpoint_id = result["id"] + self._endpoints[name] = endpoint_id + app._dev_targets[handle.spec.name] = LiveTarget(endpoint_id) + log.info( + "refreshed dev endpoint %s (%s, generation %d)", + name, + endpoint_id, + self.generation, + ) + async def stop(self) -> None: - """delete every endpoint this session created or adopted.""" - for endpoint_id in self._endpoint_ids: + """delete every endpoint this session owns.""" + for name, endpoint_id in list(self._endpoints.items()): try: await self.api.delete_endpoint(endpoint_id) - log.info("deleted dev endpoint %s", endpoint_id) + log.info("deleted dev endpoint %s (%s)", name, endpoint_id) except Exception as exc: log.warning( "failed to delete dev endpoint %s: %s", endpoint_id, exc ) - self._endpoint_ids.clear() + self._endpoints.clear() for app in self.apps: app._dev_targets.clear() diff --git a/runpod/apps/watch.py b/runpod/apps/watch.py new file mode 100644 index 000000000..7ea12d3cc --- /dev/null +++ b/runpod/apps/watch.py @@ -0,0 +1,59 @@ +"""filesystem change detection for dev sessions. + +a simple mtime poller: no extra dependency, no platform-specific event +apis, good enough for the dev loop's granularity. +""" + +import time +from pathlib import Path +from typing import Dict, Iterable, Optional + +from .discovery import _SKIP_DIRS + + +def _snapshot(roots: Iterable[Path]) -> Dict[str, float]: + state: Dict[str, float] = {} + for root in roots: + if root.is_file(): + try: + state[str(root)] = root.stat().st_mtime + except OSError: + pass + continue + for path in root.rglob("*.py"): + if any(part in _SKIP_DIRS for part in path.parts): + continue + try: + state[str(path)] = path.stat().st_mtime + except OSError: + continue + return state + + +class FileWatcher: + """tracks python file changes under a set of roots.""" + + def __init__(self, roots: Iterable[Path]): + self.roots = [Path(r) for r in roots] + self._state = _snapshot(self.roots) + + def changed(self) -> bool: + """true if any python file was added, removed, or modified since + the last call that returned true (or since construction).""" + current = _snapshot(self.roots) + if current != self._state: + self._state = current + return True + return False + + def wait_for_change( + self, poll_interval: float = 0.5, timeout: Optional[float] = None + ) -> bool: + """block until a change is detected. returns false on timeout.""" + deadline = time.monotonic() + timeout if timeout is not None else None + while True: + if self.changed(): + return True + if deadline is not None and time.monotonic() >= deadline: + return False + time.sleep(poll_interval) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 94cde5dfe..baaa8ca01 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -82,11 +82,14 @@ def dev( """start an interactive dev session for MODULE. provisions temporary live endpoints, runs the module's - @runpod.local_entrypoint, and deletes the endpoints on exit. + @runpod.local_entrypoint, watches for file changes (re-scanning and + refreshing endpoints so requests run fresh code), and deletes the + endpoints on exit. """ from runpod.apps.dev import DevSession from runpod.apps.discovery import DiscoveryError, discover_apps from runpod.apps.entrypoint import get_entrypoint, run_entrypoint + from runpod.apps.watch import FileWatcher module = module.resolve() if not module.is_file(): @@ -94,24 +97,42 @@ def dev( os.environ["RUNPOD_DEV_SESSION"] = "1" - try: - apps = discover_apps(module) - except DiscoveryError as exc: - _fail(str(exc)) - - if not apps: - _fail("no runpod.App found in module") - - entrypoint = get_entrypoint() - if entrypoint is None: - _fail( - "no @runpod.local_entrypoint found. define one:\n\n" - " @runpod.local_entrypoint\n" - " async def main(): ...\n" - ) + def _scan(): + try: + apps = discover_apps(module) + except DiscoveryError as exc: + _fail(str(exc)) + if not apps: + _fail("no runpod.App found in module") + entrypoint = get_entrypoint() + if entrypoint is None: + _fail( + "no @runpod.local_entrypoint found. define one:\n\n" + " @runpod.local_entrypoint\n" + " async def main(): ...\n" + ) + return apps, entrypoint + + apps, entrypoint = _scan() + + async def _wait_for_rerun(watcher: FileWatcher) -> str: + """race the enter key against a file change.""" + loop = asyncio.get_event_loop() + stdin_task = loop.run_in_executor(None, sys.stdin.readline) + try: + while True: + if watcher.changed(): + return "changed" + done, _ = await asyncio.wait({stdin_task}, timeout=0.5) + if done: + return "enter" + finally: + stdin_task.cancel() async def _session() -> None: + nonlocal apps, entrypoint session = DevSession(apps) + watcher = FileWatcher([module.parent]) _echo("provisioning dev endpoints ...") await session.start() try: @@ -121,8 +142,12 @@ async def _session() -> None: run_entrypoint(entrypoint) except Exception as exc: # noqa: BLE001 - dev loop survives user errors typer.secho(f"entrypoint error: {exc}", fg=typer.colors.RED) - _echo("--- press enter to re-run, ctrl-c to exit ---") - await asyncio.get_event_loop().run_in_executor(None, input) + _echo("--- press enter to re-run, edit files to reload, ctrl-c to exit ---") + reason = await _wait_for_rerun(watcher) + if reason == "changed": + _echo("--- change detected: re-scanning and refreshing endpoints ---") + apps, entrypoint = _scan() + await session.refresh(apps) finally: _echo("cleaning up dev endpoints ...") await session.stop() diff --git a/tests/e2e/apps_live_smoke.py b/tests/e2e/apps_live_smoke.py index 43ac64d34..e73958301 100644 --- a/tests/e2e/apps_live_smoke.py +++ b/tests/e2e/apps_live_smoke.py @@ -34,6 +34,15 @@ async def main() -> int: result = await smoke.remote.aio(2, 3) print(f"result: {result}") assert result == {"sum": 5, "python": True}, f"unexpected result: {result}" + + print("refreshing session (generation bump, workers recreated) ...") + await session.refresh([app]) + assert session.generation == 2 + + print("invoking smoke.remote(10, 20) post-refresh ...") + result = await smoke.remote.aio(10, 20) + print(f"result: {result}") + assert result == {"sum": 30, "python": True}, f"unexpected result: {result}" print("PASS") return 0 finally: diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 188ad2a27..ee8f701db 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -37,11 +37,39 @@ def q(): payload = _endpoint_input(app, q.spec) assert payload["name"] == "dev-a-q" assert payload["instanceIds"] == ["cpu3c-1-2"] - assert payload["locations"] == "EU-RO-1" + assert "EU-RO-1" in payload["locations"] + assert "US-KS-2" in payload["locations"] assert payload["scalerType"] == "QUEUE_DELAY" + assert payload["flashBootType"] == "FLASHBOOT" + + def test_cpu5_pins_to_stocked_datacenters(self): + app = App("a") + + @app.queue(name="q", cpu="cpu5c-2-4") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["locations"] == "EU-RO-1" + + def test_cpu_locations_are_storage_supported(self): + from runpod.apps.datacenter import DataCenter + + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + valid = {dc.value for dc in DataCenter} + for loc in payload["locations"].split(","): + assert loc in valid template = payload["template"] assert template["dockerArgs"] == "" - assert template["env"] == [] + assert template["env"] == [ + {"key": "RUNPOD_DEV_GENERATION", "value": "1"} + ] assert "gpuIds" not in payload def test_gpu_queue_payload(self): @@ -88,7 +116,19 @@ def q(): pass payload = _endpoint_input(app, q.spec) - assert payload["template"]["env"] == [{"key": "K", "value": "v"}] + assert {"key": "K", "value": "v"} in payload["template"]["env"] + + def test_generation_stamped_in_template_env(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec, generation=7) + assert {"key": "RUNPOD_DEV_GENERATION", "value": "7"} in payload[ + "template" + ]["env"] class TestDevSession: @@ -108,7 +148,7 @@ def q(): assert isinstance(target, LiveTarget) assert target.endpoint_id == "ep-new" - async def test_start_adopts_existing_endpoint(self): + async def test_start_adopts_and_reconciles_existing_endpoint(self): app = App("a") @app.queue(name="q", cpu="cpu3c-1-2") @@ -119,10 +159,13 @@ def q(): api.list_my_endpoints.return_value = [ {"id": "ep-old", "name": dev_endpoint_name("a", "q")} ] + api.save_endpoint.return_value = {"id": "ep-old"} session = DevSession([app], api=api) await session.start() - api.save_endpoint.assert_not_awaited() + # adopted endpoints are reconciled via saveEndpoint with the id set + payload = api.save_endpoint.await_args.args[0] + assert payload["id"] == "ep-old" assert app._dev_targets["q"].endpoint_id == "ep-old" async def test_stop_deletes_all_session_endpoints(self): @@ -152,3 +195,87 @@ def t(): await session.start() api.save_endpoint.assert_not_awaited() + + +class TestDevRefresh: + async def test_refresh_bumps_generation_and_updates(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = _mock_api() + api.save_endpoint.return_value = {"id": "ep-1"} + session = DevSession([app], api=api) + await session.start() + + await session.refresh([app]) + + payload = api.save_endpoint.await_args.args[0] + assert payload["id"] == "ep-1" + assert {"key": "RUNPOD_DEV_GENERATION", "value": "2"} in payload[ + "template" + ]["env"] + assert session.generation == 2 + + async def test_refresh_provisions_added_resource(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = _mock_api() + api.save_endpoint.return_value = {"id": "ep-1"} + session = DevSession([app], api=api) + await session.start() + + _clear_registry() + app2 = App("a") + + @app2.queue(name="q", cpu="cpu3c-1-2") + def q2(): + pass + + @app2.queue(name="extra", cpu="cpu3c-1-2") + def extra(): + pass + + api.save_endpoint.return_value = {"id": "ep-2"} + await session.refresh([app2]) + + names = { + c.args[0]["name"] for c in api.save_endpoint.await_args_list + } + assert dev_endpoint_name("a", "extra") in names + assert "extra" in app2._dev_targets + + async def test_refresh_deletes_removed_resource(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + @app.queue(name="gone", cpu="cpu3c-1-2") + def gone(): + pass + + api = _mock_api() + api.save_endpoint.side_effect = [{"id": "ep-q"}, {"id": "ep-gone"}] + session = DevSession([app], api=api) + await session.start() + + _clear_registry() + app2 = App("a") + + @app2.queue(name="q", cpu="cpu3c-1-2") + def q2(): + pass + + api.save_endpoint.side_effect = None + api.save_endpoint.return_value = {"id": "ep-q"} + await session.refresh([app2]) + + api.delete_endpoint.assert_awaited_once_with("ep-gone") diff --git a/tests/test_apps/test_watch.py b/tests/test_apps/test_watch.py new file mode 100644 index 000000000..62193ba1a --- /dev/null +++ b/tests/test_apps/test_watch.py @@ -0,0 +1,62 @@ +"""tests for the dev-session file watcher.""" + +import time + +from runpod.apps.watch import FileWatcher + + +def test_no_change_initially(tmp_path): + (tmp_path / "a.py").write_text("x = 1") + watcher = FileWatcher([tmp_path]) + assert watcher.changed() is False + + +def test_detects_modification(tmp_path): + f = tmp_path / "a.py" + f.write_text("x = 1") + watcher = FileWatcher([tmp_path]) + time.sleep(0.01) + f.write_text("x = 2") + # mtime granularity can be coarse; force it + import os + + os.utime(f, (time.time() + 1, time.time() + 1)) + assert watcher.changed() is True + assert watcher.changed() is False + + +def test_detects_new_file(tmp_path): + (tmp_path / "a.py").write_text("x = 1") + watcher = FileWatcher([tmp_path]) + (tmp_path / "b.py").write_text("y = 2") + assert watcher.changed() is True + + +def test_detects_deleted_file(tmp_path): + f = tmp_path / "a.py" + f.write_text("x = 1") + watcher = FileWatcher([tmp_path]) + f.unlink() + assert watcher.changed() is True + + +def test_ignores_non_python(tmp_path): + (tmp_path / "a.py").write_text("x = 1") + watcher = FileWatcher([tmp_path]) + (tmp_path / "notes.txt").write_text("hi") + assert watcher.changed() is False + + +def test_ignores_skip_dirs(tmp_path): + (tmp_path / "a.py").write_text("x = 1") + watcher = FileWatcher([tmp_path]) + venv = tmp_path / ".venv" + venv.mkdir() + (venv / "pkg.py").write_text("z = 3") + assert watcher.changed() is False + + +def test_wait_for_change_timeout(tmp_path): + (tmp_path / "a.py").write_text("x = 1") + watcher = FileWatcher([tmp_path]) + assert watcher.wait_for_change(poll_interval=0.05, timeout=0.2) is False diff --git a/tests/test_init.py b/tests/test_init.py index a5ca488a2..f5ff3ca68 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -99,7 +99,7 @@ def test_all_covers_expected_public_api(self): # Endpoint classes 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', # Apps surface - 'Api', 'App', 'CpuInstanceType', 'EndpointNotFound', 'GpuGroup', + 'Api', 'App', 'CpuInstanceType', 'DataCenter', 'EndpointNotFound', 'GpuGroup', 'GpuType', 'Queue', 'delete', 'get', 'init', 'is_local', 'local_entrypoint', 'patch', 'post', 'put', 'schedule', # Serverless module From cce017628e0e0a9dcc756523aa41a05788083008 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:04:51 -0700 Subject: [PATCH 005/120] feat: add pod task execution with runtime images and single-shot runner --- .github/workflows/runtimes.yml | 100 +++++++++ pyproject.toml | 2 + runpod/apps/api.py | 35 +++ runpod/apps/handles.py | 7 +- runpod/apps/targets.py | 53 ++++- runpod/apps/tasks.py | 261 +++++++++++++++++++++++ runpod/runtimes/README.md | 29 +++ runpod/runtimes/__init__.py | 0 runpod/runtimes/task/Dockerfile | 13 ++ runpod/runtimes/task/Dockerfile-gpu | 10 + runpod/runtimes/task/__init__.py | 0 runpod/runtimes/task/runner.py | 226 ++++++++++++++++++++ tests/e2e/task_pod_smoke.py | 37 ++++ tests/test_apps/test_task_runner_http.py | 119 +++++++++++ tests/test_apps/test_tasks.py | 254 ++++++++++++++++++++++ 15 files changed, 1135 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/runtimes.yml create mode 100644 runpod/apps/tasks.py create mode 100644 runpod/runtimes/README.md create mode 100644 runpod/runtimes/__init__.py create mode 100644 runpod/runtimes/task/Dockerfile create mode 100644 runpod/runtimes/task/Dockerfile-gpu create mode 100644 runpod/runtimes/task/__init__.py create mode 100644 runpod/runtimes/task/runner.py create mode 100644 tests/e2e/task_pod_smoke.py create mode 100644 tests/test_apps/test_task_runner_http.py create mode 100644 tests/test_apps/test_tasks.py diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml new file mode 100644 index 000000000..f41e50f36 --- /dev/null +++ b/.github/workflows/runtimes.yml @@ -0,0 +1,100 @@ +name: Runtimes + +on: + push: + branches: [main] + paths: + - "runpod/runtimes/**" + - ".github/workflows/runtimes.yml" + pull_request: + paths: + - "runpod/runtimes/**" + - ".github/workflows/runtimes.yml" + workflow_dispatch: + +env: + REGISTRY: docker.io + +jobs: + test-runner: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install test deps + run: pip install pytest pytest-asyncio cloudpickle aiohttp requests + + - name: Run task runner tests + run: | + pip install -e . --no-deps + pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" + + build-task-cpu: + runs-on: ubuntu-latest + needs: [test-runner] + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v7 + + - uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push task image + uses: docker/build-push-action@v6 + with: + context: . + file: runpod/runtimes/task/Dockerfile + platforms: linux/amd64 + build-args: | + PYTHON_VERSION=${{ matrix.python-version }} + push: ${{ github.event_name != 'pull_request' }} + tags: | + runpod/task:py${{ matrix.python-version }}-latest + cache-from: type=gha,scope=task-py${{ matrix.python-version }} + cache-to: type=gha,mode=max,scope=task-py${{ matrix.python-version }} + + build-task-gpu: + runs-on: ubuntu-latest + needs: [test-runner] + steps: + - uses: actions/checkout@v7 + + - name: Clear space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" + docker system prune -af + + - uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push gpu task image + uses: docker/build-push-action@v6 + with: + context: . + file: runpod/runtimes/task/Dockerfile-gpu + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: | + runpod/task-gpu:latest + cache-from: type=gha,scope=task-gpu + cache-to: type=gha,mode=max,scope=task-gpu diff --git a/pyproject.toml b/pyproject.toml index 6441f49b8..a5e8da034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,8 @@ include-package-data = true runpod = [ "serverless/binaries/gpu_test", "serverless/binaries/README.md", + "runtimes/**/*.py", + "runtimes/**/Dockerfile*", ] [tool.setuptools_scm] diff --git a/runpod/apps/api.py b/runpod/apps/api.py index b24ec7b19..fc16c082a 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -76,6 +76,41 @@ async def list_my_endpoints(self) -> List[Dict[str, Any]]: data = await self._execute(query) return data["myself"]["endpoints"] + # -- pods (task execution) -- + + async def deploy_task_pod( + self, pod_input: Dict[str, Any], *, is_cpu: bool + ) -> Dict[str, Any]: + """deploy an on-demand pod for a task run.""" + if is_cpu: + # deployCpuPod takes a single instanceId + instance_ids = pod_input.pop("instanceIds", None) + if instance_ids: + pod_input["instanceId"] = instance_ids[0] + mutation = """ + mutation deployCpuPod($input: deployCpuPodInput!) { + deployCpuPod(input: $input) { id desiredStatus } + } + """ + data = await self._execute(mutation, {"input": pod_input}) + return data["deployCpuPod"] + + mutation = """ + mutation deployPod($input: PodFindAndDeployOnDemandInput) { + podFindAndDeployOnDemand(input: $input) { id desiredStatus } + } + """ + data = await self._execute(mutation, {"input": pod_input}) + return data["podFindAndDeployOnDemand"] + + async def terminate_pod(self, pod_id: str) -> None: + mutation = """ + mutation terminatePod($input: PodTerminateInput!) { + podTerminate(input: $input) + } + """ + await self._execute(mutation, {"input": {"podId": pod_id}}) + # -- app lifecycle (deploy) -- async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 1702f3fdd..9d768bf74 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -105,11 +105,14 @@ async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: payload = target.build_payload(self._fn, self.spec, args, kwargs) return await target.invoke(payload) - async def _spawn_async(self, *args: Any, **kwargs: Any) -> Job: + async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) data = await target.submit(payload) - return Job(data, self) + # queue targets return raw job data; task targets return a TaskJob + if isinstance(data, dict): + return Job(data, self) + return data def _is_current_worker(self) -> bool: import os diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 96a3ee76b..a472073ae 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -343,21 +343,56 @@ class PodTarget(InvocationTarget): """ephemeral pod execution for @app.task: provision a pod, run the function body, collect the result, terminate. - transport lands with the task execution engine. + tasks run to completion far beyond the queue data plane's sync + window, so the transport is a direct http channel to a single-shot + runner on the pod (see runpod.apps.tasks). """ + # tasks default to a long window; the pod's terminateAfter is the backstop + TASK_TIMEOUT_SECONDS = 3600.0 + def __init__(self, spec: ResourceSpec, fn: Callable): self.spec = spec self.fn = fn - async def invoke( - self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS - ) -> Any: - raise NotImplementedError( - "task pod execution is not wired up yet; use .local() meanwhile" + def build_payload( + self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict + ) -> Dict[str, Any]: + from .serialization import ( + get_function_source, + serialize_args, + serialize_kwargs, ) - async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: - raise NotImplementedError( - "task pod execution is not wired up yet; use .local() meanwhile" + request = FunctionRequest( + function_name=fn.__name__, + function_code=get_function_source(fn), + args=serialize_args(args), + kwargs=serialize_kwargs(kwargs), + dependencies=spec.dependencies, + system_dependencies=spec.system_dependencies, ) + return request.to_input() + + async def invoke( + self, payload: Dict[str, Any], *, timeout: float = TASK_TIMEOUT_SECONDS + ) -> Any: + from .tasks import TaskExecution, unwrap_task_response + + execution = TaskExecution(self.spec) + await execution.start() + try: + await execution.wait_ready() + response = await execution.execute(payload, timeout) + finally: + await execution.terminate() + return unwrap_task_response(response) + + async def submit(self, payload: Dict[str, Any]) -> Any: + from .tasks import TaskExecution, TaskJob + + execution = TaskExecution(self.spec) + await execution.start() + await execution.wait_ready() + await execution.submit(payload) + return TaskJob(execution) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py new file mode 100644 index 000000000..830a2f300 --- /dev/null +++ b/runpod/apps/tasks.py @@ -0,0 +1,261 @@ +"""ephemeral pod execution for @app.task. + +lifecycle per call: + 1. deploy a pod on a task runtime image (runpod/task*, built from + runtimes/task in this repo). custom images get the runner source + delivered base64-encoded in the pod env and booted via dockerArgs, + so any image with a python3 binary works. + 2. wait for the runner's /ping via the pod http proxy + 3. POST the FunctionRequest to /execute (remote) or /submit (spawn) + 4. collect the response, terminate the pod + +`terminateAfter` is set at deploy time as a server-side safety net so a +crashed client cannot leak a running pod indefinitely. +""" + +import asyncio +import base64 +import logging +import secrets +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +import aiohttp + +from .api import AppsApiClient +from .errors import RemoteExecutionError +from .spec import ResourceSpec + +log = logging.getLogger(__name__) + +TASK_PORT = 8080 + +# safety net: pods self-terminate server-side after this long +DEFAULT_MAX_LIFETIME = timedelta(hours=1) + +# task runtime images built from runtimes/task by the runtimes workflow +DEFAULT_GPU_IMAGE = "runpod/task-gpu:latest" +DEFAULT_CPU_IMAGE = "runpod/task:py3.12-latest" + +READY_POLL_INTERVAL = 2.0 +READY_TIMEOUT = 600.0 +RESULT_POLL_INTERVAL = 2.0 + + +def _runner_source() -> str: + return ( + Path(__file__).parent.parent / "runtimes" / "task" / "runner.py" + ).read_text() + + +def _bootstrap_command() -> str: + """shell command that materializes and starts the runner.""" + return ( + "bash -c 'echo $RUNPOD_TASK_RUNNER_B64 | base64 -d > /task_runner.py " + "&& python3 /task_runner.py'" + ) + + +def _proxy_url(pod_id: str) -> str: + return f"https://{pod_id}-{TASK_PORT}.proxy.runpod.net" + + +def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any]: + """build the pod deploy input for one task execution. + + runtime images have the runner baked in (CMD starts it); custom + images bootstrap the runner from a base64 env payload via dockerArgs. + """ + terminate_after = ( + datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME + ).isoformat() + + env = { + "RUNPOD_TASK_TOKEN": token, + "RUNPOD_TASK_PORT": str(TASK_PORT), + **(spec.env or {}), + } + + pod: Dict[str, Any] = { + "name": f"task-{task_name}-{secrets.token_hex(4)}", + "imageName": spec.image + or (DEFAULT_CPU_IMAGE if spec.is_cpu else DEFAULT_GPU_IMAGE), + "ports": f"{TASK_PORT}/http", + "containerDiskInGb": 10, + "terminateAfter": terminate_after, + "supportPublicIp": True, + } + + if spec.image: + # custom image: inject the runner source and boot it explicitly + env["RUNPOD_TASK_RUNNER_B64"] = base64.b64encode( + _runner_source().encode() + ).decode() + pod["dockerArgs"] = _bootstrap_command() + + pod["env"] = [{"key": k, "value": v} for k, v in env.items()] + if spec.datacenter: + pod["dataCenterIds"] = spec.datacenter + else: + # spread across the storage-supported set; a null DC can pin + # repeated deploys to one (possibly broken) machine + from .datacenter import CPU3_DATACENTERS, DataCenter + + pool = CPU3_DATACENTERS if spec.is_cpu else DataCenter.all() + pod["dataCenterIds"] = [dc.value for dc in pool] + if spec.volume: + pod["networkVolumeId"] = spec.volume + if spec.is_cpu: + pod["instanceIds"] = spec.cpu + else: + pod["gpuTypeIdList"] = spec.gpu or ["any"] + pod["gpuCount"] = spec.gpu_count + return pod + + +class TaskExecution: + """one pod running one function.""" + + def __init__(self, spec: ResourceSpec, api: Optional[AppsApiClient] = None): + self.spec = spec + self.api = api or AppsApiClient() + self.token = secrets.token_urlsafe(32) + self.pod_id: Optional[str] = None + + @property + def _headers(self) -> Dict[str, str]: + return {"Authorization": f"Bearer {self.token}"} + + async def start(self) -> None: + result = await self.api.deploy_task_pod( + _pod_input(self.spec, self.token, self.spec.name), + is_cpu=self.spec.is_cpu, + ) + self.pod_id = result["id"] + log.info("task pod %s deployed for %s", self.pod_id, self.spec.name) + + async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: + """poll the runner's /ping through the pod proxy until it answers.""" + url = f"{_proxy_url(self.pod_id)}/ping" + deadline = time.monotonic() + timeout + async with aiohttp.ClientSession() as session: + while time.monotonic() < deadline: + try: + async with session.get( + url, timeout=aiohttp.ClientTimeout(total=5) + ) as resp: + if resp.status == 200: + return + except (aiohttp.ClientError, asyncio.TimeoutError): + pass + await asyncio.sleep(READY_POLL_INTERVAL) + raise TimeoutError( + f"task pod {self.pod_id} did not become ready within {timeout}s" + ) + + async def execute(self, request: Dict[str, Any], timeout: float) -> Dict[str, Any]: + """run to completion via /execute.""" + url = f"{_proxy_url(self.pod_id)}/execute" + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.post( + url, json=request, headers=self._headers + ) as resp: + resp.raise_for_status() + return await resp.json() + + async def submit(self, request: Dict[str, Any]) -> None: + """start in the background via /submit.""" + url = f"{_proxy_url(self.pod_id)}/submit" + async with aiohttp.ClientSession() as session: + async with session.post( + url, + json=request, + headers=self._headers, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + resp.raise_for_status() + + async def poll_result(self) -> Optional[Dict[str, Any]]: + """fetch /result; returns the response dict once DONE, else None.""" + url = f"{_proxy_url(self.pod_id)}/result" + async with aiohttp.ClientSession() as session: + async with session.get( + url, + headers=self._headers, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + resp.raise_for_status() + data = await resp.json() + if data.get("status") == "DONE": + return data.get("response") + return None + + async def terminate(self) -> None: + if self.pod_id is None: + return + try: + await self.api.terminate_pod(self.pod_id) + log.info("task pod %s terminated", self.pod_id) + except Exception as exc: # noqa: BLE001 - terminateAfter is the backstop + log.warning( + "failed to terminate task pod %s (terminateAfter is the " + "backstop): %s", + self.pod_id, + exc, + ) + self.pod_id = None + + +def unwrap_task_response(response: Dict[str, Any]) -> Any: + """extract the function result from a runner response.""" + if not response.get("success"): + raise RemoteExecutionError( + f"task execution failed: {response.get('error', 'unknown')}" + ) + if response.get("result") is not None: + from .serialization import deserialize_result + + return deserialize_result(response["result"]) + return response.get("json_result") + + +class TaskJob: + """handle for a spawned task; owns the pod until the result is read.""" + + def __init__(self, execution: TaskExecution): + self._execution = execution + self._result: Any = None + self._done = False + + @property + def pod_id(self) -> Optional[str]: + return self._execution.pod_id + + async def wait(self, timeout: Optional[float] = None) -> Any: + """block until the task finishes; terminates the pod and returns + the result.""" + deadline = time.monotonic() + timeout if timeout is not None else None + try: + while not self._done: + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError( + f"task on pod {self.pod_id} did not finish in {timeout}s" + ) + response = await self._execution.poll_result() + if response is not None: + self._result = unwrap_task_response(response) + self._done = True + break + await asyncio.sleep(RESULT_POLL_INTERVAL) + finally: + if self._done: + await self._execution.terminate() + return self._result + + async def cancel(self) -> None: + """terminate the pod, abandoning the task.""" + await self._execution.terminate() + self._done = True diff --git a/runpod/runtimes/README.md b/runpod/runtimes/README.md new file mode 100644 index 000000000..148bc1581 --- /dev/null +++ b/runpod/runtimes/README.md @@ -0,0 +1,29 @@ +# runtimes + +worker runtime scripts and the docker images built from them. images +are built and pushed by the `runtimes` github actions workflow on +release. + +## task + +single-shot task runner for `@app.task` pods. one pod runs one +function: the sdk deploys a pod on one of these images, waits for +`/ping`, posts a `FunctionRequest` to `/execute` (or `/submit` for +spawn), collects the result, and terminates the pod. + +| image | base | use | +|---|---|---| +| `runpod/task:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | cpu tasks | +| `runpod/task-gpu:{tag}` | `runpod/pytorch` | gpu tasks | + +`runner.py` is stdlib-only (cloudpickle optional, used for argument +and result serialization when present). this keeps the env-injection +fallback working: when a user supplies a custom `image=` that does not +have the runner baked in, the sdk ships `runner.py` base64-encoded in +a pod env var and boots it via `dockerArgs`, so any image with a +python3 binary works. + +the wire protocol (`FunctionRequest` / `FunctionResponse`) is defined +in `runpod.apps.protocol`; the runner and sdk must stay in sync with +it. `runner.py` deliberately avoids importing the runpod package so it +can run standalone on arbitrary images. diff --git a/runpod/runtimes/__init__.py b/runpod/runtimes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/runpod/runtimes/task/Dockerfile b/runpod/runtimes/task/Dockerfile new file mode 100644 index 000000000..a8f22cc81 --- /dev/null +++ b/runpod/runtimes/task/Dockerfile @@ -0,0 +1,13 @@ +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + PYTHONUNBUFFERED=1 + +RUN pip install --no-cache-dir cloudpickle + +COPY runpod/runtimes/task/runner.py /runner.py + +EXPOSE 8080 +CMD ["python", "/runner.py"] diff --git a/runpod/runtimes/task/Dockerfile-gpu b/runpod/runtimes/task/Dockerfile-gpu new file mode 100644 index 000000000..e08e1e6f8 --- /dev/null +++ b/runpod/runtimes/task/Dockerfile-gpu @@ -0,0 +1,10 @@ +FROM runpod/pytorch:1.0.3-cu1281-torch291-ubuntu2204 + +ENV PYTHONUNBUFFERED=1 + +RUN pip install --no-cache-dir cloudpickle + +COPY runpod/runtimes/task/runner.py /runner.py + +EXPOSE 8080 +CMD ["python", "/runner.py"] diff --git a/runpod/runtimes/task/__init__.py b/runpod/runtimes/task/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py new file mode 100644 index 000000000..e27ee4db8 --- /dev/null +++ b/runpod/runtimes/task/runner.py @@ -0,0 +1,226 @@ +"""single-shot task runner: the process a task pod boots into. + +a minimal http server speaking the FunctionRequest/FunctionResponse +protocol. stdlib-only (cloudpickle optional, required only for the +cloudpickle serialization format) so it can bootstrap onto any python +image via an env-var payload, and be pre-baked into the task runtime +image later. + +endpoints: + GET /ping readiness probe (unauthenticated) + POST /execute run a function, block, return the response + POST /submit start a function in the background, return immediately + GET /result status/result of the submitted job + +auth: every endpoint except /ping requires + Authorization: Bearer $RUNPOD_TASK_TOKEN + +one pod runs one function; the client terminates the pod after +collecting the result. +""" + +import base64 +import io +import json +import os +import subprocess +import sys +import threading +import traceback +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +PORT = int(os.environ.get("RUNPOD_TASK_PORT", "8080")) +TOKEN = os.environ.get("RUNPOD_TASK_TOKEN", "") + +# single background job slot for /submit + /result +_job_lock = threading.Lock() +_job_state = {"status": "NONE", "response": None} + + +def _load_cloudpickle(install: bool = False): + try: + import cloudpickle + + return cloudpickle + except ImportError: + if not install: + return None + # bare images (custom image= without the runtime baked in) may lack + # cloudpickle; install it on demand the same way dependencies are + if _install(["cloudpickle"], "cloudpickle"): + return None + import cloudpickle + + return cloudpickle + + +def _install(packages, label): + if not packages: + return None + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", *packages], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return f"failed to install {label}: {result.stderr[-2000:]}" + return None + + +def _deserialize_args(request): + fmt = request.get("serialization_format", "cloudpickle") + args = request.get("args") or [] + kwargs = request.get("kwargs") or {} + if fmt == "json": + return list(args), dict(kwargs) + cloudpickle = _load_cloudpickle(install=True) + if cloudpickle is None: + raise RuntimeError("cloudpickle not available for argument deserialization") + args = [cloudpickle.loads(base64.b64decode(a)) for a in args] + kwargs = {k: cloudpickle.loads(base64.b64decode(v)) for k, v in kwargs.items()} + return args, kwargs + + +def _serialize_result(result, fmt): + if fmt == "json": + return {"json_result": result} + cloudpickle = _load_cloudpickle(install=True) + if cloudpickle is None: + return {"json_result": result} + return {"result": base64.b64encode(cloudpickle.dumps(result)).decode("utf-8")} + + +def execute_request(request): + """run one FunctionRequest to completion, returning a response dict.""" + stdout_io = io.StringIO() + try: + error = _install(request.get("dependencies"), "dependencies") + if error: + return {"success": False, "error": error} + + function_name = request.get("function_name") + function_code = request.get("function_code") + if not function_name or not function_code: + return { + "success": False, + "error": "function_name and function_code are required", + } + + namespace = {} + exec(function_code, namespace) # noqa: S102 - that is the job + if function_name not in namespace: + return { + "success": False, + "error": f"function '{function_name}' not found in provided code", + } + fn = namespace[function_name] + + args, kwargs = _deserialize_args(request) + + # stdout only: sys.stderr carries the runner's own request logs + # from other threads, which must not leak into job output + with redirect_stdout(stdout_io): + result = fn(*args, **kwargs) + if hasattr(result, "__await__"): + import asyncio + + result = asyncio.run(_await(result)) + + response = {"success": True, "stdout": stdout_io.getvalue()} + response.update( + _serialize_result( + result, request.get("serialization_format", "cloudpickle") + ) + ) + return response + except Exception: # noqa: BLE001 - all failures go over the wire + return { + "success": False, + "error": traceback.format_exc(), + "stdout": stdout_io.getvalue(), + } + + +async def _await(awaitable): + return await awaitable + + +class Handler(BaseHTTPRequestHandler): + def _send(self, code, payload): + body = json.dumps(payload).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _authed(self): + header = self.headers.get("Authorization", "") + return TOKEN and header == f"Bearer {TOKEN}" + + def _read_request(self): + length = int(self.headers.get("Content-Length", "0")) + return json.loads(self.rfile.read(length) or b"{}") + + def do_GET(self): # noqa: N802 - BaseHTTPRequestHandler api + if self.path == "/ping": + self._send(200, {"ready": True}) + return + if self.path == "/result": + if not self._authed(): + self._send(401, {"error": "unauthorized"}) + return + with _job_lock: + self._send( + 200, + { + "status": _job_state["status"], + "response": _job_state["response"], + }, + ) + return + self._send(404, {"error": "not found"}) + + def do_POST(self): # noqa: N802 - BaseHTTPRequestHandler api + if not self._authed(): + self._send(401, {"error": "unauthorized"}) + return + if self.path == "/execute": + self._send(200, execute_request(self._read_request())) + return + if self.path == "/submit": + request = self._read_request() + with _job_lock: + if _job_state["status"] == "RUNNING": + self._send(409, {"error": "a job is already running"}) + return + _job_state["status"] = "RUNNING" + _job_state["response"] = None + + def run(): + response = execute_request(request) + with _job_lock: + _job_state["status"] = "DONE" + _job_state["response"] = response + + threading.Thread(target=run, daemon=True).start() + self._send(200, {"status": "RUNNING"}) + return + self._send(404, {"error": "not found"}) + + def log_message(self, format, *args): # noqa: A002 - stdlib signature + sys.stderr.write(f"[task-runner] {format % args}\n") + + +def main() -> None: + if not TOKEN: + sys.stderr.write("[task-runner] RUNPOD_TASK_TOKEN not set, exiting\n") + sys.exit(1) + server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) + sys.stderr.write(f"[task-runner] listening on :{PORT}\n") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/task_pod_smoke.py b/tests/e2e/task_pod_smoke.py new file mode 100644 index 000000000..32b628544 --- /dev/null +++ b/tests/e2e/task_pod_smoke.py @@ -0,0 +1,37 @@ +"""manual e2e smoke test for @app.task pod execution. + +deploys a real cpu pod via the env-injection bootstrap (python:3.12-slim +so the test does not depend on the runpod/task images existing yet), +executes a function, and terminates the pod. + +run directly, not via pytest: + python tests/e2e/task_pod_smoke.py +""" + +import asyncio +import sys + +from runpod.apps import App + +app = App("task-e2e") + + +@app.task(name="smoke", cpu="cpu3c-1-2", image="python:3.12-slim") +def smoke(x: int, y: int): + import platform + + return {"product": x * y, "python": platform.python_version()} + + +async def main() -> int: + print("running smoke.remote(6, 7) (deploys pod, executes, terminates) ...") + result = await smoke.remote.aio(6, 7) + print(f"result: {result}") + assert result["product"] == 42, f"unexpected result: {result}" + assert result["python"].startswith("3.12") + print("PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py new file mode 100644 index 000000000..41b4f10b1 --- /dev/null +++ b/tests/test_apps/test_task_runner_http.py @@ -0,0 +1,119 @@ +"""integration tests for the task runner http server. + +boots the real ThreadingHTTPServer on a random port and exercises the +protocol over actual sockets. +""" + +import base64 +import json +import threading +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer + +import cloudpickle +import pytest + +from runpod.runtimes.task import runner as task_runner +from runpod.runtimes.task.runner import Handler + +TOKEN = "test-token" + + +@pytest.fixture() +def server(monkeypatch): + monkeypatch.setattr(task_runner, "TOKEN", TOKEN) + monkeypatch.setattr( + task_runner, "_job_state", {"status": "NONE", "response": None} + ) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + httpd.daemon_threads = True + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{httpd.server_port}" + httpd.shutdown() + thread.join(timeout=5) + httpd.server_close() + + +def _request(url, method="GET", body=None, token=TOKEN): + req = urllib.request.Request(url, method=method) + if token: + req.add_header("Authorization", f"Bearer {token}") + data = None + if body is not None: + data = json.dumps(body).encode() + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, data=data, timeout=10) as resp: + return resp.status, json.loads(resp.read()) + + +def _b64(value): + return base64.b64encode(cloudpickle.dumps(value)).decode() + + +def _unb64(value): + return cloudpickle.loads(base64.b64decode(value)) + + +def test_ping_unauthenticated(server): + status, body = _request(f"{server}/ping", token=None) + assert status == 200 + assert body == {"ready": True} + + +def test_execute_requires_auth(server): + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(f"{server}/execute", method="POST", body={}, token="wrong") + assert exc_info.value.code == 401 + + +def test_execute_roundtrip(server): + status, body = _request( + f"{server}/execute", + method="POST", + body={ + "function_name": "mul", + "function_code": "def mul(a, b):\n return a * b", + "args": [_b64(6), _b64(7)], + "kwargs": {}, + }, + ) + assert status == 200 + assert body["success"] is True + assert _unb64(body["result"]) == 42 + + +def test_submit_and_result(server): + status, body = _request( + f"{server}/submit", + method="POST", + body={ + "function_name": "quick", + "function_code": "def quick():\n return 'done'", + "args": [], + "kwargs": {}, + "serialization_format": "json", + }, + ) + assert status == 200 + assert body == {"status": "RUNNING"} + + import time + + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + status, body = _request(f"{server}/result") + if body["status"] == "DONE": + break + time.sleep(0.1) + + assert body["status"] == "DONE" + assert body["response"]["success"] is True + assert body["response"]["json_result"] == "done" + + +def test_unknown_path_404(server): + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(f"{server}/nope") + assert exc_info.value.code == 404 diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py new file mode 100644 index 000000000..9251600b8 --- /dev/null +++ b/tests/test_apps/test_tasks.py @@ -0,0 +1,254 @@ +"""tests for the task runner and pod task transport.""" + +import base64 +import json +from unittest.mock import AsyncMock, patch + +import cloudpickle +import pytest + +import runpod +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.errors import RemoteExecutionError +from runpod.apps.spec import ResourceKind, ResourceSpec +from runpod.runtimes.task.runner import execute_request +from runpod.apps.tasks import _pod_input, unwrap_task_response +from runpod.apps.targets import PodTarget + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +def _b64(value): + return base64.b64encode(cloudpickle.dumps(value)).decode("utf-8") + + +def _unb64(value): + return cloudpickle.loads(base64.b64decode(value)) + + +class TestExecuteRequest: + def test_simple_function(self): + response = execute_request( + { + "function_name": "add", + "function_code": "def add(a, b):\n return a + b", + "args": [_b64(2), _b64(3)], + "kwargs": {}, + } + ) + assert response["success"] is True + assert _unb64(response["result"]) == 5 + + def test_json_format(self): + response = execute_request( + { + "function_name": "add", + "function_code": "def add(a, b):\n return a + b", + "args": [2, 3], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"] is True + assert response["json_result"] == 5 + + def test_async_function(self): + response = execute_request( + { + "function_name": "hello", + "function_code": ( + "async def hello(name):\n return f'hi {name}'" + ), + "args": [], + "kwargs": {"name": _b64("world")}, + } + ) + assert response["success"] is True + assert _unb64(response["result"]) == "hi world" + + def test_stdout_captured(self): + response = execute_request( + { + "function_name": "loud", + "function_code": "def loud():\n print('noise')\n return 1", + "args": [], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"] is True + assert "noise" in response["stdout"] + + def test_exception_reported(self): + response = execute_request( + { + "function_name": "boom", + "function_code": "def boom():\n raise ValueError('bad')", + "args": [], + "kwargs": {}, + } + ) + assert response["success"] is False + assert "ValueError" in response["error"] + + def test_missing_function(self): + response = execute_request( + { + "function_name": "nope", + "function_code": "x = 1", + "args": [], + "kwargs": {}, + } + ) + assert response["success"] is False + assert "not found" in response["error"] + + +class TestPodInput: + def _spec(self, **kw): + defaults = dict(kind=ResourceKind.TASK, name="t") + defaults.update(kw) + return ResourceSpec(**defaults) + + def test_cpu_pod_input(self): + pod = _pod_input(self._spec(cpu=["cpu3c-1-2"]), "tok", "t") + assert pod["instanceIds"] == ["cpu3c-1-2"] + assert pod["imageName"] == "runpod/task:py3.12-latest" + assert pod["ports"] == "8080/http" + assert pod["terminateAfter"] + env = {e["key"]: e["value"] for e in pod["env"]} + assert env["RUNPOD_TASK_TOKEN"] == "tok" + # runtime images have the runner baked in; no env payload + assert "RUNPOD_TASK_RUNNER_B64" not in env + assert "dockerArgs" not in pod + + def test_gpu_pod_input(self): + pod = _pod_input( + self._spec(gpu=["NVIDIA GeForce RTX 4090"], gpu_count=2), "tok", "t" + ) + assert pod["gpuTypeIdList"] == ["NVIDIA GeForce RTX 4090"] + assert pod["gpuCount"] == 2 + assert pod["imageName"] == "runpod/task-gpu:latest" + + def test_custom_image_bootstraps_runner(self): + pod = _pod_input( + self._spec(cpu=["cpu3c-1-2"], image="my/image:1", volume="vol-1"), + "tok", + "t", + ) + assert pod["imageName"] == "my/image:1" + assert pod["networkVolumeId"] == "vol-1" + # custom images get the env-injection fallback + env = {e["key"]: e["value"] for e in pod["env"]} + assert "RUNPOD_TASK_RUNNER_B64" in env + decoded = base64.b64decode(env["RUNPOD_TASK_RUNNER_B64"]).decode() + assert "def execute_request" in decoded + assert "task_runner.py" in pod["dockerArgs"] + + def test_datacenter_forwarded(self): + pod = _pod_input( + self._spec(cpu=["cpu3c-1-2"], datacenter=["EU-RO-1"]), "tok", "t" + ) + assert pod["dataCenterIds"] == ["EU-RO-1"] + + +class TestUnwrapTaskResponse: + def test_success_cloudpickle(self): + assert unwrap_task_response({"success": True, "result": _b64(42)}) == 42 + + def test_success_json(self): + assert ( + unwrap_task_response({"success": True, "json_result": {"a": 1}}) + == {"a": 1} + ) + + def test_failure_raises(self): + with pytest.raises(RemoteExecutionError, match="boom"): + unwrap_task_response({"success": False, "error": "boom"}) + + +class TestPodTargetPayload: + def test_payload_carries_source_and_args(self): + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2", dependencies=["numpy"]) + def t(x): + return x + + target = PodTarget(t.spec, t._fn) + payload = target.build_payload(t._fn, t.spec, (5,), {}) + assert payload["function_name"] == "t" + assert "def t(x):" in payload["function_code"] + assert _unb64(payload["args"][0]) == 5 + assert payload["dependencies"] == ["numpy"] + + def test_task_remote_runs_full_lifecycle(self): + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2") + def t(x): + return x * 2 + + with patch("runpod.apps.tasks.TaskExecution") as MockExec: + instance = MockExec.return_value + instance.start = AsyncMock() + instance.wait_ready = AsyncMock() + instance.execute = AsyncMock( + return_value={"success": True, "result": _b64(10)} + ) + instance.terminate = AsyncMock() + + result = t.remote(5) + + assert result == 10 + instance.start.assert_awaited_once() + instance.wait_ready.assert_awaited_once() + instance.terminate.assert_awaited_once() + + def test_task_terminates_pod_on_failure(self): + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2") + def t(): + pass + + with patch("runpod.apps.tasks.TaskExecution") as MockExec: + instance = MockExec.return_value + instance.start = AsyncMock() + instance.wait_ready = AsyncMock( + side_effect=TimeoutError("never ready") + ) + instance.terminate = AsyncMock() + + with pytest.raises(TimeoutError): + t.remote() + + instance.terminate.assert_awaited_once() + + def test_task_spawn_returns_job(self): + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2") + def t(): + pass + + with patch("runpod.apps.tasks.TaskExecution") as MockExec: + instance = MockExec.return_value + instance.start = AsyncMock() + instance.wait_ready = AsyncMock() + instance.submit = AsyncMock() + instance.pod_id = "pod-1" + + job = t.spawn() + + from runpod.apps.tasks import TaskJob + + assert isinstance(job, TaskJob) + assert job.pod_id == "pod-1" + instance.submit.assert_awaited_once() From a5a3a75e9ef24bf03133d92f02e30b2561abf5aa Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:17:43 -0700 Subject: [PATCH 006/120] feat: add custom image support for queue endpoints via bootstrap shim --- runpod/apps/app.py | 11 +- runpod/apps/dev.py | 30 +++++ runpod/runtimes/queue/__init__.py | 0 runpod/runtimes/queue/bootstrap.py | 208 +++++++++++++++++++++++++++++ runpod/runtimes/queue/worker.py | 121 +++++++++++++++++ tests/test_apps/test_bootstrap.py | 160 ++++++++++++++++++++++ tests/test_apps/test_tasks.py | 9 ++ 7 files changed, 538 insertions(+), 1 deletion(-) create mode 100644 runpod/runtimes/queue/__init__.py create mode 100644 runpod/runtimes/queue/bootstrap.py create mode 100644 runpod/runtimes/queue/worker.py create mode 100644 tests/test_apps/test_bootstrap.py diff --git a/runpod/apps/app.py b/runpod/apps/app.py index e45598c4c..fc5ae3249 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -89,8 +89,14 @@ def queue( volume: Optional[Any] = None, env: Optional[Dict[str, str]] = None, datacenter: Optional[Union[str, List[str]]] = None, + image: Optional[str] = None, ) -> Callable[[Callable], FunctionHandle]: - """declare a queue-based serverless endpoint from a function.""" + """declare a queue-based serverless endpoint from a function. + + image selects a custom base image for the workers; the deployed + code still arrives via the build artifact and is booted by the + runtime bootstrap, so any image with a python3 binary works. + """ def decorator(fn: Callable) -> FunctionHandle: spec = ResourceSpec( @@ -106,6 +112,7 @@ def decorator(fn: Callable) -> FunctionHandle: volume=_volume_ref(volume), env=env, datacenter=_datacenter_list(datacenter), + image=image, ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -168,6 +175,7 @@ def api( volume: Optional[Any] = None, env: Optional[Dict[str, str]] = None, datacenter: Optional[Union[str, List[str]]] = None, + image: Optional[str] = None, ) -> Callable[[Any], ApiHandle]: """declare a load-balanced serverless endpoint. @@ -205,6 +213,7 @@ def decorator(target: Any) -> ApiHandle: volume=_volume_ref(volume), env=env, datacenter=_datacenter_list(datacenter), + image=image, ) handle = ApiHandle(self, target, spec) self._register(spec.name, handle) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 9ff01fcc2..6e5aee874 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -7,8 +7,10 @@ """ import asyncio +import base64 import logging import sys +from pathlib import Path from typing import Dict, List, Optional, Union from .api import AppsApiClient @@ -37,6 +39,8 @@ def dev_endpoint_name(app_name: str, resource_name: str) -> str: def _image_for(spec: ResourceSpec) -> str: import os + if spec.image: + return spec.image override = os.getenv("RUNPOD_DEV_IMAGE") if override: return override @@ -47,6 +51,21 @@ def _image_for(spec: ResourceSpec) -> str: return image +def _bootstrap_source() -> str: + return ( + Path(__file__).parent.parent / "runtimes" / "queue" / "bootstrap.py" + ).read_text() + + +def _bootstrap_docker_args() -> str: + """shell command that materializes and starts the bootstrap on a + custom image.""" + return ( + "bash -c 'echo $RUNPOD_BOOTSTRAP_B64 | base64 -d > /bootstrap.py " + "&& python3 /bootstrap.py'" + ) + + def _cpu_locations(instance_ids: List[str]) -> str: """locations covering every requested cpu flavor's stock, limited to datacenters with storage and S3 support. the flash images are @@ -90,6 +109,17 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: ], }, } + + if spec.image: + # custom image: inject the bootstrap so the worker runtime starts + # regardless of what the image contains + payload["template"]["env"].append( + { + "key": "RUNPOD_BOOTSTRAP_B64", + "value": base64.b64encode(_bootstrap_source().encode()).decode(), + } + ) + payload["template"]["dockerArgs"] = _bootstrap_docker_args() if spec.kind is ResourceKind.API: payload["type"] = "LB" if spec.datacenter: diff --git a/runpod/runtimes/queue/__init__.py b/runpod/runtimes/queue/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/runpod/runtimes/queue/bootstrap.py b/runpod/runtimes/queue/bootstrap.py new file mode 100644 index 000000000..90fcf7bba --- /dev/null +++ b/runpod/runtimes/queue/bootstrap.py @@ -0,0 +1,208 @@ +"""stdlib-only bootstrap for queue endpoints on custom images. + +deployed code reaches the pod via the host's artifact delivery (any pod +in a flash environment gets the build tarball, regardless of image). +this shim makes that code runnable on a bare image: + + phase unpack wait for and extract the artifact into /app + phase runtime pip install the runpod package if missing + phase deps pip install this resource's manifest dependencies + phase worker exec the real worker (runpod.runtimes.queue.worker) + +if any phase fails, instead of crash-looping silently, a minimal +job-take loop starts and answers every job with the structured error so +the failure is visible in job responses, not just container logs. + +delivery: baked into the runtime images, or injected base64 in a +template env var and booted via dockerArgs on custom images. stdlib +only; must run on any image with a python3 binary. +""" + +import json +import os +import subprocess +import sys +import tarfile +import time +import urllib.request + +ARTIFACT_PATH = os.environ.get( + "FLASH_BUILD_ARTIFACT_PATH", "/root/.runpod/artifact.tar.gz" +) +APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") +ARTIFACT_WAIT_SECONDS = int(os.environ.get("RUNPOD_ARTIFACT_WAIT", "300")) +MANIFEST_NAME = "runpod_manifest.json" + + +def _log(message): + sys.stderr.write(f"[bootstrap] {message}\n") + sys.stderr.flush() + + +class PhaseError(Exception): + def __init__(self, phase, detail): + self.phase = phase + self.detail = detail + super().__init__(f"{phase}: {detail}") + + +def _unpack(): + deadline = time.monotonic() + ARTIFACT_WAIT_SECONDS + while not os.path.isfile(ARTIFACT_PATH): + if time.monotonic() >= deadline: + raise PhaseError( + "unpack", + f"artifact not found at {ARTIFACT_PATH} after " + f"{ARTIFACT_WAIT_SECONDS}s; is this endpoint part of an app " + f"deployment?", + ) + time.sleep(2) + + os.makedirs(APP_DIR, exist_ok=True) + target = os.path.realpath(APP_DIR) + try: + with tarfile.open(ARTIFACT_PATH, mode="r:*") as tar: + for member in tar.getmembers(): + dest = os.path.realpath(os.path.join(APP_DIR, member.name)) + if not dest.startswith(target + os.sep) and dest != target: + raise PhaseError( + "unpack", f"unsafe tar member path: {member.name}" + ) + tar.extractall(path=APP_DIR) + except (OSError, tarfile.TarError) as exc: + raise PhaseError("unpack", f"failed to extract artifact: {exc}") + _log(f"artifact extracted to {APP_DIR}") + + +def _pip_install(packages, phase): + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", *packages], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise PhaseError(phase, result.stderr[-3000:]) + + +def _ensure_runtime(): + try: + import runpod.runtimes.queue.worker # noqa: F401 + + return + except ImportError: + pass + # RUNPOD_PACKAGE_SPEC allows pinning or installing from git (e.g. + # prerelease testing); defaults to the published package + spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") + _log(f"worker runtime not in image, installing {spec}") + _pip_install([spec, "cloudpickle"], "runtime") + + +def _resource_entry(): + manifest_path = os.path.join(APP_DIR, MANIFEST_NAME) + if not os.path.isfile(manifest_path): + raise PhaseError("deps", f"manifest not found at {manifest_path}") + with open(manifest_path) as f: + manifest = json.load(f) + resource_name = os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( + "RUNPOD_RESOURCE_NAME", "" + ) + for entry in manifest.get("resources", []): + if entry.get("name") == resource_name: + return entry + raise PhaseError( + "deps", + f"resource '{resource_name}' not found in manifest " + f"(has: {[r.get('name') for r in manifest.get('resources', [])]})", + ) + + +def _install_deps(entry): + deps = entry.get("dependencies") or [] + if not deps: + return + _log(f"installing dependencies: {deps}") + _pip_install(deps, "deps") + + +def _run_worker(): + env = dict(os.environ, RUNPOD_APP_DIR=APP_DIR) + result = subprocess.run( + [sys.executable, "-m", "runpod.runtimes.queue.worker"], env=env + ) + sys.exit(result.returncode) + + +def _error_loop(error: PhaseError): + """answer jobs with the bootstrap error so it surfaces in responses.""" + get_url = os.environ.get("RUNPOD_WEBHOOK_GET_JOB", "") + post_url = os.environ.get("RUNPOD_WEBHOOK_POST_OUTPUT", "") + api_key = os.environ.get("RUNPOD_AI_API_KEY", "") + if not get_url or not post_url: + _log("no job webhook env; exiting with error") + sys.exit(1) + + payload = json.dumps( + { + "error": json.dumps( + { + "error_type": "BootstrapError", + "error_message": f"worker bootstrap failed during " + f"'{error.phase}': {error.detail}", + } + ) + } + ).encode() + + _log("starting error-reporting loop") + while True: + try: + req = urllib.request.Request( + get_url, headers={"Authorization": api_key} + ) + with urllib.request.urlopen(req, timeout=90) as resp: + if resp.status != 200: + time.sleep(2) + continue + job = json.loads(resp.read() or b"{}") + job_id = job.get("id") + if not job_id: + continue + done = urllib.request.Request( + post_url.replace("$ID", job_id), + data=payload, + headers={ + "Authorization": api_key, + "Content-Type": "application/x-www-form-urlencoded", + }, + method="POST", + ) + urllib.request.urlopen(done, timeout=30) + _log(f"reported bootstrap error for job {job_id}") + except Exception as exc: # noqa: BLE001 - loop must survive + _log(f"error loop: {exc}") + time.sleep(5) + + +def main(): + deployed = bool( + os.environ.get("FLASH_RESOURCE_NAME") + or os.environ.get("RUNPOD_RESOURCE_NAME") + ) + try: + if deployed: + _unpack() + _ensure_runtime() + _install_deps(_resource_entry()) + else: + # live mode (rp dev): no artifact, source ships per request + _ensure_runtime() + except PhaseError as exc: + _log(f"FAILED {exc}") + _error_loop(exc) + return + _run_worker() + + +if __name__ == "__main__": + main() diff --git a/runpod/runtimes/queue/worker.py b/runpod/runtimes/queue/worker.py new file mode 100644 index 000000000..145237cb0 --- /dev/null +++ b/runpod/runtimes/queue/worker.py @@ -0,0 +1,121 @@ +"""generic queue worker for app endpoints. + +two serving modes, chosen at startup: + +deployed mode (rp deploy): + the build artifact is unpacked at RUNPOD_APP_DIR and + FLASH_RESOURCE_NAME identifies this resource. the worker imports + the user's module, resolves the decorated function from the + manifest, runs its @init hook, and serves jobs by calling the + function body directly. job input is plain kwargs; source never + travels with requests. + +live mode (rp dev): + no artifact. each job carries a FunctionRequest (source + + serialized args) which is executed via the task runner's + execute_request. +""" + +import importlib +import inspect +import json +import os +import sys + +import runpod + +APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") +MANIFEST_NAME = "runpod_manifest.json" + + +def _resource_name() -> str: + return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( + "RUNPOD_RESOURCE_NAME", "" + ) + + +def _is_deployed() -> bool: + return bool(_resource_name()) and os.path.isfile( + os.path.join(APP_DIR, MANIFEST_NAME) + ) + + +def _load_deployed_handle(): + """import the user's module and return the FunctionHandle for this + resource.""" + with open(os.path.join(APP_DIR, MANIFEST_NAME)) as f: + manifest = json.load(f) + + name = _resource_name() + entry = next( + (r for r in manifest.get("resources", []) if r.get("name") == name), + None, + ) + if entry is None: + raise RuntimeError( + f"resource '{name}' not in manifest " + f"(has: {[r.get('name') for r in manifest.get('resources', [])]})" + ) + + if APP_DIR not in sys.path: + sys.path.insert(0, APP_DIR) + module = importlib.import_module(entry["module"]) + + from runpod.apps.handles import FunctionHandle + + for attr in vars(module).values(): + if ( + isinstance(attr, FunctionHandle) + and attr.spec.name == name + ): + return attr + raise RuntimeError( + f"no @app.queue/@app.task handle named '{name}' found in " + f"module '{entry['module']}'" + ) + + +def _make_deployed_handler(handle): + """serverless handler that calls the user function directly.""" + fn = handle._fn + + async def handler(job: dict) -> dict: + body = dict(job.get("input") or {}) + body.pop("__empty", None) + result = fn(**body) + if inspect.isawaitable(result): + result = await result + return result + + return handler + + +def _run_init(handle) -> None: + """run the resource's @init hook before serving.""" + init_fn = getattr(handle, "_init_fn", None) + if init_fn is None: + return + result = init_fn() + if inspect.isawaitable(result): + import asyncio + + asyncio.run(result) + + +def _live_handler(job: dict) -> dict: + from runpod.runtimes.task.runner import execute_request + + return execute_request(job.get("input") or {}) + + +def main() -> None: + if _is_deployed(): + handle = _load_deployed_handle() + _run_init(handle) + runpod.serverless.start({"handler": _make_deployed_handler(handle)}) + else: + runpod.serverless.start({"handler": _live_handler}) + + +if __name__ == "__main__": + main() diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py new file mode 100644 index 000000000..5b285a638 --- /dev/null +++ b/tests/test_apps/test_bootstrap.py @@ -0,0 +1,160 @@ +"""tests for the queue bootstrap and custom-image support.""" + +import base64 +import json +import textwrap + +import pytest + +import runpod +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.dev import _endpoint_input +from runpod.runtimes.queue import bootstrap + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +class TestBootstrapPhases: + def test_unpack_missing_artifact_fails_with_phase(self, tmp_path, monkeypatch): + monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(tmp_path / "nope.tar.gz")) + monkeypatch.setattr(bootstrap, "ARTIFACT_WAIT_SECONDS", 0) + with pytest.raises(bootstrap.PhaseError) as exc_info: + bootstrap._unpack() + assert exc_info.value.phase == "unpack" + + def test_unpack_extracts_artifact(self, tmp_path, monkeypatch): + import tarfile + + src = tmp_path / "src" + src.mkdir() + (src / "main.py").write_text("x = 1") + artifact = tmp_path / "artifact.tar.gz" + with tarfile.open(artifact, "w:gz") as tar: + tar.add(src / "main.py", arcname="main.py") + + app_dir = tmp_path / "app" + monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) + monkeypatch.setattr(bootstrap, "APP_DIR", str(app_dir)) + bootstrap._unpack() + assert (app_dir / "main.py").read_text() == "x = 1" + + def test_unpack_rejects_traversal(self, tmp_path, monkeypatch): + import io + import tarfile + + artifact = tmp_path / "evil.tar.gz" + with tarfile.open(artifact, "w:gz") as tar: + data = b"evil" + info = tarfile.TarInfo(name="../../etc/evil") + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + + monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) + monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path / "app")) + with pytest.raises(bootstrap.PhaseError) as exc_info: + bootstrap._unpack() + assert exc_info.value.phase == "unpack" + + def test_resource_entry_found(self, tmp_path, monkeypatch): + manifest = { + "resources": [ + {"name": "q1", "dependencies": ["numpy"]}, + {"name": "q2"}, + ] + } + (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) + monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q1") + entry = bootstrap._resource_entry() + assert entry["dependencies"] == ["numpy"] + + def test_resource_entry_missing_names_available(self, tmp_path, monkeypatch): + (tmp_path / "runpod_manifest.json").write_text( + json.dumps({"resources": [{"name": "other"}]}) + ) + monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q1") + with pytest.raises(bootstrap.PhaseError, match="other"): + bootstrap._resource_entry() + + +class TestCustomImagePayloads: + def test_queue_custom_image_gets_bootstrap(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", image="pytorch/pytorch:latest") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + template = payload["template"] + assert template["imageName"] == "pytorch/pytorch:latest" + assert "bootstrap.py" in template["dockerArgs"] + env = {e["key"]: e["value"] for e in template["env"]} + decoded = base64.b64decode(env["RUNPOD_BOOTSTRAP_B64"]).decode() + assert "def _unpack" in decoded + + def test_queue_default_image_no_bootstrap(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + template = payload["template"] + assert template["imageName"] == "runpod/flash-cpu:py3.12-latest" + assert template["dockerArgs"] == "" + env = {e["key"]: e["value"] for e in template["env"]} + assert "RUNPOD_BOOTSTRAP_B64" not in env + + def test_api_custom_image_supported(self): + app = App("a") + + @app.api(name="api", cpu="cpu3c-1-2", image="my/server:1") + class Api: + @runpod.post("/x") + def x(self, body: dict): + return body + + payload = _endpoint_input(app, Api.spec) + assert payload["template"]["imageName"] == "my/server:1" + + def test_image_in_manifest(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", image="custom:1") + def q(): + pass + + assert q.spec.to_manifest()["imageName"] == "custom:1" + + +class TestBootstrapIsStdlibOnly: + def test_no_third_party_imports(self): + import ast + from pathlib import Path + + source = ( + Path(bootstrap.__file__).read_text() + ) + tree = ast.parse(source) + # module-level imports only: function-local imports (the runpod + # availability probe in _ensure_runtime) run after installation + imports = set() + for node in tree.body: + if isinstance(node, ast.Import): + imports.update(a.name.split(".")[0] for a in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0: + imports.add(node.module.split(".")[0]) + stdlib = { + "json", "os", "subprocess", "sys", "tarfile", "time", + "urllib", "io", + } + assert imports <= stdlib, f"non-stdlib imports: {imports - stdlib}" diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 9251600b8..ab67f9b6f 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -1,5 +1,14 @@ """tests for the task runner and pod task transport.""" +# the sync bridge finalizes coroutines on a background loop thread; +# AsyncMock coroutines observed there trip the unraisable checker +# non-deterministically under full-suite gc timing +import pytest as _pytest + +pytestmark = _pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" +) + import base64 import json from unittest.mock import AsyncMock, patch From 32dea10ad1cb09451c410f76abdb37ccebc3f8a8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:19:56 -0700 Subject: [PATCH 007/120] feat: add queue runtime image and dev tag channel for runtime images --- .github/workflows/runtimes.yml | 41 ++++++++++++++++++++++++++++--- runpod/apps/dev.py | 13 +++++++--- runpod/apps/tasks.py | 11 ++++++--- runpod/runtimes/queue/Dockerfile | 16 ++++++++++++ tests/test_apps/test_bootstrap.py | 2 +- 5 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 runpod/runtimes/queue/Dockerfile diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index f41e50f36..120793f52 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -2,7 +2,7 @@ name: Runtimes on: push: - branches: [main] + branches: [main, feat/apps-sdk] paths: - "runpod/runtimes/**" - ".github/workflows/runtimes.yml" @@ -14,6 +14,8 @@ on: env: REGISTRY: docker.io + # dev-suffixed tags for non-main branches so testing never clobbers latest + TAG_SUFFIX: ${{ github.ref == 'refs/heads/main' && 'latest' || 'dev' }} jobs: test-runner: @@ -62,7 +64,7 @@ jobs: PYTHON_VERSION=${{ matrix.python-version }} push: ${{ github.event_name != 'pull_request' }} tags: | - runpod/task:py${{ matrix.python-version }}-latest + runpod/task:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} cache-from: type=gha,scope=task-py${{ matrix.python-version }} cache-to: type=gha,mode=max,scope=task-py${{ matrix.python-version }} @@ -95,6 +97,39 @@ jobs: platforms: linux/amd64 push: ${{ github.event_name != 'pull_request' }} tags: | - runpod/task-gpu:latest + runpod/task-gpu:${{ env.TAG_SUFFIX }} cache-from: type=gha,scope=task-gpu cache-to: type=gha,mode=max,scope=task-gpu + + build-queue: + runs-on: ubuntu-latest + needs: [test-runner] + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v7 + + - uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push queue image + uses: docker/build-push-action@v6 + with: + context: . + file: runpod/runtimes/queue/Dockerfile + platforms: linux/amd64 + build-args: | + PYTHON_VERSION=${{ matrix.python-version }} + push: ${{ github.event_name != 'pull_request' }} + tags: | + runpod/queue:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} + cache-from: type=gha,scope=queue-py${{ matrix.python-version }} + cache-to: type=gha,mode=max,scope=queue-py${{ matrix.python-version }} diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 6e5aee874..44d2c0ba9 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -23,10 +23,17 @@ DEV_PREFIX = "dev" -# worker runtime images by (kind, is_cpu); tag pinned via env override +# worker runtime images by (kind, is_cpu). queue images are built from +# runpod/runtimes/queue in this repo; api still rides the flash lb +# images until the lb runtime lands. RUNPOD_RUNTIME_TAG selects the +# image channel (latest, dev, or a pinned version). +import os as _os + +_TAG = _os.environ.get("RUNPOD_RUNTIME_TAG", "latest") + DEFAULT_IMAGES = { - ("queue", False): "runpod/flash:py3.12-latest", - ("queue", True): "runpod/flash-cpu:py3.12-latest", + ("queue", False): f"runpod/queue:py3.12-{_TAG}", + ("queue", True): f"runpod/queue:py3.12-{_TAG}", ("api", False): "runpod/flash-lb:py3.12-latest", ("api", True): "runpod/flash-lb-cpu:py3.12-latest", } diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 830a2f300..e612135ec 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -35,9 +35,14 @@ # safety net: pods self-terminate server-side after this long DEFAULT_MAX_LIFETIME = timedelta(hours=1) -# task runtime images built from runtimes/task by the runtimes workflow -DEFAULT_GPU_IMAGE = "runpod/task-gpu:latest" -DEFAULT_CPU_IMAGE = "runpod/task:py3.12-latest" +# task runtime images built from runpod/runtimes/task by the runtimes +# workflow. RUNPOD_RUNTIME_TAG selects the channel (latest, dev, pinned). +import os as _os + +_TAG = _os.environ.get("RUNPOD_RUNTIME_TAG", "latest") + +DEFAULT_GPU_IMAGE = f"runpod/task-gpu:{_TAG}" +DEFAULT_CPU_IMAGE = f"runpod/task:py3.12-{_TAG}" READY_POLL_INTERVAL = 2.0 READY_TIMEOUT = 600.0 diff --git a/runpod/runtimes/queue/Dockerfile b/runpod/runtimes/queue/Dockerfile new file mode 100644 index 000000000..871e0156f --- /dev/null +++ b/runpod/runtimes/queue/Dockerfile @@ -0,0 +1,16 @@ +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# install the runpod package (worker loop + runtimes) from the build context +COPY . /src +RUN pip install --no-cache-dir /src cloudpickle && rm -rf /src + +CMD ["python", "-m", "runpod.runtimes.queue.bootstrap"] diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 5b285a638..3eaede520 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -109,7 +109,7 @@ def q(): payload = _endpoint_input(app, q.spec) template = payload["template"] - assert template["imageName"] == "runpod/flash-cpu:py3.12-latest" + assert template["imageName"].startswith("runpod/queue:py3.12-") assert template["dockerArgs"] == "" env = {e["key"]: e["value"] for e in template["env"]} assert "RUNPOD_BOOTSTRAP_B64" not in env From 948e49f868388fe4057cd60766d1b70dfeda4ca3 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:20:45 -0700 Subject: [PATCH 008/120] fix: install full package deps in runtimes test job --- .github/workflows/runtimes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 120793f52..4e1508cd6 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -28,12 +28,12 @@ jobs: python-version: "3.12" - name: Install test deps - run: pip install pytest pytest-asyncio cloudpickle aiohttp requests + run: | + pip install -e . + pip install pytest pytest-asyncio cloudpickle - name: Run task runner tests - run: | - pip install -e . --no-deps - pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" + run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" build-task-cpu: runs-on: ubuntu-latest From eccea06fdae3dfd3b86a9e907cf86ace58aefc09 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:21:50 -0700 Subject: [PATCH 009/120] fix: filter lazy socket resource warnings in runner http tests --- tests/test_apps/test_task_runner_http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index 41b4f10b1..7524d3f85 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -17,6 +17,12 @@ from runpod.runtimes.task import runner as task_runner from runpod.runtimes.task.runner import Handler +# http.server's per-request sockets are collected lazily; the unraisable +# checker flags them as ResourceWarnings non-deterministically +pytestmark = pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" +) + TOKEN = "test-token" From f1814dd4423407e73668556a4a442602eeec9c92 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:23:56 -0700 Subject: [PATCH 010/120] fix: disable unraisable plugin in runtimes test job --- .github/workflows/runtimes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 4e1508cd6..b9e50c93a 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -33,7 +33,7 @@ jobs: pip install pytest pytest-asyncio cloudpickle - name: Run task runner tests - run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" + run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" -p no:unraisableexception build-task-cpu: runs-on: ubuntu-latest From ad98a6d925d71bd29f342117fe639b468c6a20be Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:42:41 -0700 Subject: [PATCH 011/120] feat: add api runtime server and unified bootstrap entrypoint --- .github/workflows/runtimes.yml | 33 ++++ runpod/apps/dev.py | 21 ++- runpod/runtimes/api/Dockerfile | 18 ++ runpod/runtimes/api/__init__.py | 0 runpod/runtimes/api/server.py | 199 +++++++++++++++++++++++ runpod/runtimes/{queue => }/bootstrap.py | 31 +++- runpod/runtimes/queue/Dockerfile | 5 +- tests/test_apps/test_api_server.py | 159 ++++++++++++++++++ tests/test_apps/test_bootstrap.py | 2 +- 9 files changed, 448 insertions(+), 20 deletions(-) create mode 100644 runpod/runtimes/api/Dockerfile create mode 100644 runpod/runtimes/api/__init__.py create mode 100644 runpod/runtimes/api/server.py rename runpod/runtimes/{queue => }/bootstrap.py (87%) create mode 100644 tests/test_apps/test_api_server.py diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index b9e50c93a..342e85c68 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -133,3 +133,36 @@ jobs: runpod/queue:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} cache-from: type=gha,scope=queue-py${{ matrix.python-version }} cache-to: type=gha,mode=max,scope=queue-py${{ matrix.python-version }} + + build-api: + runs-on: ubuntu-latest + needs: [test-runner] + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v7 + + - uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push api image + uses: docker/build-push-action@v6 + with: + context: . + file: runpod/runtimes/api/Dockerfile + platforms: linux/amd64 + build-args: | + PYTHON_VERSION=${{ matrix.python-version }} + push: ${{ github.event_name != 'pull_request' }} + tags: | + runpod/api:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} + cache-from: type=gha,scope=api-py${{ matrix.python-version }} + cache-to: type=gha,mode=max,scope=api-py${{ matrix.python-version }} diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 44d2c0ba9..23f536139 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -34,8 +34,8 @@ DEFAULT_IMAGES = { ("queue", False): f"runpod/queue:py3.12-{_TAG}", ("queue", True): f"runpod/queue:py3.12-{_TAG}", - ("api", False): "runpod/flash-lb:py3.12-latest", - ("api", True): "runpod/flash-lb-cpu:py3.12-latest", + ("api", False): f"runpod/api:py3.12-{_TAG}", + ("api", True): f"runpod/api:py3.12-{_TAG}", } @@ -60,7 +60,7 @@ def _image_for(spec: ResourceSpec) -> str: def _bootstrap_source() -> str: return ( - Path(__file__).parent.parent / "runtimes" / "queue" / "bootstrap.py" + Path(__file__).parent.parent / "runtimes" / "bootstrap.py" ).read_text() @@ -120,11 +120,16 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: if spec.image: # custom image: inject the bootstrap so the worker runtime starts # regardless of what the image contains - payload["template"]["env"].append( - { - "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode(_bootstrap_source().encode()).decode(), - } + payload["template"]["env"].extend( + [ + { + "key": "RUNPOD_BOOTSTRAP_B64", + "value": base64.b64encode( + _bootstrap_source().encode() + ).decode(), + }, + {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, + ] ) payload["template"]["dockerArgs"] = _bootstrap_docker_args() if spec.kind is ResourceKind.API: diff --git a/runpod/runtimes/api/Dockerfile b/runpod/runtimes/api/Dockerfile new file mode 100644 index 000000000..99231b951 --- /dev/null +++ b/runpod/runtimes/api/Dockerfile @@ -0,0 +1,18 @@ +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + PYTHONUNBUFFERED=1 \ + RUNPOD_RUNTIME_KIND=api + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# install the runpod package (worker loop + runtimes) from the build context +COPY . /src +RUN pip install --no-cache-dir /src cloudpickle "uvicorn>=0.30" && rm -rf /src + +EXPOSE 80 +CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/runpod/runtimes/api/__init__.py b/runpod/runtimes/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py new file mode 100644 index 000000000..c4e258676 --- /dev/null +++ b/runpod/runtimes/api/server.py @@ -0,0 +1,199 @@ +"""generic api (load-balanced) server for app endpoints. + +serves an asgi app on the port runpod's load balancer routes to +(PORT env, default 80), with /ping kept healthy for LB health checks. + +two serving modes, chosen at startup: + +deployed mode (rp deploy): + the build artifact is unpacked at RUNPOD_APP_DIR and + FLASH_RESOURCE_NAME identifies this resource. the server imports + the user's module, finds the ApiHandle, and builds the asgi app: + - class-based api: instantiate the class, run its @init method + before /ping reports healthy, mount each @get/@post route + - asgi factory: call the factory, serve what it returns + +live mode (rp dev): + no artifact. serves /execute, which runs FunctionRequest payloads + (source per request) via the task runner's execute_request. +""" + +import importlib +import inspect +import json +import logging +import os +import sys +from typing import Any, Optional + +log = logging.getLogger("runpod.runtimes.api") + +APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") +MANIFEST_NAME = "runpod_manifest.json" +PORT = int(os.environ.get("PORT", "80")) + + +def _resource_name() -> str: + return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( + "RUNPOD_RESOURCE_NAME", "" + ) + + +def _is_deployed() -> bool: + return bool(_resource_name()) and os.path.isfile( + os.path.join(APP_DIR, MANIFEST_NAME) + ) + + +def _load_api_handle(): + """import the user's module and return the ApiHandle for this resource.""" + with open(os.path.join(APP_DIR, MANIFEST_NAME)) as f: + manifest = json.load(f) + + name = _resource_name() + entry = next( + (r for r in manifest.get("resources", []) if r.get("name") == name), + None, + ) + if entry is None: + raise RuntimeError( + f"resource '{name}' not in manifest " + f"(has: {[r.get('name') for r in manifest.get('resources', [])]})" + ) + + if APP_DIR not in sys.path: + sys.path.insert(0, APP_DIR) + module = importlib.import_module(entry["module"]) + + from runpod.apps.handles import ApiHandle + + for attr in vars(module).values(): + if isinstance(attr, ApiHandle) and attr.spec.name == name: + return attr + raise RuntimeError( + f"no @app.api handle named '{name}' found in module '{entry['module']}'" + ) + + +async def _maybe_await(value: Any) -> Any: + if inspect.isawaitable(value): + return await value + return value + + +def _build_class_app(handle) -> Any: + """construct a fastapi app from an ApiHandle's decorated class. + + the class is instantiated once per worker; @init runs before /ping + reports healthy so the LB only routes to ready workers. + """ + from contextlib import asynccontextmanager + + from fastapi import FastAPI, Request + + cls = handle._cls + instance = cls() + ready = {"ok": False} + + @asynccontextmanager + async def lifespan(_app): + if handle._init_name: + await _maybe_await(getattr(instance, handle._init_name)()) + ready["ok"] = True + yield + + app = FastAPI(title=handle.spec.name, lifespan=lifespan) + + @app.get("/ping") + async def ping(): + from fastapi.responses import JSONResponse + + if not ready["ok"]: + return JSONResponse({"status": "initializing"}, status_code=204) + return {"status": "healthy"} + + for route in handle.spec.routes: + method = getattr(route, "method", None) or route["method"] + path = getattr(route, "path", None) or route["path"] + handler_name = ( + getattr(route, "handler_name", None) or route["handler"] + ) + bound = getattr(instance, handler_name) + + def make_endpoint(fn): + async def endpoint(request: Request): + body = None + if request.method in ("POST", "PUT", "PATCH", "DELETE"): + try: + body = await request.json() + except Exception: # noqa: BLE001 - empty/non-json body + body = None + if body is not None: + return await _maybe_await(fn(body)) + return await _maybe_await(fn()) + + return endpoint + + app.add_api_route( + path, make_endpoint(bound), methods=[method], name=handler_name + ) + + return app + + +def _build_factory_app(handle) -> Any: + """call the user's asgi factory and ensure /ping exists.""" + app = handle._asgi_factory() + + routes = getattr(app, "routes", []) + if not any(getattr(r, "path", None) == "/ping" for r in routes): + + @app.get("/ping") + async def ping(): + return {"status": "healthy"} + + return app + + +def _build_live_app() -> Any: + """generic /execute server for dev sessions (source per request).""" + from fastapi import FastAPI + + app = FastAPI(title="runpod-live-api") + + @app.get("/ping") + async def ping(): + return {"status": "healthy"} + + @app.post("/execute") + async def execute(request: dict): + from runpod.runtimes.task.runner import execute_request + + return execute_request(request.get("input", request)) + + return app + + +def build_app() -> Any: + if _is_deployed(): + handle = _load_api_handle() + if handle._cls is not None: + return _build_class_app(handle) + return _build_factory_app(handle) + return _build_live_app() + + +def main() -> None: + import uvicorn + + uvicorn.run( + build_app(), + host="0.0.0.0", + port=PORT, + timeout_keep_alive=600, + log_level="info", + ) + + +if __name__ == "__main__": + main() diff --git a/runpod/runtimes/queue/bootstrap.py b/runpod/runtimes/bootstrap.py similarity index 87% rename from runpod/runtimes/queue/bootstrap.py rename to runpod/runtimes/bootstrap.py index 90fcf7bba..a95016caa 100644 --- a/runpod/runtimes/queue/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -1,4 +1,4 @@ -"""stdlib-only bootstrap for queue endpoints on custom images. +"""stdlib-only bootstrap for serverless endpoints on custom images. deployed code reaches the pod via the host's artifact delivery (any pod in a flash environment gets the build tarball, regardless of image). @@ -7,15 +7,18 @@ phase unpack wait for and extract the artifact into /app phase runtime pip install the runpod package if missing phase deps pip install this resource's manifest dependencies - phase worker exec the real worker (runpod.runtimes.queue.worker) + phase worker exec the real runtime: runpod.runtimes.queue.worker + or runpod.runtimes.api.server, selected by + RUNPOD_RUNTIME_KIND (queue | api) if any phase fails, instead of crash-looping silently, a minimal job-take loop starts and answers every job with the structured error so the failure is visible in job responses, not just container logs. -delivery: baked into the runtime images, or injected base64 in a -template env var and booted via dockerArgs on custom images. stdlib -only; must run on any image with a python3 binary. +this module is the entrypoint for every serverless runtime image +(runpod/queue, runpod/api) and for custom images, where it is injected +base64 in a template env var and booted via dockerArgs. stdlib only; +must run on any image with a python3 binary. """ import json @@ -84,6 +87,10 @@ def _pip_install(packages, phase): raise PhaseError(phase, result.stderr[-3000:]) +def _runtime_kind(): + return os.environ.get("RUNPOD_RUNTIME_KIND", "queue") + + def _ensure_runtime(): try: import runpod.runtimes.queue.worker # noqa: F401 @@ -95,7 +102,10 @@ def _ensure_runtime(): # prerelease testing); defaults to the published package spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") _log(f"worker runtime not in image, installing {spec}") - _pip_install([spec, "cloudpickle"], "runtime") + packages = [spec, "cloudpickle"] + if _runtime_kind() == "api": + packages.append("uvicorn>=0.30") + _pip_install(packages, "runtime") def _resource_entry(): @@ -126,10 +136,13 @@ def _install_deps(entry): def _run_worker(): - env = dict(os.environ, RUNPOD_APP_DIR=APP_DIR) - result = subprocess.run( - [sys.executable, "-m", "runpod.runtimes.queue.worker"], env=env + module = ( + "runpod.runtimes.api.server" + if _runtime_kind() == "api" + else "runpod.runtimes.queue.worker" ) + env = dict(os.environ, RUNPOD_APP_DIR=APP_DIR) + result = subprocess.run([sys.executable, "-m", module], env=env) sys.exit(result.returncode) diff --git a/runpod/runtimes/queue/Dockerfile b/runpod/runtimes/queue/Dockerfile index 871e0156f..867636d59 100644 --- a/runpod/runtimes/queue/Dockerfile +++ b/runpod/runtimes/queue/Dockerfile @@ -3,7 +3,8 @@ FROM python:${PYTHON_VERSION}-slim ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + RUNPOD_RUNTIME_KIND=queue RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates \ @@ -13,4 +14,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY . /src RUN pip install --no-cache-dir /src cloudpickle && rm -rf /src -CMD ["python", "-m", "runpod.runtimes.queue.bootstrap"] +CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py new file mode 100644 index 000000000..84afaccec --- /dev/null +++ b/tests/test_apps/test_api_server.py @@ -0,0 +1,159 @@ +"""tests for the api runtime server.""" + +import json +import textwrap + +import pytest + +pytest.importorskip("fastapi") +from fastapi.testclient import TestClient + +import runpod +from runpod.apps.app import _clear_registry +from runpod.runtimes.api import server + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +def _write_project(tmp_path, monkeypatch, module="main_cls"): + (tmp_path / f"{module}.py").write_text( + textwrap.dedent( + """ + import runpod + from runpod import App, init, get, post + + app = App("api-test") + + @app.api(name="inference", cpu="cpu3c-1-2") + class Inference: + @init + def setup(self): + self.model = "loaded" + + @get("/health") + def health(self): + return {"model": self.model} + + @post("/generate") + async def generate(self, body: dict): + return {"echo": body, "model": self.model} + """ + ) + ) + manifest = { + "version": 1, + "app": "api-test", + "resources": [ + { + "kind": "api", + "name": "inference", + "module": module, + "routes": [ + {"method": "GET", "path": "/health", "handler": "health"}, + { + "method": "POST", + "path": "/generate", + "handler": "generate", + }, + ], + } + ], + } + (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) + monkeypatch.setattr(server, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "inference") + + +class TestDeployedClassApi: + def test_routes_and_init(self, tmp_path, monkeypatch): + _write_project(tmp_path, monkeypatch) + app = server.build_app() + with TestClient(app) as client: + # startup ran init before serving + response = client.get("/ping") + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + response = client.get("/health") + assert response.json() == {"model": "loaded"} + + response = client.post("/generate", json={"prompt": "hi"}) + assert response.json() == { + "echo": {"prompt": "hi"}, + "model": "loaded", + } + + def test_missing_resource_errors(self, tmp_path, monkeypatch): + _write_project(tmp_path, monkeypatch) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "nope") + with pytest.raises(RuntimeError, match="nope"): + server.build_app() + + +class TestFactoryApi: + def test_asgi_factory_served(self, tmp_path, monkeypatch): + (tmp_path / "main_factory.py").write_text( + textwrap.dedent( + """ + from runpod import App + + app = App("api-test") + + @app.api(name="web", cpu="cpu3c-1-2") + def web(): + from fastapi import FastAPI + + server = FastAPI() + + @server.post("/echo") + async def echo(body: dict): + return body + + return server + """ + ) + ) + manifest = { + "version": 1, + "app": "api-test", + "resources": [ + {"kind": "api", "name": "web", "module": "main_factory"} + ], + } + (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) + monkeypatch.setattr(server, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "web") + + app = server.build_app() + with TestClient(app) as client: + assert client.get("/ping").json() == {"status": "healthy"} + assert client.post("/echo", json={"a": 1}).json() == {"a": 1} + + +class TestLiveApi: + def test_execute_endpoint(self, monkeypatch): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) + app = server.build_app() + with TestClient(app) as client: + assert client.get("/ping").json() == {"status": "healthy"} + response = client.post( + "/execute", + json={ + "input": { + "function_name": "f", + "function_code": "def f(a, b):\n return a * b", + "args": [6, 7], + "kwargs": {}, + "serialization_format": "json", + } + }, + ) + data = response.json() + assert data["success"] is True + assert data["json_result"] == 42 diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 3eaede520..09a39af53 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -10,7 +10,7 @@ from runpod.apps import App from runpod.apps.app import _clear_registry from runpod.apps.dev import _endpoint_input -from runpod.runtimes.queue import bootstrap +from runpod.runtimes import bootstrap @pytest.fixture(autouse=True) From 37168864403146bed86157c95c1c8466cdad87d8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:49:24 -0700 Subject: [PATCH 012/120] fix: upgrade stale runpod installs in bootstrap and forward package spec --- runpod/apps/dev.py | 7 +++++++ runpod/apps/tasks.py | 3 +++ runpod/runtimes/bootstrap.py | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 23f536139..afd3850c9 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -131,6 +131,13 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, ] ) + # the bootstrap pip-installs the runpod package on bare images; + # forward a pinned spec (e.g. a git branch during prerelease) + package_spec = _os.environ.get("RUNPOD_PACKAGE_SPEC") + if package_spec: + payload["template"]["env"].append( + {"key": "RUNPOD_PACKAGE_SPEC", "value": package_spec} + ) payload["template"]["dockerArgs"] = _bootstrap_docker_args() if spec.kind is ResourceKind.API: payload["type"] = "LB" diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index e612135ec..12ff4bad3 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -98,6 +98,9 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] env["RUNPOD_TASK_RUNNER_B64"] = base64.b64encode( _runner_source().encode() ).decode() + package_spec = _os.environ.get("RUNPOD_PACKAGE_SPEC") + if package_spec: + env["RUNPOD_PACKAGE_SPEC"] = package_spec pod["dockerArgs"] = _bootstrap_command() pod["env"] = [{"key": k, "value": v} for k, v in env.items()] diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index a95016caa..494fcd585 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -99,13 +99,24 @@ def _ensure_runtime(): except ImportError: pass # RUNPOD_PACKAGE_SPEC allows pinning or installing from git (e.g. - # prerelease testing); defaults to the published package + # prerelease testing); defaults to the published package. --upgrade + # matters: the image may carry an older runpod without the runtimes + # modules, and a plain install would no-op against it. spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") _log(f"worker runtime not in image, installing {spec}") - packages = [spec, "cloudpickle"] + packages = ["--upgrade", spec, "cloudpickle"] if _runtime_kind() == "api": packages.append("uvicorn>=0.30") _pip_install(packages, "runtime") + try: + import runpod.runtimes.queue.worker # noqa: F401 + except ImportError as exc: + raise PhaseError( + "runtime", + f"installed {spec} but the worker runtime is still missing: " + f"{exc}. the installed runpod version may predate the runtimes " + f"modules; set RUNPOD_PACKAGE_SPEC to a version that has them.", + ) def _resource_entry(): From 62cc621d56004d2d4abe5ffcc88d61f01ab76446 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:39:03 -0700 Subject: [PATCH 013/120] feat: probe interpreters and fetch standalone python in image shim --- runpod/apps/dev.py | 7 ++-- runpod/apps/shim.py | 67 ++++++++++++++++++++++++++++++++++++ runpod/apps/tasks.py | 7 ++-- tests/test_apps/test_shim.py | 57 ++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 runpod/apps/shim.py create mode 100644 tests/test_apps/test_shim.py diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index afd3850c9..a1c1c7ecb 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -67,10 +67,9 @@ def _bootstrap_source() -> str: def _bootstrap_docker_args() -> str: """shell command that materializes and starts the bootstrap on a custom image.""" - return ( - "bash -c 'echo $RUNPOD_BOOTSTRAP_B64 | base64 -d > /bootstrap.py " - "&& python3 /bootstrap.py'" - ) + from .shim import shell_launcher + + return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") def _cpu_locations(instance_ids: List[str]) -> str: diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py new file mode 100644 index 000000000..1e9a7e394 --- /dev/null +++ b/runpod/apps/shim.py @@ -0,0 +1,67 @@ +"""shell launcher for booting injected runtime scripts on custom images. + +builds the dockerArgs command that materializes a base64 env payload +into a file and runs it. constraints, in order of hostility: + + - no bash: POSIX sh only (busybox/alpine images) + - no cpython on PATH: probe well-known interpreter locations (conda, + venv, uv, system) before giving up + - no base64 binary: the discovered python does the decode + - no python at all: download a standalone build via curl or wget + +the launcher must stay single-quote-free internally: the host parses +dockerArgs with a shell lexer and the whole script rides inside one +pair of single quotes. +""" + +# well-known interpreter locations beyond PATH +_PYTHON_CANDIDATES = ( + "python3", + "python", + "/usr/local/bin/python3", + "/usr/bin/python3", + "/opt/conda/bin/python", + "/opt/venv/bin/python", + "/venv/bin/python", + "/root/.venv/bin/python", + "/app/.venv/bin/python", + "/usr/local/bin/python", +) + +# standalone cpython for images with no interpreter at all; the musl +# variant covers alpine, detected via the musl loader path +_STANDALONE_BASE = ( + "https://github.com/astral-sh/python-build-standalone/releases/download/" + "20250409/cpython-3.12.10%2B20250409-x86_64-unknown-linux-" +) +_STANDALONE_GNU = f"{_STANDALONE_BASE}gnu-install_only.tar.gz" +_STANDALONE_MUSL = f"{_STANDALONE_BASE}musl-install_only.tar.gz" + + +def shell_launcher(env_var: str, dest: str) -> str: + """dockerArgs command that decodes $env_var into dest and execs it.""" + probes = " ".join(_PYTHON_CANDIDATES) + script = ( + f'PY=""; ' + f"for c in {probes}; do " + f'if command -v "$c" >/dev/null 2>&1; then PY="$c"; break; fi; ' + f"done; " + f'if [ -z "$PY" ]; then ' + f'echo "[shim] no python found, downloading standalone build" >&2; ' + f"if ls /lib/ld-musl-* >/dev/null 2>&1; then URL={_STANDALONE_MUSL}; " + f"else URL={_STANDALONE_GNU}; fi; " + f"mkdir -p /tmp/rp-python && cd /tmp/rp-python && " + f'( command -v curl >/dev/null 2>&1 && curl -fsSL "$URL" -o py.tgz ' + f'|| wget -q "$URL" -O py.tgz ) && ' + f"tar -xzf py.tgz && PY=/tmp/rp-python/python/bin/python3; " + f"fi; " + f'if [ -z "$PY" ]; then ' + f'echo "[shim] FATAL: no python interpreter available and no curl/wget "' + f'"to fetch one. use an image with python3, curl, or wget." >&2; ' + f"exit 1; fi; " + f'echo "${env_var}" | "$PY" -c ' + f'"import base64,sys;sys.stdout.buffer.write(base64.b64decode(sys.stdin.read()))" ' + f"> {dest} && " + f'exec "$PY" {dest}' + ) + return f"sh -c '{script}'" diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 12ff4bad3..525d050ce 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -57,10 +57,9 @@ def _runner_source() -> str: def _bootstrap_command() -> str: """shell command that materializes and starts the runner.""" - return ( - "bash -c 'echo $RUNPOD_TASK_RUNNER_B64 | base64 -d > /task_runner.py " - "&& python3 /task_runner.py'" - ) + from .shim import shell_launcher + + return shell_launcher("RUNPOD_TASK_RUNNER_B64", "/task_runner.py") def _proxy_url(pod_id: str) -> str: diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py new file mode 100644 index 000000000..3ac348e3a --- /dev/null +++ b/tests/test_apps/test_shim.py @@ -0,0 +1,57 @@ +"""tests for the shell launcher shim.""" + +import base64 +import subprocess + +import pytest + +from runpod.apps.shim import shell_launcher + + +def test_no_inner_single_quotes(): + # the host lexes dockerArgs; an inner single quote would break the + # outer quoting + cmd = shell_launcher("VAR", "/dest.py") + assert cmd.startswith("sh -c '") + assert cmd.endswith("'") + inner = cmd[len("sh -c '") : -1] + assert "'" not in inner + + +def test_posix_sh_not_bash(): + cmd = shell_launcher("VAR", "/dest.py") + assert cmd.startswith("sh -c "), "must not require bash" + + +def test_decodes_and_execs_locally(tmp_path): + """run the inner script under sh with a payload that writes a marker.""" + marker = tmp_path / "marker" + payload = f"open({str(marker)!r}, 'w').write('ran')" + b64 = base64.b64encode(payload.encode()).decode() + + dest = tmp_path / "injected.py" + cmd = shell_launcher("TESTVAR", str(dest)) + inner = cmd[len("sh -c '") : -1] + + result = subprocess.run( + ["sh", "-c", inner], + env={"TESTVAR": b64, "PATH": "/usr/bin:/bin:/usr/local/bin"}, + capture_output=True, + text=True, + timeout=30, + ) + assert result.returncode == 0, result.stderr + assert marker.read_text() == "ran" + + +def test_probes_beyond_path(): + cmd = shell_launcher("VAR", "/dest.py") + assert "/opt/conda/bin/python" in cmd + assert "/opt/venv/bin/python" in cmd + + +def test_standalone_fallback_present(): + cmd = shell_launcher("VAR", "/dest.py") + assert "python-build-standalone" in cmd + assert "musl" in cmd + assert "curl" in cmd and "wget" in cmd From b51a03a88a2e3c86158ecc8b86c9c876ddb14e23 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:41:51 -0700 Subject: [PATCH 014/120] fix: pretend scm version for tarball installs in bootstrap --- runpod/runtimes/bootstrap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index 494fcd585..44b8f7a49 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -78,10 +78,15 @@ def _unpack(): def _pip_install(packages, phase): + env = dict(os.environ) + # tarball installs (e.g. github archive urls) have no .git for + # setuptools-scm version inference; pretend a version so they build + env.setdefault("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD", "0.0.0.dev0") result = subprocess.run( [sys.executable, "-m", "pip", "install", "-q", *packages], capture_output=True, text=True, + env=env, ) if result.returncode != 0: raise PhaseError(phase, result.stderr[-3000:]) From e74b604b9b9d044c64d2bd000cd01fec7b715571 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:48:52 -0700 Subject: [PATCH 015/120] fix: fail loudly on pythonless custom images instead of fetching an interpreter --- runpod/apps/shim.py | 36 ++++++++++-------------------------- tests/test_apps/test_shim.py | 7 +++---- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py index 1e9a7e394..13cfa8f57 100644 --- a/runpod/apps/shim.py +++ b/runpod/apps/shim.py @@ -1,13 +1,15 @@ """shell launcher for booting injected runtime scripts on custom images. builds the dockerArgs command that materializes a base64 env payload -into a file and runs it. constraints, in order of hostility: +into a file and runs it with the image's python. constraints: - - no bash: POSIX sh only (busybox/alpine images) - - no cpython on PATH: probe well-known interpreter locations (conda, - venv, uv, system) before giving up - - no base64 binary: the discovered python does the decode - - no python at all: download a standalone build via curl or wget + - POSIX sh only (no bash on busybox/alpine images) + - python may not be on sh's PATH (conda/venv images), so well-known + interpreter locations are probed + - no base64 binary assumed: the discovered python does the decode + - no python at all is a configuration error: the whole feature is + running the user's python function on their image, so a pythonless + image gets a loud, clear failure the launcher must stay single-quote-free internally: the host parses dockerArgs with a shell lexer and the whole script rides inside one @@ -28,15 +30,6 @@ "/usr/local/bin/python", ) -# standalone cpython for images with no interpreter at all; the musl -# variant covers alpine, detected via the musl loader path -_STANDALONE_BASE = ( - "https://github.com/astral-sh/python-build-standalone/releases/download/" - "20250409/cpython-3.12.10%2B20250409-x86_64-unknown-linux-" -) -_STANDALONE_GNU = f"{_STANDALONE_BASE}gnu-install_only.tar.gz" -_STANDALONE_MUSL = f"{_STANDALONE_BASE}musl-install_only.tar.gz" - def shell_launcher(env_var: str, dest: str) -> str: """dockerArgs command that decodes $env_var into dest and execs it.""" @@ -47,17 +40,8 @@ def shell_launcher(env_var: str, dest: str) -> str: f'if command -v "$c" >/dev/null 2>&1; then PY="$c"; break; fi; ' f"done; " f'if [ -z "$PY" ]; then ' - f'echo "[shim] no python found, downloading standalone build" >&2; ' - f"if ls /lib/ld-musl-* >/dev/null 2>&1; then URL={_STANDALONE_MUSL}; " - f"else URL={_STANDALONE_GNU}; fi; " - f"mkdir -p /tmp/rp-python && cd /tmp/rp-python && " - f'( command -v curl >/dev/null 2>&1 && curl -fsSL "$URL" -o py.tgz ' - f'|| wget -q "$URL" -O py.tgz ) && ' - f"tar -xzf py.tgz && PY=/tmp/rp-python/python/bin/python3; " - f"fi; " - f'if [ -z "$PY" ]; then ' - f'echo "[shim] FATAL: no python interpreter available and no curl/wget "' - f'"to fetch one. use an image with python3, curl, or wget." >&2; ' + f'echo "[shim] FATAL: no python interpreter found in this image. "' + f'"custom images must include python3." >&2; ' f"exit 1; fi; " f'echo "${env_var}" | "$PY" -c ' f'"import base64,sys;sys.stdout.buffer.write(base64.b64decode(sys.stdin.read()))" ' diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py index 3ac348e3a..5b55a21ba 100644 --- a/tests/test_apps/test_shim.py +++ b/tests/test_apps/test_shim.py @@ -50,8 +50,7 @@ def test_probes_beyond_path(): assert "/opt/venv/bin/python" in cmd -def test_standalone_fallback_present(): +def test_pythonless_image_fails_loudly(): cmd = shell_launcher("VAR", "/dest.py") - assert "python-build-standalone" in cmd - assert "musl" in cmd - assert "curl" in cmd and "wget" in cmd + assert "FATAL" in cmd + assert "must include python3" in cmd From fe7a49264864cec9634fe56594fd71c6c04e621b Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:42:47 -0700 Subject: [PATCH 016/120] feat: vendor full runtime environment into artifacts at deploy time --- runpod/apps/build.py | 277 +++++++++++++++++++++++ runpod/apps/deploy.py | 115 ++++++++-- runpod/rp_cli/main.py | 20 +- runpod/runtimes/README.md | 56 +++-- runpod/runtimes/bootstrap.py | 357 +++++++++++++++++++++--------- tests/test_apps/test_bootstrap.py | 112 +++++++--- tests/test_apps/test_build.py | 159 +++++++++++++ tests/test_apps/test_deploy.py | 37 +++- 8 files changed, 961 insertions(+), 172 deletions(-) create mode 100644 runpod/apps/build.py create mode 100644 tests/test_apps/test_build.py diff --git a/runpod/apps/build.py b/runpod/apps/build.py new file mode 100644 index 000000000..eaae09174 --- /dev/null +++ b/runpod/apps/build.py @@ -0,0 +1,277 @@ +"""deploy-time environment build: vendor dependencies into the artifact. + +cold starts do no dependency resolution. everything a resource needs to +run (including the runpod runtime itself) is resolved on the build +machine into a site-packages tree that ships inside the artifact under +env/. workers put it on sys.path and start. + +the exception is size-prohibitive CUDA packages (torch and friends) +that the gpu worker images already provide: vendoring them would add +gigabytes to every artifact for nothing. they are excluded here, +recorded in the manifest, and the worker bootstrap verifies they exist +in the image at startup (installing only if genuinely absent). + +wheels are selected for the worker platform (manylinux/amd64) and the +target python version, not the build machine's, so a mac or windows +build produces a linux-correct environment. +""" + +import logging +import os +import re +import subprocess +import sys +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Tuple + +from .errors import AppError + +log = logging.getLogger(__name__) + +DEFAULT_PYTHON_VERSION = "3.12" + +# manylinux tags matching runpod worker glibc, newest first +MANYLINUX_PLATFORMS = ( + "manylinux_2_28_x86_64", + "manylinux_2_17_x86_64", + "manylinux2014_x86_64", +) + +# CUDA-scale packages provided by the gpu worker images. excluded by +# size, not by image contents: do not add packages just because an +# image happens to ship them. +SIZE_PROHIBITIVE_PACKAGES = frozenset( + { + "torch", + "torchvision", + "torchaudio", + "triton", + } +) + +# runpod's serverless limit for build artifacts +MAX_ARTIFACT_MB = 1500 + +_REQ_NAME_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)") + + +class BuildError(AppError): + """environment vendoring failed.""" + + +@dataclass +class BuildResult: + """output of build_environment.""" + + env_dir: Path + requirements: List[str] = field(default_factory=list) + excluded: List[str] = field(default_factory=list) + + +def requirement_name(requirement: str) -> str: + """distribution name from a requirement string (drops extras/pins).""" + match = _REQ_NAME_RE.match(requirement) + return (match.group(1) if match else requirement).lower().replace("_", "-") + + +def collect_requirements(project_root: Path, app) -> List[str]: + """project requirements.txt plus per-resource dependencies, deduped.""" + requirements: List[str] = [] + + req_file = project_root / "requirements.txt" + if req_file.exists(): + for line in req_file.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#") and not line.startswith("-"): + requirements.append(line) + + for handle in app.resources.values(): + for dep in handle.spec.dependencies or []: + requirements.append(dep) + + seen = set() + unique = [] + for req in requirements: + key = requirement_name(req) + if key not in seen: + seen.add(key) + unique.append(req) + return unique + + +def split_exclusions( + requirements: List[str], + extra_excludes: Optional[List[str]] = None, +) -> Tuple[List[str], List[str]]: + """(vendored, excluded-names). excluded packages resolve from the + worker image at runtime instead of the artifact.""" + excluded_names = SIZE_PROHIBITIVE_PACKAGES | { + requirement_name(e) for e in (extra_excludes or []) + } + kept: List[str] = [] + excluded: List[str] = [] + for req in requirements: + name = requirement_name(req) + if name in excluded_names: + excluded.append(name) + else: + kept.append(req) + # size-prohibitive packages are expected from the image whether or + # not the user listed them, so the manifest always records the auto + # set that user requirements referenced + return kept, sorted(set(excluded)) + + +def _is_pinnable_name(spec: str) -> bool: + """true if the spec is a plain requirement (name, optional pin), as + opposed to a url, git ref, or filesystem path.""" + return not any(token in spec for token in ("/", "\\", ":", "@")) or bool( + _REQ_NAME_RE.match(spec) and re.match(r"^[A-Za-z0-9._-]+(\[[^\]]+\])?([<>=!~;].*)?$", spec) + ) + + +def runtime_requirement(scratch_dir: Path) -> str: + """the runpod runtime spec to vendor into the artifact. + + RUNPOD_PACKAGE_SPEC overrides the published package (version pins, + git refs, tarball urls, local checkouts). non-index specs are built + into a wheel first so the platform-targeted vendoring install can + stay binary-only. + """ + spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") + if _is_pinnable_name(spec): + return spec + return str(_build_wheel(spec, scratch_dir)) + + +def _build_wheel(spec: str, scratch_dir: Path) -> Path: + """build a wheel from a url/path spec for binary-only vendoring.""" + wheel_dir = scratch_dir / "wheels" + wheel_dir.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + # tarball specs (e.g. github archive urls) have no .git for + # setuptools-scm version inference; pretend a version so they build + env.setdefault("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD", "0.0.0.dev0") + result = subprocess.run( + [ + sys.executable, + "-m", + "pip", + "wheel", + "--no-deps", + "-q", + "-w", + str(wheel_dir), + spec, + ], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + raise BuildError( + f"failed to build wheel from RUNPOD_PACKAGE_SPEC={spec!r}: " + f"{result.stderr[-2000:]}" + ) + wheels = sorted(wheel_dir.glob("runpod-*.whl")) + if not wheels: + raise BuildError( + f"pip wheel produced no runpod wheel from {spec!r} " + f"(found: {[w.name for w in wheel_dir.glob('*.whl')]})" + ) + return wheels[-1] + + +def vendor( + env_dir: Path, + requirements: List[str], + python_version: str, + *, + no_deps: bool = False, +) -> None: + """resolve requirements into env_dir for the worker platform.""" + if not requirements: + return + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "-q", + "--upgrade", + "--target", + str(env_dir), + "--python-version", + python_version, + "--implementation", + "cp", + "--only-binary", + ":all:", + ] + for platform in MANYLINUX_PLATFORMS: + cmd.extend(["--platform", platform]) + if no_deps: + cmd.append("--no-deps") + cmd.extend(requirements) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise BuildError( + f"dependency vendoring failed: {result.stderr[-3000:]}\n" + f"packages without linux wheels cannot be vendored; " + f"exclude them (they must then come from the image)" + ) + + +def _verify_runtime(env_dir: Path) -> None: + """the vendored runpod must include the worker runtimes.""" + if not (env_dir / "runpod" / "runtimes").is_dir(): + raise BuildError( + "the vendored runpod package has no runtimes modules; the " + "resolved version predates them. set RUNPOD_PACKAGE_SPEC to a " + "version that includes runpod.runtimes" + ) + + +def build_environment( + app, + project_root: Path, + *, + python_version: str = DEFAULT_PYTHON_VERSION, + exclude: Optional[List[str]] = None, + scratch_dir: Optional[Path] = None, +) -> BuildResult: + """vendor the app's full runtime environment into a directory tree.""" + scratch = scratch_dir or Path(tempfile.mkdtemp(prefix="rp-build-")) + env_dir = scratch / "env" + env_dir.mkdir(parents=True, exist_ok=True) + + user_requirements = collect_requirements(project_root, app) + vendored, excluded = split_exclusions(user_requirements, exclude) + + runtime = [runtime_requirement(scratch), "cloudpickle"] + from .spec import ResourceKind + + if any( + h.spec.kind is ResourceKind.API for h in app.resources.values() + ): + runtime.append("uvicorn>=0.30") + + all_requirements = runtime + vendored + log.info( + "vendoring %d packages for python %s (%d excluded: %s)", + len(all_requirements), + python_version, + len(excluded), + ", ".join(excluded) or "none", + ) + vendor(env_dir, all_requirements, python_version) + _verify_runtime(env_dir) + + return BuildResult( + env_dir=env_dir, + requirements=all_requirements, + excluded=excluded, + ) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 289d0e42f..d7c8e9674 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -1,12 +1,15 @@ -"""deploy pipeline: discovered apps -> manifest -> tarball -> activated build. +"""deploy pipeline: discovered apps -> manifest -> artifact -> activated build. per app: 1. validate resources 2. build the manifest from the app registry - 3. package the project into a tarball (honoring .runpodignore) - 4. get-or-create the flash app + environment - 5. upload via presigned url, finalize the build - 6. activate the build on the environment (coordinator provisions) + 3. vendor the runtime environment (runpod + all deps, resolved for + the worker platform) so cold starts install nothing + 4. package source + env + manifest into a tarball (.runpodignore + honored for source) + 5. get-or-create the flash app + environment + 6. upload via presigned url, finalize the build + 7. activate the build on the environment (coordinator provisions) """ import fnmatch @@ -20,6 +23,13 @@ from .api import AppsApiClient from .app import App +from .build import ( + DEFAULT_PYTHON_VERSION, + MAX_ARTIFACT_MB, + BuildError, + BuildResult, + build_environment, +) from .errors import ScheduleNotSupported from .schedule import SCHEDULES_ENABLED from .spec import ResourceKind @@ -42,6 +52,27 @@ ] +def _module_path_for(fn, project_root: Path) -> str: + """dotted import path for fn's file relative to the project root. + + discovery imports files under synthetic module names, so + fn.__module__ is meaningless on the worker; the import path must be + derived from where the file sits in the shipped tree. + """ + import inspect + + module = getattr(fn, "__module__", "") or "" + try: + source = Path(inspect.getfile(fn)).resolve() + rel = source.relative_to(project_root.resolve()) + except (TypeError, ValueError, OSError): + return module + parts = list(rel.with_suffix("").parts) + if parts[-1] == "__init__": + parts = parts[:-1] + return ".".join(parts) or module + + @dataclass class DeployResult: app_name: str @@ -50,25 +81,40 @@ class DeployResult: resources: List[str] -def build_manifest(app: App, project_root: Path) -> Dict[str, Any]: - """serialize an app's resources for the coordinator.""" +def build_manifest( + app: App, + project_root: Path, + *, + python_version: str = DEFAULT_PYTHON_VERSION, + excluded_packages: Optional[List[str]] = None, +) -> Dict[str, Any]: + """serialize an app's resources for the coordinator and workers.""" resources = [] for handle in app.resources.values(): spec = handle.spec if spec.schedule and not SCHEDULES_ENABLED: raise ScheduleNotSupported() entry = spec.to_manifest() - fn = getattr(handle, "_fn", None) or getattr(handle, "__wrapped__", None) + fn = ( + getattr(handle, "_fn", None) + or getattr(handle, "_cls", None) + or getattr(handle, "__wrapped__", None) + ) if fn is not None: - module = getattr(fn, "__module__", "") - entry["module"] = module + entry["module"] = _module_path_for(fn, project_root) entry["qualname"] = getattr(fn, "__qualname__", fn.__name__) resources.append(entry) - return { + manifest: Dict[str, Any] = { "version": MANIFEST_VERSION, "app": app.name, + "pythonVersion": python_version, "resources": resources, } + if excluded_packages: + # packages the build expects from the worker image rather than + # the vendored env (size-prohibitive CUDA packages) + manifest["excludedPackages"] = excluded_packages + return manifest def _load_ignores(project_root: Path) -> List[str]: @@ -92,26 +138,48 @@ def _is_ignored(rel_path: str, patterns: List[str]) -> bool: return False +ENV_DIR_NAME = "env" + + def package_project( project_root: Path, manifest: Dict[str, Any], output: Optional[Path] = None, + env_dir: Optional[Path] = None, ) -> Path: - """tar the project plus the manifest into a build artifact.""" + """tar source + vendored env + manifest into a build artifact. + + layout inside the tarball: + {source files} project code, .runpodignore honored + env/ vendored site-packages tree + runpod_manifest.json + """ if output is None: output = Path(tempfile.mkdtemp()) / "artifact.tar.gz" patterns = _load_ignores(project_root) + env_resolved = env_dir.resolve() if env_dir is not None else None with tarfile.open(output, "w:gz") as tar: for path in sorted(project_root.rglob("*")): if not path.is_file(): continue + if env_resolved is not None and env_resolved in path.resolve().parents: + continue rel = path.relative_to(project_root).as_posix() if _is_ignored(rel, patterns): continue + if rel == ENV_DIR_NAME or rel.startswith(f"{ENV_DIR_NAME}/"): + continue tar.add(path, arcname=rel) + if env_dir is not None and env_dir.is_dir(): + for path in sorted(env_dir.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(env_dir).as_posix() + tar.add(path, arcname=f"{ENV_DIR_NAME}/{rel}") + manifest_bytes = json.dumps(manifest, indent=2).encode() info = tarfile.TarInfo(name="runpod_manifest.json") info.size = len(manifest_bytes) @@ -128,15 +196,32 @@ async def deploy_app( *, env_name: Optional[str] = None, api: Optional[AppsApiClient] = None, + python_version: str = DEFAULT_PYTHON_VERSION, + exclude: Optional[List[str]] = None, ) -> DeployResult: """run the full deploy pipeline for one app.""" client = api or AppsApiClient() env_name = env_name or app.env - manifest = build_manifest(app, project_root) - tar_path = package_project(project_root, manifest) + build: BuildResult = build_environment( + app, project_root, python_version=python_version, exclude=exclude + ) + manifest = build_manifest( + app, + project_root, + python_version=python_version, + excluded_packages=build.excluded, + ) + tar_path = package_project(project_root, manifest, env_dir=build.env_dir) tar_size = tar_path.stat().st_size - log.info("packaged %s (%d bytes)", app.name, tar_size) + size_mb = tar_size / (1024 * 1024) + if size_mb > MAX_ARTIFACT_MB: + raise BuildError( + f"artifact is {size_mb:.0f} MB (limit {MAX_ARTIFACT_MB} MB). " + f"exclude large packages with --exclude (they must then come " + f"from the worker image) or trim project files via .runpodignore" + ) + log.info("packaged %s (%.1f MB)", app.name, size_mb) remote_app = await client.get_app_by_name(app.name) if remote_app is None: diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index baaa8ca01..aadc5dc9f 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -38,6 +38,18 @@ def deploy( env: Optional[str] = typer.Option( None, "--env", "-e", help="Target environment name." ), + python_version: str = typer.Option( + "3.12", + "--python-version", + help="Worker python version for dependency wheels (3.10-3.12).", + ), + exclude: Optional[str] = typer.Option( + None, + "--exclude", + help="Comma-separated packages to exclude from the artifact " + "(they must then come from the worker image). torch packages " + "are always excluded.", + ), ): """package and deploy all apps found in TARGET.""" from runpod.apps.deploy import deploy_app @@ -66,7 +78,13 @@ def deploy( names = ", ".join(found.resources) or "(no resources)" _echo(f"deploying app '{found.name}': {names}") result = asyncio.run( - deploy_app(found, project_root, env_name=env) + deploy_app( + found, + project_root, + env_name=env, + python_version=python_version, + exclude=exclude.split(",") if exclude else None, + ) ) _echo( f" build {result.build_id} active on " diff --git a/runpod/runtimes/README.md b/runpod/runtimes/README.md index 148bc1581..2918e5a82 100644 --- a/runpod/runtimes/README.md +++ b/runpod/runtimes/README.md @@ -1,27 +1,57 @@ # runtimes worker runtime scripts and the docker images built from them. images -are built and pushed by the `runtimes` github actions workflow on -release. +are built and pushed by the `runtimes` github actions workflow. -## task +## cold-start model -single-shot task runner for `@app.task` pods. one pod runs one -function: the sdk deploys a pod on one of these images, waits for -`/ping`, posts a `FunctionRequest` to `/execute` (or `/submit` for -spawn), collects the result, and terminates the pod. +deployed workers do no dependency resolution. `rp deploy` vendors the +full runtime environment (the runpod package plus every python +dependency, resolved for the worker platform) into the build artifact +under `env/`. at cold start the bootstrap: + +1. **locate** the app tree: a host-provided pre-unpacked directory if + available, else extract the artifact tarball once (marker file + short-circuits warm restarts) +2. **attach** `env/` and the source tree to the worker's PYTHONPATH, + vendored env first so it wins over image packages +3. **verify** size-excluded packages (torch and friends) exist in the + image, installing only if genuinely absent (loudly, as a fallback) +4. **serve** exec the worker runtime + +torch-family packages are excluded from artifacts by size and expected +from the gpu worker images; the exclusion list is recorded in the +manifest as `excludedPackages`. + +if any phase fails, the worker does not crash-loop silently: queue +workers answer jobs with a structured `BootstrapError`, api workers +answer http requests with it. + +live mode (`rp dev`) has no artifact; source arrives per request and +the bootstrap only ensures the runtime package is importable (a no-op +on the baked images). + +## images | image | base | use | |---|---|---| +| `runpod/queue:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | queue endpoints | +| `runpod/api:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | load-balanced api endpoints | | `runpod/task:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | cpu tasks | | `runpod/task-gpu:{tag}` | `runpod/pytorch` | gpu tasks | -`runner.py` is stdlib-only (cloudpickle optional, used for argument -and result serialization when present). this keeps the env-injection -fallback working: when a user supplies a custom `image=` that does not -have the runner baked in, the sdk ships `runner.py` base64-encoded in -a pod env var and boots it via `dockerArgs`, so any image with a -python3 binary works. +custom images work everywhere: the sdk injects `bootstrap.py` (queue/ +api) or `runner.py` (task) base64-encoded in an env var and boots it +via `dockerArgs`. both scripts are stdlib-only, so any image with a +python3 binary works. for deployed resources the vendored env then +provides everything else, including the runpod package itself. + +## task + +single-shot task runner for `@app.task` pods. one pod runs one +function: the sdk deploys a pod, waits for `/ping`, posts a +`FunctionRequest` to `/execute` (or `/submit` for spawn), collects the +result, and terminates the pod. the wire protocol (`FunctionRequest` / `FunctionResponse`) is defined in `runpod.apps.protocol`; the runner and sdk must stay in sync with diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index 44b8f7a49..7f51a34bb 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -1,26 +1,36 @@ -"""stdlib-only bootstrap for serverless endpoints on custom images. - -deployed code reaches the pod via the host's artifact delivery (any pod -in a flash environment gets the build tarball, regardless of image). -this shim makes that code runnable on a bare image: - - phase unpack wait for and extract the artifact into /app - phase runtime pip install the runpod package if missing - phase deps pip install this resource's manifest dependencies - phase worker exec the real runtime: runpod.runtimes.queue.worker - or runpod.runtimes.api.server, selected by - RUNPOD_RUNTIME_KIND (queue | api) - -if any phase fails, instead of crash-looping silently, a minimal -job-take loop starts and answers every job with the structured error so -the failure is visible in job responses, not just container logs. - -this module is the entrypoint for every serverless runtime image -(runpod/queue, runpod/api) and for custom images, where it is injected -base64 in a template env var and booted via dockerArgs. stdlib only; -must run on any image with a python3 binary. +"""stdlib-only entrypoint for app workers. + +cold starts do no dependency resolution: the build artifact ships a +vendored environment (env/ inside the artifact) containing the runpod +runtime and every python dependency, resolved at deploy time for the +worker platform. this entrypoint only has to make that environment +visible and exec the worker. + +deployed mode (FLASH_RESOURCE_NAME set): + locate find the app tree: a pre-unpacked directory if the host + provides one, else extract the artifact tarball once + attach put {app} and {app}/env at the front of PYTHONPATH + verify packages the build excluded by size (torch and friends) + must come from the image; genuinely absent ones are + installed as a fallback, loudly + serve exec the worker runtime from the vendored environment + +live mode (no resource name, `rp dev`): + no artifact exists; source arrives per request. the runtime images + bake the runpod package in, custom images get it installed here. + serve exec the worker runtime + +if any phase fails the worker does not crash-loop silently: a minimal +job-take loop answers every job with a structured BootstrapError so +the failure surfaces in job responses (queue endpoints), or an http +server answers every request with the error (api endpoints). + +stdlib only: this file runs on the image's python before any +environment exists. it is baked into the runtime images as CMD and +injected into custom images via RUNPOD_BOOTSTRAP_B64 + dockerArgs. """ +import importlib.util import json import os import subprocess @@ -32,9 +42,19 @@ ARTIFACT_PATH = os.environ.get( "FLASH_BUILD_ARTIFACT_PATH", "/root/.runpod/artifact.tar.gz" ) +# a host-provided, already-unpacked app tree; when present, unpacking +# is skipped entirely +PREBUILT_APP_DIR = os.environ.get("RUNPOD_PREBUILT_APP_DIR", "") APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") +ENV_SUBDIR = "env" ARTIFACT_WAIT_SECONDS = int(os.environ.get("RUNPOD_ARTIFACT_WAIT", "300")) MANIFEST_NAME = "runpod_manifest.json" +UNPACK_MARKER = ".rp-unpacked" + +WORKER_MODULES = { + "queue": "runpod.runtimes.queue.worker", + "api": "runpod.runtimes.api.server", +} def _log(message): @@ -49,41 +69,118 @@ def __init__(self, phase, detail): super().__init__(f"{phase}: {detail}") -def _unpack(): +def _runtime_kind(): + return os.environ.get("RUNPOD_RUNTIME_KIND", "queue") + + +def _resource_name(): + return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( + "RUNPOD_RESOURCE_NAME", "" + ) + + +# ---------------------------------------------------------------- locate + + +def _locate(): + """return the app tree root, unpacking the artifact if needed.""" + if PREBUILT_APP_DIR and os.path.isdir(PREBUILT_APP_DIR): + _log(f"using pre-unpacked app tree at {PREBUILT_APP_DIR}") + return PREBUILT_APP_DIR + + marker = os.path.join(APP_DIR, UNPACK_MARKER) + if os.path.isfile(marker): + _log(f"app tree already unpacked at {APP_DIR}") + return APP_DIR + deadline = time.monotonic() + ARTIFACT_WAIT_SECONDS while not os.path.isfile(ARTIFACT_PATH): if time.monotonic() >= deadline: raise PhaseError( - "unpack", + "locate", f"artifact not found at {ARTIFACT_PATH} after " - f"{ARTIFACT_WAIT_SECONDS}s; is this endpoint part of an app " - f"deployment?", + f"{ARTIFACT_WAIT_SECONDS}s; is this endpoint part of an " + f"app deployment?", ) time.sleep(2) os.makedirs(APP_DIR, exist_ok=True) target = os.path.realpath(APP_DIR) + started = time.monotonic() try: with tarfile.open(ARTIFACT_PATH, mode="r:*") as tar: for member in tar.getmembers(): dest = os.path.realpath(os.path.join(APP_DIR, member.name)) if not dest.startswith(target + os.sep) and dest != target: raise PhaseError( - "unpack", f"unsafe tar member path: {member.name}" + "locate", f"unsafe tar member path: {member.name}" ) tar.extractall(path=APP_DIR) except (OSError, tarfile.TarError) as exc: - raise PhaseError("unpack", f"failed to extract artifact: {exc}") - _log(f"artifact extracted to {APP_DIR}") + raise PhaseError("locate", f"failed to extract artifact: {exc}") + + with open(marker, "w") as f: + f.write(str(time.time())) + _log( + f"artifact extracted to {APP_DIR} " + f"in {time.monotonic() - started:.1f}s" + ) + return APP_DIR + + +# ---------------------------------------------------------------- attach + + +def _attach(app_dir): + """paths that must lead sys.path in the worker process. + + the vendored env comes first so the artifact's runpod (matched to + the client that built it) wins over anything baked into the image. + """ + env_dir = os.path.join(app_dir, ENV_SUBDIR) + paths = [app_dir] + if os.path.isdir(env_dir): + paths.insert(0, env_dir) + else: + _log( + f"artifact has no vendored env at {env_dir}; " + f"falling back to image packages" + ) + return paths + + +def _manifest(app_dir): + path = os.path.join(app_dir, MANIFEST_NAME) + if not os.path.isfile(path): + raise PhaseError("attach", f"manifest not found at {path}") + with open(path) as f: + return json.load(f) + + +def _worker_importable(paths): + """can the worker module be imported with these paths leading?""" + module = WORKER_MODULES[_runtime_kind()] + probe = ( + "import sys; sys.path[:0] = sys.argv[1:]; " + f"import importlib; importlib.import_module({module!r})" + ) + result = subprocess.run( + [sys.executable, "-c", probe, *paths], + capture_output=True, + text=True, + ) + return result.returncode == 0, result.stderr[-2000:] + + +# ---------------------------------------------------------------- verify def _pip_install(packages, phase): env = dict(os.environ) - # tarball installs (e.g. github archive urls) have no .git for - # setuptools-scm version inference; pretend a version so they build + # tarball specs have no .git for setuptools-scm version inference env.setdefault("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD", "0.0.0.dev0") result = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", *packages], + [sys.executable, "-m", "pip", "install", "-q", "--upgrade", *packages], capture_output=True, text=True, env=env, @@ -92,78 +189,92 @@ def _pip_install(packages, phase): raise PhaseError(phase, result.stderr[-3000:]) -def _runtime_kind(): - return os.environ.get("RUNPOD_RUNTIME_KIND", "queue") +def _verify_excluded(manifest): + """packages the build excluded must come from the image. + + a miss means the image was swapped for one without them; install as + a fallback so the worker still comes up, but say so loudly because + it re-adds the cold-start cost the exclusion existed to avoid. + """ + excluded = manifest.get("excludedPackages") or [] + missing = [ + name + for name in excluded + if importlib.util.find_spec(name.replace("-", "_")) is None + ] + if not missing: + return + _log( + f"WARNING: excluded packages {missing} are not in this image; " + f"installing at cold start. use an image that provides them " + f"(e.g. runpod/pytorch) to avoid this cost." + ) + _pip_install(missing, "verify") -def _ensure_runtime(): - try: - import runpod.runtimes.queue.worker # noqa: F401 +# ---------------------------------------------------------------- serve + + +def _serve(paths): + """exec the worker runtime with the environment paths leading.""" + module = WORKER_MODULES[_runtime_kind()] + env = dict(os.environ) + existing = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + paths + ([existing] if existing else []) + ) + if paths: + env["RUNPOD_APP_DIR"] = paths[-1] + os.execve( + sys.executable, + [sys.executable, "-m", module], + env, + ) + + +# ------------------------------------------------------------- live mode + +def _ensure_runtime_installed(): + """live mode on a custom image: the runtime must be importable. + + baked runtime images make this a no-op. --upgrade matters: the + image may carry an older runpod without the runtimes modules, and + a plain install would no-op against it. + """ + ok, _ = _worker_importable([]) + if ok: return - except ImportError: - pass - # RUNPOD_PACKAGE_SPEC allows pinning or installing from git (e.g. - # prerelease testing); defaults to the published package. --upgrade - # matters: the image may carry an older runpod without the runtimes - # modules, and a plain install would no-op against it. spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") _log(f"worker runtime not in image, installing {spec}") - packages = ["--upgrade", spec, "cloudpickle"] + packages = [spec, "cloudpickle"] if _runtime_kind() == "api": packages.append("uvicorn>=0.30") _pip_install(packages, "runtime") - try: - import runpod.runtimes.queue.worker # noqa: F401 - except ImportError as exc: + ok, err = _worker_importable([]) + if not ok: raise PhaseError( "runtime", - f"installed {spec} but the worker runtime is still missing: " - f"{exc}. the installed runpod version may predate the runtimes " - f"modules; set RUNPOD_PACKAGE_SPEC to a version that has them.", + f"installed {spec} but the worker runtime is still not " + f"importable: {err}. set RUNPOD_PACKAGE_SPEC to a version " + f"that includes runpod.runtimes.", ) -def _resource_entry(): - manifest_path = os.path.join(APP_DIR, MANIFEST_NAME) - if not os.path.isfile(manifest_path): - raise PhaseError("deps", f"manifest not found at {manifest_path}") - with open(manifest_path) as f: - manifest = json.load(f) - resource_name = os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( - "RUNPOD_RESOURCE_NAME", "" - ) - for entry in manifest.get("resources", []): - if entry.get("name") == resource_name: - return entry - raise PhaseError( - "deps", - f"resource '{resource_name}' not found in manifest " - f"(has: {[r.get('name') for r in manifest.get('resources', [])]})", - ) - - -def _install_deps(entry): - deps = entry.get("dependencies") or [] - if not deps: - return - _log(f"installing dependencies: {deps}") - _pip_install(deps, "deps") +# ----------------------------------------------------------- error surface -def _run_worker(): - module = ( - "runpod.runtimes.api.server" - if _runtime_kind() == "api" - else "runpod.runtimes.queue.worker" - ) - env = dict(os.environ, RUNPOD_APP_DIR=APP_DIR) - result = subprocess.run([sys.executable, "-m", module], env=env) - sys.exit(result.returncode) +def _error_payload(error): + return { + "error_type": "BootstrapError", + "error_message": ( + f"worker bootstrap failed during '{error.phase}': {error.detail}" + ), + } -def _error_loop(error: PhaseError): - """answer jobs with the bootstrap error so it surfaces in responses.""" +def _queue_error_loop(error): + """answer queue jobs with the bootstrap error.""" get_url = os.environ.get("RUNPOD_WEBHOOK_GET_JOB", "") post_url = os.environ.get("RUNPOD_WEBHOOK_POST_OUTPUT", "") api_key = os.environ.get("RUNPOD_AI_API_KEY", "") @@ -171,17 +282,7 @@ def _error_loop(error: PhaseError): _log("no job webhook env; exiting with error") sys.exit(1) - payload = json.dumps( - { - "error": json.dumps( - { - "error_type": "BootstrapError", - "error_message": f"worker bootstrap failed during " - f"'{error.phase}': {error.detail}", - } - ) - } - ).encode() + payload = json.dumps({"error": json.dumps(_error_payload(error))}).encode() _log("starting error-reporting loop") while True: @@ -213,24 +314,60 @@ def _error_loop(error: PhaseError): time.sleep(5) +def _api_error_server(error): + """answer every http request with the bootstrap error.""" + from http.server import BaseHTTPRequestHandler, HTTPServer + + body = json.dumps(_error_payload(error)).encode() + port = int(os.environ.get("PORT", "80")) + + class Handler(BaseHTTPRequestHandler): + def _respond(self): + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = _respond + + def log_message(self, *args): + pass + + _log(f"starting error-reporting http server on :{port}") + HTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +def _report_error(error): + _log(f"FAILED {error}") + if _runtime_kind() == "api": + _api_error_server(error) + else: + _queue_error_loop(error) + + +# ------------------------------------------------------------------ main + + def main(): - deployed = bool( - os.environ.get("FLASH_RESOURCE_NAME") - or os.environ.get("RUNPOD_RESOURCE_NAME") - ) try: - if deployed: - _unpack() - _ensure_runtime() - _install_deps(_resource_entry()) + if _resource_name(): + app_dir = _locate() + paths = _attach(app_dir) + _verify_excluded(_manifest(app_dir)) + ok, err = _worker_importable(paths) + if not ok: + raise PhaseError( + "attach", + f"worker runtime not importable from the artifact " + f"env or the image: {err}", + ) + _serve(paths) else: - # live mode (rp dev): no artifact, source ships per request - _ensure_runtime() + _ensure_runtime_installed() + _serve([]) except PhaseError as exc: - _log(f"FAILED {exc}") - _error_loop(exc) - return - _run_worker() + _report_error(exc) if __name__ == "__main__": diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 09a39af53..83f90982b 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -20,15 +20,16 @@ def clean_registry(): _clear_registry() -class TestBootstrapPhases: - def test_unpack_missing_artifact_fails_with_phase(self, tmp_path, monkeypatch): +class TestLocatePhase: + def test_missing_artifact_fails_with_phase(self, tmp_path, monkeypatch): monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(tmp_path / "nope.tar.gz")) monkeypatch.setattr(bootstrap, "ARTIFACT_WAIT_SECONDS", 0) + monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path / "app")) with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._unpack() - assert exc_info.value.phase == "unpack" + bootstrap._locate() + assert exc_info.value.phase == "locate" - def test_unpack_extracts_artifact(self, tmp_path, monkeypatch): + def test_extracts_artifact(self, tmp_path, monkeypatch): import tarfile src = tmp_path / "src" @@ -41,10 +42,36 @@ def test_unpack_extracts_artifact(self, tmp_path, monkeypatch): app_dir = tmp_path / "app" monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) monkeypatch.setattr(bootstrap, "APP_DIR", str(app_dir)) - bootstrap._unpack() + assert bootstrap._locate() == str(app_dir) assert (app_dir / "main.py").read_text() == "x = 1" - def test_unpack_rejects_traversal(self, tmp_path, monkeypatch): + def test_unpack_happens_once(self, tmp_path, monkeypatch): + import tarfile + + src = tmp_path / "src" + src.mkdir() + (src / "main.py").write_text("x = 1") + artifact = tmp_path / "artifact.tar.gz" + with tarfile.open(artifact, "w:gz") as tar: + tar.add(src / "main.py", arcname="main.py") + + app_dir = tmp_path / "app" + monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) + monkeypatch.setattr(bootstrap, "APP_DIR", str(app_dir)) + bootstrap._locate() + # second boot on a warm container: marker short-circuits + artifact.unlink() + assert bootstrap._locate() == str(app_dir) + + def test_prebuilt_dir_skips_unpack(self, tmp_path, monkeypatch): + prebuilt = tmp_path / "prebuilt" + prebuilt.mkdir() + monkeypatch.setattr(bootstrap, "PREBUILT_APP_DIR", str(prebuilt)) + # no artifact anywhere; the host-provided tree wins outright + monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(tmp_path / "no.tar.gz")) + assert bootstrap._locate() == str(prebuilt) + + def test_rejects_traversal(self, tmp_path, monkeypatch): import io import tarfile @@ -58,30 +85,55 @@ def test_unpack_rejects_traversal(self, tmp_path, monkeypatch): monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path / "app")) with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._unpack() - assert exc_info.value.phase == "unpack" - - def test_resource_entry_found(self, tmp_path, monkeypatch): - manifest = { - "resources": [ - {"name": "q1", "dependencies": ["numpy"]}, - {"name": "q2"}, - ] - } - (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) - monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "q1") - entry = bootstrap._resource_entry() - assert entry["dependencies"] == ["numpy"] + bootstrap._locate() + assert exc_info.value.phase == "locate" + + +class TestAttachPhase: + def test_env_dir_leads_path_order(self, tmp_path): + (tmp_path / "env").mkdir() + paths = bootstrap._attach(str(tmp_path)) + assert paths == [str(tmp_path / "env"), str(tmp_path)] - def test_resource_entry_missing_names_available(self, tmp_path, monkeypatch): + def test_no_env_dir_falls_back_to_source_only(self, tmp_path): + paths = bootstrap._attach(str(tmp_path)) + assert paths == [str(tmp_path)] + + def test_manifest_read(self, tmp_path): (tmp_path / "runpod_manifest.json").write_text( - json.dumps({"resources": [{"name": "other"}]}) + json.dumps({"resources": [{"name": "q1"}]}) + ) + manifest = bootstrap._manifest(str(tmp_path)) + assert manifest["resources"][0]["name"] == "q1" + + def test_manifest_missing_fails_with_phase(self, tmp_path): + with pytest.raises(bootstrap.PhaseError) as exc_info: + bootstrap._manifest(str(tmp_path)) + assert exc_info.value.phase == "attach" + + +class TestVerifyPhase: + def test_no_exclusions_no_op(self): + bootstrap._verify_excluded({"resources": []}) + + def test_present_exclusions_no_install(self, monkeypatch): + installed = [] + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: installed.extend(pkgs) + ) + # json is definitely importable; stands in for torch-in-image + bootstrap._verify_excluded({"excludedPackages": ["json"]}) + assert installed == [] + + def test_missing_exclusions_installed(self, monkeypatch): + installed = [] + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: installed.extend(pkgs) + ) + bootstrap._verify_excluded( + {"excludedPackages": ["definitely-not-installed-xyz"]} ) - monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "q1") - with pytest.raises(bootstrap.PhaseError, match="other"): - bootstrap._resource_entry() + assert installed == ["definitely-not-installed-xyz"] class TestCustomImagePayloads: @@ -98,7 +150,7 @@ def q(): assert "bootstrap.py" in template["dockerArgs"] env = {e["key"]: e["value"] for e in template["env"]} decoded = base64.b64decode(env["RUNPOD_BOOTSTRAP_B64"]).decode() - assert "def _unpack" in decoded + assert "def _locate" in decoded def test_queue_default_image_no_bootstrap(self): app = App("a") @@ -155,6 +207,6 @@ def test_no_third_party_imports(self): imports.add(node.module.split(".")[0]) stdlib = { "json", "os", "subprocess", "sys", "tarfile", "time", - "urllib", "io", + "urllib", "io", "importlib", "http", } assert imports <= stdlib, f"non-stdlib imports: {imports - stdlib}" diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py new file mode 100644 index 000000000..add1a753a --- /dev/null +++ b/tests/test_apps/test_build.py @@ -0,0 +1,159 @@ +"""tests for deploy-time environment vendoring.""" + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.build import ( + SIZE_PROHIBITIVE_PACKAGES, + BuildError, + collect_requirements, + requirement_name, + runtime_requirement, + split_exclusions, + vendor, +) + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +class TestRequirementName: + @pytest.mark.parametrize( + "spec,name", + [ + ("numpy", "numpy"), + ("numpy==1.26.0", "numpy"), + ("Torch>=2.0", "torch"), + ("scikit_learn~=1.4", "scikit-learn"), + ("pillow[extras]==10", "pillow"), + ], + ) + def test_extracts_distribution_name(self, spec, name): + assert requirement_name(spec) == name + + +class TestCollectRequirements: + def test_merges_requirements_txt_and_decorator_deps(self, tmp_path): + (tmp_path / "requirements.txt").write_text( + "numpy==1.26\n# comment\n\npandas\n" + ) + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", dependencies=["requests", "numpy"]) + def q(): + pass + + reqs = collect_requirements(tmp_path, app) + # numpy deduped by name, requirements.txt wins on order + assert reqs == ["numpy==1.26", "pandas", "requests"] + + def test_no_requirements_file(self, tmp_path): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + assert collect_requirements(tmp_path, app) == [] + + +class TestSplitExclusions: + def test_torch_family_auto_excluded(self): + kept, excluded = split_exclusions( + ["numpy", "torch==2.4", "torchvision", "requests"] + ) + assert kept == ["numpy", "requests"] + assert excluded == ["torch", "torchvision"] + + def test_user_excludes_added(self): + kept, excluded = split_exclusions( + ["numpy", "transformers"], extra_excludes=["transformers"] + ) + assert kept == ["numpy"] + assert excluded == ["transformers"] + + def test_exclusion_set_is_size_based(self): + # sanity-pin the auto set; additions must be size-justified + assert SIZE_PROHIBITIVE_PACKAGES == { + "torch", + "torchvision", + "torchaudio", + "triton", + } + + +class TestRuntimeRequirement: + def test_default_is_published_package(self, tmp_path, monkeypatch): + monkeypatch.delenv("RUNPOD_PACKAGE_SPEC", raising=False) + assert runtime_requirement(tmp_path) == "runpod" + + def test_version_pin_passes_through(self, tmp_path, monkeypatch): + monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==1.8.0") + assert runtime_requirement(tmp_path) == "runpod==1.8.0" + + def test_url_spec_builds_wheel(self, tmp_path, monkeypatch): + monkeypatch.setenv( + "RUNPOD_PACKAGE_SPEC", "https://example.com/runpod.tar.gz" + ) + wheel = tmp_path / "wheels" / "runpod-0.0.0.dev0-py3-none-any.whl" + + def fake_run(cmd, **kwargs): + wheel.parent.mkdir(parents=True, exist_ok=True) + wheel.write_bytes(b"") + + class R: + returncode = 0 + stderr = "" + + return R() + + with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): + assert runtime_requirement(tmp_path) == str(wheel) + + +class TestVendor: + def test_targets_worker_platform(self, tmp_path): + captured = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = cmd + + class R: + returncode = 0 + stderr = "" + + return R() + + with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): + vendor(tmp_path, ["numpy"], "3.12") + + cmd = captured["cmd"] + assert "--target" in cmd + assert "--python-version" in cmd and "3.12" in cmd + assert "--only-binary" in cmd + assert "manylinux_2_28_x86_64" in cmd + + def test_empty_requirements_no_op(self, tmp_path): + with patch("runpod.apps.build.subprocess.run") as run: + vendor(tmp_path, [], "3.12") + run.assert_not_called() + + def test_failure_raises_build_error(self, tmp_path): + def fake_run(cmd, **kwargs): + class R: + returncode = 1 + stderr = "no matching distribution" + + return R() + + with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): + with pytest.raises(BuildError, match="no matching distribution"): + vendor(tmp_path, ["nonexistent-xyz"], "3.12") diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index ed1a3b21d..37a986bee 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -5,7 +5,7 @@ import tarfile import textwrap from pathlib import Path -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, patch import pytest @@ -144,6 +144,35 @@ def test_ignores_applied(self, tmp_path): assert "secret.env" not in names assert not any("__pycache__" in n for n in names) + def test_vendored_env_included_under_env(self, tmp_path): + _write_project(tmp_path) + env_dir = tmp_path / "built-env" + (env_dir / "numpy").mkdir(parents=True) + (env_dir / "numpy" / "__init__.py").write_text("") + + tar_path = package_project( + tmp_path, {"version": 1, "resources": []}, env_dir=env_dir + ) + with tarfile.open(tar_path) as tar: + names = tar.getnames() + assert "env/numpy/__init__.py" in names + # env dir under project root must not be double-added as source + assert "built-env/numpy/__init__.py" not in names + assert "main.py" in names + + +def _stub_build(tmp_path): + """patch environment vendoring with a tiny fake env tree.""" + from runpod.apps.build import BuildResult + + env_dir = tmp_path / "fake-env" + env_dir.mkdir(exist_ok=True) + (env_dir / "vendored_pkg.py").write_text("x = 1") + return patch( + "runpod.apps.deploy.build_environment", + return_value=BuildResult(env_dir=env_dir, requirements=["runpod"]), + ) + class TestDeployPipeline: async def test_deploy_app_full_flow(self, tmp_path): @@ -161,7 +190,8 @@ async def test_deploy_app_full_flow(self, tmp_path): api.finalize_artifact_upload.return_value = {"id": "build-1"} api.deploy_build.return_value = {"id": "env-1"} - result = await deploy_app(app, tmp_path, api=api) + with _stub_build(tmp_path): + result = await deploy_app(app, tmp_path, api=api) assert isinstance(result, DeployResult) assert result.build_id == "build-1" @@ -184,7 +214,8 @@ async def test_deploy_reuses_existing_app_and_env(self, tmp_path): } api.finalize_artifact_upload.return_value = {"id": "build-2"} - await deploy_app(app, tmp_path, api=api) + with _stub_build(tmp_path): + await deploy_app(app, tmp_path, api=api) api.create_app.assert_not_awaited() api.create_environment.assert_not_awaited() From 55dd9d44686ba5c8cb18301aa061ef90a1c1f8ef Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:58:44 -0700 Subject: [PATCH 017/120] feat: support python 3.10-3.14 with per-version gpu runtime images --- .github/workflows/runtimes.yml | 116 ++++++++--------------- runpod/apps/build.py | 34 +++++-- runpod/apps/dev.py | 28 ++---- runpod/apps/images.py | 84 ++++++++++++++++ runpod/apps/tasks.py | 14 +-- runpod/rp_cli/main.py | 2 +- runpod/runtimes/README.md | 26 ++++- runpod/runtimes/api/Dockerfile | 6 +- runpod/runtimes/bootstrap.py | 51 +++++++++- runpod/runtimes/gpu-base/Dockerfile | 18 ++++ runpod/runtimes/queue/Dockerfile | 6 +- runpod/runtimes/task/Dockerfile | 6 +- runpod/runtimes/task/Dockerfile-gpu | 10 -- runpod/runtimes/task/runner.py | 40 ++++++++ tests/test_apps/test_bootstrap.py | 56 ++++++++++- tests/test_apps/test_build.py | 15 +++ tests/test_apps/test_images.py | 83 ++++++++++++++++ tests/test_apps/test_task_runner_http.py | 67 +++++++++++++ tests/test_apps/test_tasks.py | 15 ++- 19 files changed, 535 insertions(+), 142 deletions(-) create mode 100644 runpod/apps/images.py create mode 100644 runpod/runtimes/gpu-base/Dockerfile delete mode 100644 runpod/runtimes/task/Dockerfile-gpu create mode 100644 tests/test_apps/test_images.py diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 342e85c68..662426ce2 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -32,45 +32,15 @@ jobs: pip install -e . pip install pytest pytest-asyncio cloudpickle - - name: Run task runner tests - run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py -q -o addopts="" -p no:unraisableexception + - name: Run runtime tests + run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py tests/test_apps/test_bootstrap.py -q -o addopts="" -p no:unraisableexception - build-task-cpu: + build-gpu-base: runs-on: ubuntu-latest needs: [test-runner] strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v7 - - - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push task image - uses: docker/build-push-action@v6 - with: - context: . - file: runpod/runtimes/task/Dockerfile - platforms: linux/amd64 - build-args: | - PYTHON_VERSION=${{ matrix.python-version }} - push: ${{ github.event_name != 'pull_request' }} - tags: | - runpod/task:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} - cache-from: type=gha,scope=task-py${{ matrix.python-version }} - cache-to: type=gha,mode=max,scope=task-py${{ matrix.python-version }} - - build-task-gpu: - runs-on: ubuntu-latest - needs: [test-runner] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v7 @@ -89,60 +59,37 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push gpu task image - uses: docker/build-push-action@v6 - with: - context: . - file: runpod/runtimes/task/Dockerfile-gpu - platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: | - runpod/task-gpu:${{ env.TAG_SUFFIX }} - cache-from: type=gha,scope=task-gpu - cache-to: type=gha,mode=max,scope=task-gpu - - build-queue: - runs-on: ubuntu-latest - needs: [test-runner] - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v7 - - - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push queue image + - name: Build and push gpu base image uses: docker/build-push-action@v6 with: context: . - file: runpod/runtimes/queue/Dockerfile + file: runpod/runtimes/gpu-base/Dockerfile platforms: linux/amd64 build-args: | PYTHON_VERSION=${{ matrix.python-version }} push: ${{ github.event_name != 'pull_request' }} tags: | - runpod/queue:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} - cache-from: type=gha,scope=queue-py${{ matrix.python-version }} - cache-to: type=gha,mode=max,scope=queue-py${{ matrix.python-version }} + runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} + cache-from: type=gha,scope=gpu-base-py${{ matrix.python-version }} + cache-to: type=gha,mode=max,scope=gpu-base-py${{ matrix.python-version }} - build-api: + build-runtimes: runs-on: ubuntu-latest - needs: [test-runner] + needs: [test-runner, build-gpu-base] strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + kind: [task, queue, api] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + accelerator: [cpu, gpu] steps: - uses: actions/checkout@v7 + - name: Clear space + if: matrix.accelerator == 'gpu' + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" + docker system prune -af + - uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub @@ -153,16 +100,27 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push api image + - name: Compute image names + id: names + run: | + if [ "${{ matrix.accelerator }}" = "gpu" ]; then + echo "base=runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }}" >> "$GITHUB_OUTPUT" + echo "repo=runpod/${{ matrix.kind }}-gpu" >> "$GITHUB_OUTPUT" + else + echo "base=python:${{ matrix.python-version }}-slim" >> "$GITHUB_OUTPUT" + echo "repo=runpod/${{ matrix.kind }}" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push runtime image uses: docker/build-push-action@v6 with: context: . - file: runpod/runtimes/api/Dockerfile + file: runpod/runtimes/${{ matrix.kind }}/Dockerfile platforms: linux/amd64 build-args: | - PYTHON_VERSION=${{ matrix.python-version }} + BASE_IMAGE=${{ steps.names.outputs.base }} push: ${{ github.event_name != 'pull_request' }} tags: | - runpod/api:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} - cache-from: type=gha,scope=api-py${{ matrix.python-version }} - cache-to: type=gha,mode=max,scope=api-py${{ matrix.python-version }} + ${{ steps.names.outputs.repo }}:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} + cache-from: type=gha,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} + cache-to: type=gha,mode=max,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} diff --git a/runpod/apps/build.py b/runpod/apps/build.py index eaae09174..198bebf18 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -27,11 +27,10 @@ from typing import List, Optional, Tuple from .errors import AppError +from .images import DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS log = logging.getLogger(__name__) -DEFAULT_PYTHON_VERSION = "3.12" - # manylinux tags matching runpod worker glibc, newest first MANYLINUX_PLATFORMS = ( "manylinux_2_28_x86_64", @@ -104,12 +103,20 @@ def collect_requirements(project_root: Path, app) -> List[str]: def split_exclusions( requirements: List[str], extra_excludes: Optional[List[str]] = None, + *, + auto_exclude: bool = True, ) -> Tuple[List[str], List[str]]: """(vendored, excluded-names). excluded packages resolve from the - worker image at runtime instead of the artifact.""" - excluded_names = SIZE_PROHIBITIVE_PACKAGES | { - requirement_name(e) for e in (extra_excludes or []) - } + worker image at runtime instead of the artifact. + + auto_exclude applies the size-prohibitive set; it is only sound + when the workers run on builtin gpu images that preinstall those + packages. custom images vendor everything unless the user excludes + explicitly. + """ + excluded_names = {requirement_name(e) for e in (extra_excludes or [])} + if auto_exclude: + excluded_names |= SIZE_PROHIBITIVE_PACKAGES kept: List[str] = [] excluded: List[str] = [] for req in requirements: @@ -244,12 +251,25 @@ def build_environment( scratch_dir: Optional[Path] = None, ) -> BuildResult: """vendor the app's full runtime environment into a directory tree.""" + if python_version not in SUPPORTED_PYTHON_VERSIONS: + raise BuildError( + f"python {python_version} is not supported " + f"(supported: {', '.join(SUPPORTED_PYTHON_VERSIONS)})" + ) scratch = scratch_dir or Path(tempfile.mkdtemp(prefix="rp-build-")) env_dir = scratch / "env" env_dir.mkdir(parents=True, exist_ok=True) user_requirements = collect_requirements(project_root, app) - vendored, excluded = split_exclusions(user_requirements, exclude) + # size-prohibitive packages come preinstalled on the builtin gpu + # images; a custom image carries no such guarantee, so any custom + # image in the app disables auto-exclusion and everything vendors + auto_exclude = not any( + h.spec.image for h in app.resources.values() + ) + vendored, excluded = split_exclusions( + user_requirements, exclude, auto_exclude=auto_exclude + ) runtime = [runtime_requirement(scratch), "cloudpickle"] from .spec import ResourceKind diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index a1c1c7ecb..67f3194bc 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -23,39 +23,25 @@ DEV_PREFIX = "dev" -# worker runtime images by (kind, is_cpu). queue images are built from -# runpod/runtimes/queue in this repo; api still rides the flash lb -# images until the lb runtime lands. RUNPOD_RUNTIME_TAG selects the -# image channel (latest, dev, or a pinned version). import os as _os -_TAG = _os.environ.get("RUNPOD_RUNTIME_TAG", "latest") - -DEFAULT_IMAGES = { - ("queue", False): f"runpod/queue:py3.12-{_TAG}", - ("queue", True): f"runpod/queue:py3.12-{_TAG}", - ("api", False): f"runpod/api:py3.12-{_TAG}", - ("api", True): f"runpod/api:py3.12-{_TAG}", -} - def dev_endpoint_name(app_name: str, resource_name: str) -> str: return f"{DEV_PREFIX}-{app_name}-{resource_name}" def _image_for(spec: ResourceSpec) -> str: - import os - + """dev worker image: custom image, env override, or the builtin + runtime image matched to the local python (dev requests carry + pickled source, so client and worker versions should align).""" if spec.image: return spec.image - override = os.getenv("RUNPOD_DEV_IMAGE") + override = _os.getenv("RUNPOD_DEV_IMAGE") if override: return override - key = (spec.kind.value, spec.is_cpu) - image = DEFAULT_IMAGES.get(key) - if image is None: - raise ValueError(f"no dev image for resource kind {spec.kind.value}") - return image + from .images import image_for_spec, local_python_version + + return image_for_spec(spec, python_version=local_python_version()) def _bootstrap_source() -> str: diff --git a/runpod/apps/images.py b/runpod/apps/images.py new file mode 100644 index 000000000..122b4903c --- /dev/null +++ b/runpod/apps/images.py @@ -0,0 +1,84 @@ +"""runtime image selection. + +one place maps (resource kind, gpu/cpu, python version) to the runtime +image built by the runtimes workflow. cpu images are python:X.Y-slim +based; gpu images additionally preinstall the torch family (matching +the build-time exclusion set) so deployed artifacts never install it +at cold start. + +RUNPOD_RUNTIME_TAG selects the image channel (latest, dev, or a pinned +version). +""" + +import os +import sys +from typing import Optional + +from .spec import ResourceKind, ResourceSpec + +# policy: non-EOL cpython versions with torch wheel support. 3.10 +# ages out at its 2026-10 EOL; new versions join once torch ships +# wheels for them. +SUPPORTED_PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") +DEFAULT_PYTHON_VERSION = "3.12" + +_REPOS = { + (ResourceKind.QUEUE, False): "runpod/queue", + (ResourceKind.QUEUE, True): "runpod/queue-gpu", + (ResourceKind.API, False): "runpod/api", + (ResourceKind.API, True): "runpod/api-gpu", + (ResourceKind.TASK, False): "runpod/task", + (ResourceKind.TASK, True): "runpod/task-gpu", +} + + +def runtime_tag() -> str: + return os.environ.get("RUNPOD_RUNTIME_TAG", "latest") + + +def local_python_version() -> str: + """the local interpreter's version, which the worker must match. + + dev sessions and tasks ship cloudpickle payloads produced on the + client interpreter; those are not reliably portable across python + versions, so a silent fallback would surface as cryptic + deserialization failures on the worker. mismatches fail here, + loudly, instead. + """ + version = f"{sys.version_info.major}.{sys.version_info.minor}" + if version not in SUPPORTED_PYTHON_VERSIONS: + raise RuntimeError( + f"python {version} has no runtime image " + f"(supported: {', '.join(SUPPORTED_PYTHON_VERSIONS)}). " + f"run under a supported python or use a custom image= built " + f"for {version}." + ) + return version + + +def runtime_image( + kind: ResourceKind, + *, + gpu: bool, + python_version: Optional[str] = None, +) -> str: + """the builtin runtime image for a resource shape.""" + version = python_version or DEFAULT_PYTHON_VERSION + if version not in SUPPORTED_PYTHON_VERSIONS: + raise ValueError( + f"python {version} is not supported by the runtime images " + f"(supported: {', '.join(SUPPORTED_PYTHON_VERSIONS)})" + ) + repo = _REPOS[(kind, gpu)] + return f"{repo}:py{version}-{runtime_tag()}" + + +def image_for_spec( + spec: ResourceSpec, *, python_version: Optional[str] = None +) -> str: + """the image a resource runs on: its custom image or the builtin.""" + if spec.image: + return spec.image + return runtime_image( + spec.kind, gpu=not spec.is_cpu, python_version=python_version + ) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 525d050ce..898a4c0b2 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -35,15 +35,8 @@ # safety net: pods self-terminate server-side after this long DEFAULT_MAX_LIFETIME = timedelta(hours=1) -# task runtime images built from runpod/runtimes/task by the runtimes -# workflow. RUNPOD_RUNTIME_TAG selects the channel (latest, dev, pinned). import os as _os -_TAG = _os.environ.get("RUNPOD_RUNTIME_TAG", "latest") - -DEFAULT_GPU_IMAGE = f"runpod/task-gpu:{_TAG}" -DEFAULT_CPU_IMAGE = f"runpod/task:py3.12-{_TAG}" - READY_POLL_INTERVAL = 2.0 READY_TIMEOUT = 600.0 RESULT_POLL_INTERVAL = 2.0 @@ -82,10 +75,13 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] **(spec.env or {}), } + from .images import image_for_spec, local_python_version + pod: Dict[str, Any] = { "name": f"task-{task_name}-{secrets.token_hex(4)}", - "imageName": spec.image - or (DEFAULT_CPU_IMAGE if spec.is_cpu else DEFAULT_GPU_IMAGE), + "imageName": image_for_spec( + spec, python_version=local_python_version() + ), "ports": f"{TASK_PORT}/http", "containerDiskInGb": 10, "terminateAfter": terminate_after, diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index aadc5dc9f..85c5f1db9 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -41,7 +41,7 @@ def deploy( python_version: str = typer.Option( "3.12", "--python-version", - help="Worker python version for dependency wheels (3.10-3.12).", + help="Worker python version for dependency wheels (3.10-3.14).", ), exclude: Optional[str] = typer.Option( None, diff --git a/runpod/runtimes/README.md b/runpod/runtimes/README.md index 2918e5a82..d2675e3c0 100644 --- a/runpod/runtimes/README.md +++ b/runpod/runtimes/README.md @@ -33,18 +33,34 @@ on the baked images). ## images +every runtime kind ships a cpu and a gpu variant per python version. +gpu variants build on `runpod/gpu-base:pyX.Y-{tag}` (python:X.Y-slim +plus the torch family), which preinstalls exactly the packages the +build auto-excludes from artifacts. the two lists must stay in sync: +`SIZE_PROHIBITIVE_PACKAGES` in `runpod/apps/build.py` and the pip +install in `gpu-base/Dockerfile`. + | image | base | use | |---|---|---| -| `runpod/queue:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | queue endpoints | -| `runpod/api:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | load-balanced api endpoints | -| `runpod/task:py{3.10,3.11,3.12}-{tag}` | `python:X.Y-slim` | cpu tasks | -| `runpod/task-gpu:{tag}` | `runpod/pytorch` | gpu tasks | +| `runpod/gpu-base:py{3.10-3.14}-{tag}` | `python:X.Y-slim` + torch | shared gpu base | +| `runpod/queue:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu queue endpoints | +| `runpod/queue-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu queue endpoints | +| `runpod/api:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu api endpoints | +| `runpod/api-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu api endpoints | +| `runpod/task:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu tasks | +| `runpod/task-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu tasks | + +image selection lives in `runpod.apps.images`; dev sessions and tasks +pick the variant matching the local python so pickled payloads stay +compatible. custom images work everywhere: the sdk injects `bootstrap.py` (queue/ api) or `runner.py` (task) base64-encoded in an env var and boots it via `dockerArgs`. both scripts are stdlib-only, so any image with a python3 binary works. for deployed resources the vendored env then -provides everything else, including the runpod package itself. +provides everything else, including the runpod package itself. builds +targeting custom images vendor the torch family too (no auto +exclusion), since only the builtin gpu images guarantee it. ## task diff --git a/runpod/runtimes/api/Dockerfile b/runpod/runtimes/api/Dockerfile index 99231b951..3509bd730 100644 --- a/runpod/runtimes/api/Dockerfile +++ b/runpod/runtimes/api/Dockerfile @@ -1,5 +1,7 @@ -ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim +# api (load-balanced) worker image. BASE_IMAGE is python:X.Y-slim for +# cpu or runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). +ARG BASE_IMAGE=python:3.12-slim +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index 7f51a34bb..d758fce09 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -13,6 +13,9 @@ verify packages the build excluded by size (torch and friends) must come from the image; genuinely absent ones are installed as a fallback, loudly + system apt packages from the resource's system_dependencies; + these cannot ride in the artifact, so this is the one + install that legitimately remains at cold start serve exec the worker runtime from the vendored environment live mode (no resource name, `rp dev`): @@ -189,6 +192,50 @@ def _pip_install(packages, phase): raise PhaseError(phase, result.stderr[-3000:]) +def _resource_entry(manifest): + name = _resource_name() + for entry in manifest.get("resources", []): + if entry.get("name") == name: + return entry + return {} + + +def _install_system(manifest): + """apt packages cannot be vendored; install this resource's + system_dependencies at cold start.""" + packages = _resource_entry(manifest).get("systemDependencies") or [] + if not packages: + return + import shutil + + if shutil.which("apt-get") is None: + raise PhaseError( + "system", + f"system dependencies {packages} requested but apt-get is " + f"not available in this image; use a debian-based image or " + f"bake them in", + ) + env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") + _log(f"installing system dependencies: {packages}") + update = subprocess.run( + ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env + ) + if update.returncode != 0: + raise PhaseError("system", f"apt-get update failed: {update.stderr[-2000:]}") + result = subprocess.run( + ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + raise PhaseError( + "system", + f"failed to install system dependencies {packages}: " + f"{result.stderr[-2000:]}", + ) + + def _verify_excluded(manifest): """packages the build excluded must come from the image. @@ -354,7 +401,9 @@ def main(): if _resource_name(): app_dir = _locate() paths = _attach(app_dir) - _verify_excluded(_manifest(app_dir)) + manifest = _manifest(app_dir) + _verify_excluded(manifest) + _install_system(manifest) ok, err = _worker_importable(paths) if not ok: raise PhaseError( diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile new file mode 100644 index 000000000..c596dba01 --- /dev/null +++ b/runpod/runtimes/gpu-base/Dockerfile @@ -0,0 +1,18 @@ +# shared gpu base for the runtime images: python:X.Y-slim plus the +# torch family. this preinstalled set must stay in sync with +# SIZE_PROHIBITIVE_PACKAGES in runpod/apps/build.py: deploys targeting +# builtin gpu images exclude exactly these packages from artifacts and +# expect the image to provide them. torch cuda wheels bundle the cuda +# runtime; the host injects the driver. +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=Etc/UTC \ + PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir torch torchvision torchaudio diff --git a/runpod/runtimes/queue/Dockerfile b/runpod/runtimes/queue/Dockerfile index 867636d59..df7870957 100644 --- a/runpod/runtimes/queue/Dockerfile +++ b/runpod/runtimes/queue/Dockerfile @@ -1,5 +1,7 @@ -ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim +# queue worker image. BASE_IMAGE is python:X.Y-slim for cpu or +# runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). +ARG BASE_IMAGE=python:3.12-slim +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ diff --git a/runpod/runtimes/task/Dockerfile b/runpod/runtimes/task/Dockerfile index a8f22cc81..0ab5be41d 100644 --- a/runpod/runtimes/task/Dockerfile +++ b/runpod/runtimes/task/Dockerfile @@ -1,5 +1,7 @@ -ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim +# task runner image. BASE_IMAGE is python:X.Y-slim for cpu or +# runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). +ARG BASE_IMAGE=python:3.12-slim +FROM ${BASE_IMAGE} ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ diff --git a/runpod/runtimes/task/Dockerfile-gpu b/runpod/runtimes/task/Dockerfile-gpu deleted file mode 100644 index e08e1e6f8..000000000 --- a/runpod/runtimes/task/Dockerfile-gpu +++ /dev/null @@ -1,10 +0,0 @@ -FROM runpod/pytorch:1.0.3-cu1281-torch291-ubuntu2204 - -ENV PYTHONUNBUFFERED=1 - -RUN pip install --no-cache-dir cloudpickle - -COPY runpod/runtimes/task/runner.py /runner.py - -EXPOSE 8080 -CMD ["python", "/runner.py"] diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index e27ee4db8..fa295420e 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -23,6 +23,7 @@ import io import json import os +import shutil import subprocess import sys import threading @@ -68,6 +69,42 @@ def _install(packages, label): return None +_apt_updated = False + + +def _install_system(packages): + """install apt packages; requires a debian-family image with root.""" + global _apt_updated + if not packages: + return None + if shutil.which("apt-get") is None: + return ( + f"system dependencies {packages} requested but apt-get is not " + f"available in this image; use a debian-based image or bake " + f"them in" + ) + env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") + if not _apt_updated: + update = subprocess.run( + ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env + ) + if update.returncode != 0: + return f"apt-get update failed: {update.stderr[-2000:]}" + _apt_updated = True + result = subprocess.run( + ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + return ( + f"failed to install system dependencies {packages}: " + f"{result.stderr[-2000:]}" + ) + return None + + def _deserialize_args(request): fmt = request.get("serialization_format", "cloudpickle") args = request.get("args") or [] @@ -95,6 +132,9 @@ def execute_request(request): """run one FunctionRequest to completion, returning a response dict.""" stdout_io = io.StringIO() try: + error = _install_system(request.get("system_dependencies")) + if error: + return {"success": False, "error": error} error = _install(request.get("dependencies"), "dependencies") if error: return {"success": False, "error": error} diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 83f90982b..2d46b5b53 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -112,6 +112,56 @@ def test_manifest_missing_fails_with_phase(self, tmp_path): assert exc_info.value.phase == "attach" +class TestSystemPhase: + def test_no_system_deps_no_op(self): + bootstrap._install_system({"resources": [{"name": "q"}]}) + + def test_missing_apt_fails_with_phase(self, monkeypatch): + import shutil + + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") + monkeypatch.setattr(shutil, "which", lambda _: None) + manifest = { + "resources": [{"name": "q", "systemDependencies": ["ffmpeg"]}] + } + with pytest.raises(bootstrap.PhaseError) as exc_info: + bootstrap._install_system(manifest) + assert exc_info.value.phase == "system" + assert "ffmpeg" in str(exc_info.value) + + def test_installs_resource_system_deps(self, monkeypatch): + import shutil + + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") + calls = [] + + class R: + returncode = 0 + stderr = "" + + monkeypatch.setattr( + bootstrap.subprocess, + "run", + lambda cmd, **kw: calls.append(cmd) or R(), + ) + manifest = { + "resources": [{"name": "q", "systemDependencies": ["ffmpeg", "sox"]}] + } + bootstrap._install_system(manifest) + assert calls[0][:2] == ["apt-get", "update"] + assert calls[1][:3] == ["apt-get", "install", "-y"] + assert "ffmpeg" in calls[1] and "sox" in calls[1] + + def test_other_resources_deps_ignored(self, monkeypatch): + monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") + manifest = { + "resources": [{"name": "other", "systemDependencies": ["ffmpeg"]}] + } + # q has no system deps; must not attempt anything + bootstrap._install_system(manifest) + + class TestVerifyPhase: def test_no_exclusions_no_op(self): bootstrap._verify_excluded({"resources": []}) @@ -161,7 +211,11 @@ def q(): payload = _endpoint_input(app, q.spec) template = payload["template"] - assert template["imageName"].startswith("runpod/queue:py3.12-") + from runpod.apps.images import local_python_version + + assert template["imageName"].startswith( + f"runpod/queue:py{local_python_version()}-" + ) assert template["dockerArgs"] == "" env = {e["key"]: e["value"] for e in template["env"]} assert "RUNPOD_BOOTSTRAP_B64" not in env diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index add1a753a..60a3c23c2 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -80,6 +80,21 @@ def test_user_excludes_added(self): assert kept == ["numpy"] assert excluded == ["transformers"] + def test_auto_exclude_off_vendors_everything(self): + # custom images carry no torch guarantee: nothing auto-excluded + kept, excluded = split_exclusions( + ["numpy", "torch==2.4", "triton"], auto_exclude=False + ) + assert kept == ["numpy", "torch==2.4", "triton"] + assert excluded == [] + + def test_auto_exclude_off_user_excludes_still_apply(self): + kept, excluded = split_exclusions( + ["numpy", "torch"], extra_excludes=["torch"], auto_exclude=False + ) + assert kept == ["numpy"] + assert excluded == ["torch"] + def test_exclusion_set_is_size_based(self): # sanity-pin the auto set; additions must be size-justified assert SIZE_PROHIBITIVE_PACKAGES == { diff --git a/tests/test_apps/test_images.py b/tests/test_apps/test_images.py new file mode 100644 index 000000000..87e7dfc2a --- /dev/null +++ b/tests/test_apps/test_images.py @@ -0,0 +1,83 @@ +"""tests for runtime image selection.""" + +import pytest + +from runpod.apps.images import ( + SUPPORTED_PYTHON_VERSIONS, + image_for_spec, + local_python_version, + runtime_image, +) +from runpod.apps.spec import ResourceKind, ResourceSpec + + +class TestRuntimeImage: + @pytest.mark.parametrize("kind", list(ResourceKind)) + @pytest.mark.parametrize("gpu", [False, True]) + @pytest.mark.parametrize("version", SUPPORTED_PYTHON_VERSIONS) + def test_full_matrix_resolves(self, kind, gpu, version): + image = runtime_image(kind, gpu=gpu, python_version=version) + assert image.startswith("runpod/") + assert f"py{version}-" in image + assert ("-gpu:" in image) == gpu + + def test_unsupported_python_rejected(self): + with pytest.raises(ValueError, match="3.9"): + runtime_image(ResourceKind.QUEUE, gpu=False, python_version="3.9") + + def test_tag_channel_from_env(self, monkeypatch): + monkeypatch.setenv("RUNPOD_RUNTIME_TAG", "dev") + image = runtime_image(ResourceKind.TASK, gpu=True, python_version="3.11") + assert image == "runpod/task-gpu:py3.11-dev" + + def test_default_channel_is_latest(self, monkeypatch): + monkeypatch.delenv("RUNPOD_RUNTIME_TAG", raising=False) + image = runtime_image(ResourceKind.API, gpu=False, python_version="3.12") + assert image == "runpod/api:py3.12-latest" + + +class TestImageForSpec: + def test_custom_image_wins(self): + spec = ResourceSpec( + kind=ResourceKind.QUEUE, name="q", image="my/image:1" + ) + assert image_for_spec(spec) == "my/image:1" + + def test_cpu_spec_gets_cpu_image(self): + spec = ResourceSpec(kind=ResourceKind.QUEUE, name="q", cpu=["cpu3c-1-2"]) + assert image_for_spec(spec, python_version="3.12") == ( + "runpod/queue:py3.12-latest" + ) + + def test_gpu_spec_gets_gpu_image(self): + spec = ResourceSpec(kind=ResourceKind.QUEUE, name="q", gpu=["ADA_24"]) + assert image_for_spec(spec, python_version="3.12") == ( + "runpod/queue-gpu:py3.12-latest" + ) + + +class TestLocalPythonVersion: + def test_returns_supported_version(self): + assert local_python_version() in SUPPORTED_PYTHON_VERSIONS + + def test_unsupported_local_python_fails_loudly(self, monkeypatch): + # cloudpickle payloads are version-bound; a silent fallback + # would break at deserialization time on the worker + import runpod.apps.images as images + + class FakeVersion: + major, minor = 3, 9 + + monkeypatch.setattr(images.sys, "version_info", FakeVersion) + with pytest.raises(RuntimeError, match="3.9"): + local_python_version() + + def test_supported_range_policy(self): + # non-EOL cpython with torch wheels; update at EOL boundaries + assert SUPPORTED_PYTHON_VERSIONS == ( + "3.10", + "3.11", + "3.12", + "3.13", + "3.14", + ) diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index 7524d3f85..7f42c4fa6 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -123,3 +123,70 @@ def test_unknown_path_404(server): with pytest.raises(urllib.error.HTTPError) as exc_info: _request(f"{server}/nope") assert exc_info.value.code == 404 + + +class TestSystemDependencies: + def test_missing_apt_reports_error(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: None) + response = task_runner.execute_request( + { + "function_name": "f", + "function_code": "def f():\n return 1", + "system_dependencies": ["ffmpeg"], + "serialization_format": "json", + } + ) + assert response["success"] is False + assert "apt-get is not available" in response["error"] + + def test_system_deps_installed_before_execution(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") + monkeypatch.setattr(task_runner, "_apt_updated", False) + calls = [] + + class R: + returncode = 0 + stderr = "" + + monkeypatch.setattr( + task_runner.subprocess, + "run", + lambda cmd, **kw: calls.append(cmd) or R(), + ) + response = task_runner.execute_request( + { + "function_name": "f", + "function_code": "def f():\n return 40 + 2", + "system_dependencies": ["ffmpeg"], + "serialization_format": "json", + } + ) + assert response["success"] is True + assert response["json_result"] == 42 + assert calls[0][:2] == ["apt-get", "update"] + assert "ffmpeg" in calls[1] + + def test_apt_update_runs_once(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") + monkeypatch.setattr(task_runner, "_apt_updated", False) + calls = [] + + class R: + returncode = 0 + stderr = "" + + monkeypatch.setattr( + task_runner.subprocess, + "run", + lambda cmd, **kw: calls.append(cmd) or R(), + ) + task_runner._install_system(["ffmpeg"]) + task_runner._install_system(["sox"]) + updates = [c for c in calls if c[:2] == ["apt-get", "update"]] + assert len(updates) == 1 diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index ab67f9b6f..a7fc2f913 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -128,7 +128,12 @@ def _spec(self, **kw): def test_cpu_pod_input(self): pod = _pod_input(self._spec(cpu=["cpu3c-1-2"]), "tok", "t") assert pod["instanceIds"] == ["cpu3c-1-2"] - assert pod["imageName"] == "runpod/task:py3.12-latest" + from runpod.apps.images import local_python_version + + assert ( + pod["imageName"] + == f"runpod/task:py{local_python_version()}-latest" + ) assert pod["ports"] == "8080/http" assert pod["terminateAfter"] env = {e["key"]: e["value"] for e in pod["env"]} @@ -143,7 +148,13 @@ def test_gpu_pod_input(self): ) assert pod["gpuTypeIdList"] == ["NVIDIA GeForce RTX 4090"] assert pod["gpuCount"] == 2 - assert pod["imageName"] == "runpod/task-gpu:latest" + # gpu runtime image matched to the local python version + from runpod.apps.images import local_python_version + + assert ( + pod["imageName"] + == f"runpod/task-gpu:py{local_python_version()}-latest" + ) def test_custom_image_bootstraps_runner(self): pod = _pod_input( From 9538f6e3a7e96db0f8a2cc0f09c53a2ff3e0e4dd Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:02:27 -0700 Subject: [PATCH 018/120] fix: use tarfile data filter for artifact extraction --- runpod/runtimes/bootstrap.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index d758fce09..8d41d1452 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -112,13 +112,20 @@ def _locate(): started = time.monotonic() try: with tarfile.open(ARTIFACT_PATH, mode="r:*") as tar: - for member in tar.getmembers(): - dest = os.path.realpath(os.path.join(APP_DIR, member.name)) - if not dest.startswith(target + os.sep) and dest != target: - raise PhaseError( - "locate", f"unsafe tar member path: {member.name}" + if hasattr(tarfile, "data_filter"): + # 3.12+: the data filter rejects traversal, devices, + # and absolute paths + tar.extractall(path=APP_DIR, filter="data") + else: + for member in tar.getmembers(): + dest = os.path.realpath( + os.path.join(APP_DIR, member.name) ) - tar.extractall(path=APP_DIR) + if not dest.startswith(target + os.sep) and dest != target: + raise PhaseError( + "locate", f"unsafe tar member path: {member.name}" + ) + tar.extractall(path=APP_DIR) except (OSError, tarfile.TarError) as exc: raise PhaseError("locate", f"failed to extract artifact: {exc}") From f2e0da5ef3711d8a305753e2873c469ee8473712 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 11:53:57 -0700 Subject: [PATCH 019/120] feat: reconcile flash-bound endpoints during deploy --- runpod/apps/api.py | 8 ++- runpod/apps/deploy.py | 148 +++++++++++++++++++++++++++++++++++++++++- runpod/rp_cli/main.py | 23 ++++++- 3 files changed, 174 insertions(+), 5 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index fc16c082a..ae3d97159 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -119,7 +119,13 @@ async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: flashAppByName(flashAppName: $flashAppName) { id name - flashEnvironments { id name state activeBuildId } + flashEnvironments { + id + name + state + activeBuildId + endpoints { id name } + } } } """ diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index d7c8e9674..e39a008b7 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -9,15 +9,22 @@ honored for source) 5. get-or-create the flash app + environment 6. upload via presigned url, finalize the build - 7. activate the build on the environment (coordinator provisions) + 7. activate the build on the environment (hosts deliver the artifact + to workers via the flash volume) + 8. reconcile endpoints: one per queue/api resource, named exactly + the resource name and bound to the environment via + flashEnvironmentId (the relation the sentinel's + resolveFlashEndpoint walks). removed resources' endpoints are + deleted; tasks have no standing infra. """ +import base64 import fnmatch import json import logging import tarfile import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional @@ -79,6 +86,7 @@ class DeployResult: build_id: str environment_id: str resources: List[str] + endpoints: Dict[str, str] = field(default_factory=dict) def build_manifest( @@ -190,6 +198,137 @@ def package_project( return output +def _deployed_endpoint_input( + app: App, + spec, + environment_id: str, + build_id: str, + python_version: str, +) -> Dict[str, Any]: + """saveEndpoint payload for one deployed queue/api resource. + + the endpoint name must equal the resource name: sentinel + resolution matches X-Flash-Endpoint against endpoint.name within + the environment. binding via flashEnvironmentId is also what makes + hosts deliver the build artifact to this endpoint's workers. + """ + from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS + from .images import image_for_spec + + template_env = { + "FLASH_RESOURCE_NAME": spec.name, + # version-triggering: a new build recreates all workers + "RUNPOD_BUILD_ID": build_id, + **(spec.env or {}), + } + + payload: Dict[str, Any] = { + "name": spec.name, + "flashEnvironmentId": environment_id, + "workersMin": spec.workers[0], + "workersMax": spec.workers[1], + "idleTimeout": spec.idle_timeout, + "scalerType": ( + "REQUEST_COUNT" if spec.kind is ResourceKind.API else "QUEUE_DELAY" + ), + "scalerValue": 4, + "flashBootType": "FLASHBOOT", + "template": { + "name": f"{app.name}-{spec.name}-template", + "imageName": image_for_spec(spec, python_version=python_version), + "containerDiskInGb": 10, + "dockerArgs": "", + "env": [ + {"key": k, "value": v} for k, v in template_env.items() + ], + }, + } + + if spec.image: + # custom image: inject the bootstrap; the vendored env in the + # artifact provides the runtime once the bootstrap unpacks it + from .dev import _bootstrap_docker_args, _bootstrap_source + + payload["template"]["env"].extend( + [ + { + "key": "RUNPOD_BOOTSTRAP_B64", + "value": base64.b64encode( + _bootstrap_source().encode() + ).decode(), + }, + {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, + ] + ) + payload["template"]["dockerArgs"] = _bootstrap_docker_args() + + if spec.kind is ResourceKind.API: + payload["type"] = "LB" + if spec.datacenter: + payload["locations"] = ",".join(spec.datacenter) + if spec.is_cpu: + payload["instanceIds"] = spec.cpu + if not spec.datacenter: + if any(i.startswith("cpu5") for i in spec.cpu or []): + payload["locations"] = ",".join( + dc.value for dc in CPU5_DATACENTERS + ) + else: + payload["locations"] = ",".join( + dc.value for dc in CPU3_DATACENTERS + ) + else: + payload["gpuIds"] = ",".join(spec.gpu or ["any"]) + payload["gpuCount"] = spec.gpu_count + return payload + + +async def reconcile_endpoints( + client: AppsApiClient, + app: App, + environment: Dict[str, Any], + build_id: str, + python_version: str, +) -> Dict[str, str]: + """converge the environment's endpoints to the app's resources. + + resource name -> endpoint id for everything provisioned. existing + endpoints (matched by name) are updated in place; endpoints whose + resources disappeared are deleted. + """ + existing = { + e["name"]: e["id"] for e in environment.get("endpoints") or [] + } + provisionable = { + h.spec.name: h + for h in app.resources.values() + if h.spec.kind in (ResourceKind.QUEUE, ResourceKind.API) + } + + endpoints: Dict[str, str] = {} + for name, handle in sorted(provisionable.items()): + payload = _deployed_endpoint_input( + app, handle.spec, environment["id"], build_id, python_version + ) + if name in existing: + payload["id"] = existing[name] + result = await client.save_endpoint(payload) + endpoints[name] = result["id"] + log.info( + "%s endpoint %s (%s)", + "updated" if name in existing else "provisioned", + name, + result["id"], + ) + + for name, endpoint_id in existing.items(): + if name not in provisionable: + await client.delete_endpoint(endpoint_id) + log.info("deleted removed endpoint %s (%s)", name, endpoint_id) + + return endpoints + + async def deploy_app( app: App, project_root: Path, @@ -247,9 +386,14 @@ async def deploy_app( await client.deploy_build(environment["id"], build["id"]) log.info("activated build %s on %s/%s", build["id"], app.name, env_name) + endpoints = await reconcile_endpoints( + client, app, environment, build["id"], python_version + ) + return DeployResult( app_name=app.name, build_id=build["id"], environment_id=environment["id"], resources=[r["name"] for r in manifest["resources"]], + endpoints=endpoints, ) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 85c5f1db9..b83dcc17b 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -96,6 +96,11 @@ def deploy( @app.command() def dev( module: Path = typer.Argument(..., help="Python module to run."), + once: bool = typer.Option( + False, + "--once", + help="Run the entrypoint once, tear down, and exit (for scripts/CI).", + ), ): """start an interactive dev session for MODULE. @@ -143,11 +148,21 @@ async def _wait_for_rerun(watcher: FileWatcher) -> str: return "changed" done, _ = await asyncio.wait({stdin_task}, timeout=0.5) if done: + # eof (piped stdin, ci): interactive re-run is + # impossible, wait on file changes only + if not stdin_task.result(): + return await _wait_for_change(watcher) return "enter" finally: stdin_task.cancel() - async def _session() -> None: + async def _wait_for_change(watcher: FileWatcher) -> str: + while True: + if watcher.changed(): + return "changed" + await asyncio.sleep(0.5) + + async def _session() -> int: nonlocal apps, entrypoint session = DevSession(apps) watcher = FileWatcher([module.parent]) @@ -160,6 +175,10 @@ async def _session() -> None: run_entrypoint(entrypoint) except Exception as exc: # noqa: BLE001 - dev loop survives user errors typer.secho(f"entrypoint error: {exc}", fg=typer.colors.RED) + if once: + return 1 + if once: + return 0 _echo("--- press enter to re-run, edit files to reload, ctrl-c to exit ---") reason = await _wait_for_rerun(watcher) if reason == "changed": @@ -171,7 +190,7 @@ async def _session() -> None: await session.stop() try: - asyncio.run(_session()) + raise typer.Exit(asyncio.run(_session())) except (KeyboardInterrupt, EOFError): _echo("\nsession ended.") From 781870419d3142b8593e16ca0d5bf97fd13c8411 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:09:45 -0700 Subject: [PATCH 020/120] feat: retry transient http failures and harden discovery scans --- runpod/apps/discovery.py | 58 ++++++++++++++++++- runpod/apps/handles.py | 12 ++++ runpod/apps/targets.py | 74 ++++++++++++++++++++---- tests/test_apps/test_deploy.py | 36 ++++++++++++ tests/test_apps/test_retry.py | 102 +++++++++++++++++++++++++++++++++ 5 files changed, 267 insertions(+), 15 deletions(-) create mode 100644 tests/test_apps/test_retry.py diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 080e91a19..d011b3754 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -7,6 +7,7 @@ import importlib.util import logging +import os import sys import threading from pathlib import Path @@ -18,6 +19,27 @@ IMPORT_TIMEOUT_SECONDS = 30 +# set while discovery imports modules; handle invocation checks it so +# module-level .remote()/.spawn() calls fail fast with a precise +# diagnosis instead of attempting network calls mid-scan +DISCOVERY_ENV = "RUNPOD_DISCOVERY_SCAN" + + +def in_discovery() -> bool: + return bool(os.environ.get(DISCOVERY_ENV)) + + +class DiscoveryInvocationError(Exception): + """a handle was invoked at module level during a discovery scan.""" + + def __init__(self, resource_name: str): + super().__init__( + f"'{resource_name}' was invoked at import time. move calls " + f"into a function, an entrypoint, or an " + f'`if __name__ == "__main__":` guard so importing the file ' + f"has no side effects." + ) + _SKIP_DIRS = { ".git", ".venv", @@ -80,7 +102,14 @@ def run() -> None: def discover_apps(target: Path) -> List[App]: - """import all python files under target and return the apps they define.""" + """import python files under target and return the apps they define. + + a single-file target imports strictly and raises on any failure. + a directory walk is tolerant: files that fail to import are + collected as warnings, and discovery only fails outright when no + app was found anywhere (the failures are then the likely cause and + are included in the error). + """ before = set(id(a) for a in get_registered_apps()) sys_path_added = False @@ -90,11 +119,34 @@ def discover_apps(target: Path) -> List[App]: sys.path.insert(0, root_str) sys_path_added = True + strict = target.is_file() + failures: List[str] = [] + os.environ[DISCOVERY_ENV] = "1" try: for path in _python_files(target): - _import_module(path) + try: + _import_module(path) + except DiscoveryError as exc: + if strict: + raise + failures.append(str(exc)) + log.warning("%s", exc) finally: + os.environ.pop(DISCOVERY_ENV, None) if sys_path_added: sys.path.remove(root_str) - return [a for a in get_registered_apps() if id(a) not in before] + found = [a for a in get_registered_apps() if id(a) not in before] + # a file imported both by the scan (synthetic name) and by another + # file (real name, e.g. `from main import q1`) registers its app + # twice; deploying duplicates would be wasteful and confusing + by_name = {} + for app in found: + by_name.setdefault(app.name, app) + found = list(by_name.values()) + if not found and failures: + raise DiscoveryError( + "no runpod.App found; some files failed to import:\n " + + "\n ".join(failures) + ) + return found diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 9d768bf74..1e3d0dc6d 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -93,6 +93,7 @@ def local(self, *args: Any, **kwargs: Any) -> Any: # -- remote execution -- async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: + self._guard_discovery() ctx = current_context() if ctx is Context.WORKER and self._is_current_worker(): @@ -106,6 +107,7 @@ async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: return await target.invoke(payload) async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: + self._guard_discovery() target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) data = await target.submit(payload) @@ -114,6 +116,12 @@ async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: return Job(data, self) return data + def _guard_discovery(self) -> None: + from .discovery import DiscoveryInvocationError, in_discovery + + if in_discovery(): + raise DiscoveryInvocationError(self.spec.name) + def _is_current_worker(self) -> bool: import os @@ -153,6 +161,10 @@ def __call__(self, path: str, body: Any = None, **kwargs: Any) -> Any: return block(self.aio(path, body, **kwargs)) async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: + from .discovery import DiscoveryInvocationError, in_discovery + + if in_discovery(): + raise DiscoveryInvocationError(self._handle.spec.name) target = await self._handle._app._resolve(self._handle.spec) return await target.request(self._method, path, body, **kwargs) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index a472073ae..fe2a64b3c 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -127,6 +127,58 @@ async def request( raise NotImplementedError(f"{type(self).__name__} does not serve http routes") +# transient statuses worth retrying: gateway/edge errors that occur +# while the sentinel cache warms or an edge node hiccups. 4xx (other +# than 429) are never retried; the request itself is wrong. +RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504, 520, 522, 524}) +RETRY_ATTEMPTS = 4 +RETRY_BASE_DELAY = 0.5 + + +async def _request_json( + method: str, + url: str, + headers: Dict[str, str], + timeout: float, + *, + payload: Any = None, + app_name: str = "", + resource_name: str = "", +) -> Dict[str, Any]: + """http json call with exponential backoff on transient failures.""" + import asyncio + + client_timeout = aiohttp.ClientTimeout(total=timeout) + last_exc: Optional[Exception] = None + for attempt in range(RETRY_ATTEMPTS): + if attempt: + await asyncio.sleep(RETRY_BASE_DELAY * (2 ** (attempt - 1))) + try: + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.request( + method, url, json=payload, headers=headers + ) as resp: + if resp.status == 404: + raise EndpointNotFound(app_name, resource_name) + if resp.status in RETRYABLE_STATUSES: + last_exc = aiohttp.ClientResponseError( + resp.request_info, + resp.history, + status=resp.status, + message=await resp.text(), + ) + continue + resp.raise_for_status() + return await resp.json() + except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as exc: + # connection-level failures (reset, ssl hiccup, dns) are + # transient by nature; response-level errors above already + # decided retryability + last_exc = exc + continue + raise last_exc # type: ignore[misc] + + async def _post_json( url: str, payload: Any, @@ -136,23 +188,21 @@ async def _post_json( app_name: str = "", resource_name: str = "", ) -> Dict[str, Any]: - client_timeout = aiohttp.ClientTimeout(total=timeout) - async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.post(url, json=payload, headers=headers) as resp: - if resp.status == 404: - raise EndpointNotFound(app_name, resource_name) - resp.raise_for_status() - return await resp.json() + return await _request_json( + "POST", + url, + headers, + timeout, + payload=payload, + app_name=app_name, + resource_name=resource_name, + ) async def _get_json( url: str, headers: Dict[str, str], timeout: float ) -> Dict[str, Any]: - client_timeout = aiohttp.ClientTimeout(total=timeout) - async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.get(url, headers=headers) as resp: - resp.raise_for_status() - return await resp.json() + return await _request_json("GET", url, headers, timeout) async def _wait_terminal( diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index 37a986bee..240274f2c 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -219,3 +219,39 @@ async def test_deploy_reuses_existing_app_and_env(self, tmp_path): api.create_app.assert_not_awaited() api.create_environment.assert_not_awaited() + + +class TestTolerantDiscovery: + def test_broken_bystander_warns_but_discovers(self, tmp_path): + _write_project(tmp_path) + (tmp_path / "scratch.py").write_text("this is not python !!!") + apps = discover_apps(tmp_path) + assert len(apps) == 1 + + def test_single_file_target_stays_strict(self, tmp_path): + broken = tmp_path / "broken.py" + broken.write_text("import nonexistent_module_xyz") + with pytest.raises(DiscoveryError, match="broken.py"): + discover_apps(broken) + + def test_no_apps_and_failures_raises_with_causes(self, tmp_path): + (tmp_path / "broken.py").write_text("import nonexistent_module_xyz") + with pytest.raises(DiscoveryError, match="broken.py"): + discover_apps(tmp_path) + + def test_import_time_invocation_diagnosed(self, tmp_path): + _write_project(tmp_path) + (tmp_path / "client.py").write_text( + "from main import q1\nq1.remote(1)\n" + ) + # directory walk: the client file fails with the precise + # diagnosis but the app still discovers + apps = discover_apps(tmp_path) + assert len(apps) == 1 + + def test_import_time_invocation_message(self, tmp_path): + _write_project(tmp_path) + client = tmp_path / "client.py" + client.write_text("from main import q1\nq1.remote(1)\n") + with pytest.raises(DiscoveryError, match="invoked at import time"): + discover_apps(client) diff --git a/tests/test_apps/test_retry.py b/tests/test_apps/test_retry.py new file mode 100644 index 000000000..a973cb3cb --- /dev/null +++ b/tests/test_apps/test_retry.py @@ -0,0 +1,102 @@ +"""tests for transient-failure retry in the http layer.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import patch + +import aiohttp +import pytest + +from runpod.apps.errors import EndpointNotFound +from runpod.apps.targets import ( + RETRY_ATTEMPTS, + RETRYABLE_STATUSES, + _post_json, +) + + +@pytest.fixture +def flaky_server(): + """local server whose first responses are scripted status codes.""" + state = {"script": [], "hits": 0} + + class Handler(BaseHTTPRequestHandler): + def _respond(self): + state["hits"] += 1 + if state["hits"] <= len(state["script"]): + status = state["script"][state["hits"] - 1] + self.send_response(status) + self.end_headers() + self.wfile.write(b"scripted") + return + body = json.dumps({"ok": True, "hits": state["hits"]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = _respond + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + server.state = state + server.url = f"http://127.0.0.1:{server.server_address[1]}/run" + yield server + server.shutdown() + + +async def test_retries_transient_5xx_then_succeeds(flaky_server): + flaky_server.state["script"] = [520, 502] + with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): + data = await _post_json(flaky_server.url, {"x": 1}, {}, timeout=10) + assert data["ok"] is True + assert data["hits"] == 3 + + +async def test_exhausted_retries_raise_last_error(flaky_server): + flaky_server.state["script"] = [503] * (RETRY_ATTEMPTS + 2) + with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): + with pytest.raises(aiohttp.ClientResponseError) as exc_info: + await _post_json(flaky_server.url, {"x": 1}, {}, timeout=10) + assert exc_info.value.status == 503 + assert flaky_server.state["hits"] == RETRY_ATTEMPTS + + +async def test_404_not_retried_maps_to_endpoint_not_found(flaky_server): + flaky_server.state["script"] = [404] + with pytest.raises(EndpointNotFound): + await _post_json( + flaky_server.url, + {"x": 1}, + {}, + timeout=10, + app_name="a", + resource_name="r", + ) + assert flaky_server.state["hits"] == 1 + + +async def test_client_4xx_not_retried(flaky_server): + flaky_server.state["script"] = [400] + with pytest.raises(aiohttp.ClientResponseError) as exc_info: + await _post_json(flaky_server.url, {"x": 1}, {}, timeout=10) + assert exc_info.value.status == 400 + assert flaky_server.state["hits"] == 1 + + +def test_429_is_retryable(): + assert 429 in RETRYABLE_STATUSES + + +async def test_connection_error_retried(unused_tcp_port): + # nothing listening: pure connection failures, all attempts consumed + url = f"http://127.0.0.1:{unused_tcp_port}/run" + with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): + with pytest.raises(aiohttp.ClientConnectionError): + await _post_json(url, {"x": 1}, {}, timeout=5) From dd2f0fd9f509b6f4d8fb5bfff9b43343fbe0b440 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:35:05 -0700 Subject: [PATCH 021/120] fix: gpu driver injection, pool id expansion for pods, and job results --- runpod/apps/deploy.py | 30 ++++++- runpod/apps/dev.py | 2 +- runpod/apps/gpu.py | 8 ++ runpod/apps/handles.py | 13 ++- runpod/apps/targets.py | 30 +++++++ runpod/apps/tasks.py | 21 ++++- runpod/runtimes/gpu-base/Dockerfile | 6 +- tests/e2e/matrix/main.py | 103 ++++++++++++++++++++++++ tests/e2e/matrix/run.py | 119 ++++++++++++++++++++++++++++ tests/test_apps/test_tasks.py | 23 ++++++ 10 files changed, 349 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/matrix/main.py create mode 100644 tests/e2e/matrix/run.py diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index e39a008b7..3e99ba9c7 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -198,6 +198,27 @@ def package_project( return output +# gpu runtime images carry the torch family (~9 GB uncompressed); the +# container disk must hold the image layers plus the unpacked artifact +CPU_CONTAINER_DISK_GB = 10 +GPU_CONTAINER_DISK_GB = 30 + + +def _container_disk_gb(spec) -> int: + return CPU_CONTAINER_DISK_GB if spec.is_cpu else GPU_CONTAINER_DISK_GB + + +def _account_api_key() -> Optional[str]: + import os + + key = os.getenv("RUNPOD_API_KEY") + if key: + return key + import runpod + + return runpod.api_key + + def _deployed_endpoint_input( app: App, spec, @@ -222,6 +243,13 @@ def _deployed_endpoint_input( **(spec.env or {}), } + # cross-resource calls from inside a worker go through the sentinel, + # which authenticates with the account api key; without it any + # .remote() from worker code would fail + api_key = _account_api_key() + if api_key and "RUNPOD_API_KEY" not in template_env: + template_env["RUNPOD_API_KEY"] = api_key + payload: Dict[str, Any] = { "name": spec.name, "flashEnvironmentId": environment_id, @@ -236,7 +264,7 @@ def _deployed_endpoint_input( "template": { "name": f"{app.name}-{spec.name}-template", "imageName": image_for_spec(spec, python_version=python_version), - "containerDiskInGb": 10, + "containerDiskInGb": _container_disk_gb(spec), "dockerArgs": "", "env": [ {"key": k, "value": v} for k, v in template_env.items() diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 67f3194bc..7a6852111 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -93,7 +93,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: "template": { "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), - "containerDiskInGb": 10, + "containerDiskInGb": 10 if spec.is_cpu else 30, "dockerArgs": "", "env": [ {"key": GENERATION_ENV, "value": str(generation)}, diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py index 73e026573..251d61de0 100644 --- a/runpod/apps/gpu.py +++ b/runpod/apps/gpu.py @@ -51,6 +51,14 @@ def all(cls) -> List["GpuGroup"]: """all gpu groups except ANY.""" return [g for g in cls if g is not cls.ANY] + def device_names(self) -> List[str]: + """exact device display names in this pool. + + serverless endpoints take pool ids directly (gpuIds), but pods + take device names (gpuTypeIdList), so tasks need the expansion. + """ + return [t.value for t in POOLS_TO_TYPES.get(self, [])] + class GpuType(Enum): ANY = "any" diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 1e3d0dc6d..a7b23ad27 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -30,11 +30,18 @@ def load_model(): ... # worker startup hook class Job: - """a submitted queue job, returned by .spawn().""" + """a submitted queue job, returned by .spawn(). + + `job.result()` blocks until the job finishes and returns its + output; `await job.result.aio()` is the async form. + """ def __init__(self, data: Dict[str, Any], handle: "FunctionHandle"): self._data = data self._handle = handle + from .invoker import Invoker + + self.result = Invoker(self._result_async) @property def id(self) -> str: @@ -44,6 +51,10 @@ def id(self) -> str: def status(self) -> str: return self._data.get("status", "UNKNOWN") + async def _result_async(self, timeout: float = 300.0) -> Any: + target = await self._handle._app._resolve(self._handle.spec) + return await target.wait(self._data, timeout=timeout) + def __repr__(self) -> str: return f"Job(id={self.id!r}, status={self.status!r})" diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index fe2a64b3c..1330fd6eb 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -120,6 +120,14 @@ async def invoke(self, payload: Dict[str, Any], *, timeout: float) -> Any: async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: """submit without waiting; returns the raw job data.""" + async def wait( + self, job_data: Dict[str, Any], *, timeout: float + ) -> Any: + """wait for a submitted job to finish and return its output.""" + raise NotImplementedError( + f"{type(self).__name__} does not support job polling" + ) + async def request( self, method: str, path: str, body: Any = None, *, timeout: float ) -> Any: @@ -285,6 +293,18 @@ async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: resource_name=self.resource_name, ) + async def wait( + self, + job_data: Dict[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Any: + base = f"{_endpoint_url_base()}/{SENTINEL_ID}" + data = await _wait_terminal( + base, job_data, self._sentinel_headers(), timeout + ) + return unwrap_job_output(data) + async def request( self, method: str, @@ -367,6 +387,16 @@ async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: url = f"{_endpoint_url_base()}/{self.endpoint_id}/run" return await _post_json(url, payload, _headers(), DEFAULT_TIMEOUT_SECONDS) + async def wait( + self, + job_data: Dict[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Any: + base = f"{_endpoint_url_base()}/{self.endpoint_id}" + data = await _wait_terminal(base, job_data, _headers(), timeout) + return self.unwrap(data) + async def request( self, method: str, diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 898a4c0b2..c1bbf4765 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -55,6 +55,23 @@ def _bootstrap_command() -> str: return shell_launcher("RUNPOD_TASK_RUNNER_B64", "/task_runner.py") +def _device_names(gpu: Optional[List[str]]) -> List[str]: + """pods take exact device names, not pool ids; expand any pool ids + (e.g. ADA_24 -> NVIDIA GeForce RTX 4090) and pass device names + through.""" + from .gpu import GpuGroup + + if not gpu: + return ["any"] + names: List[str] = [] + for entry in gpu: + try: + names.extend(GpuGroup(entry).device_names() or [entry]) + except ValueError: + names.append(entry) + return names + + def _proxy_url(pod_id: str) -> str: return f"https://{pod_id}-{TASK_PORT}.proxy.runpod.net" @@ -83,7 +100,7 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] spec, python_version=local_python_version() ), "ports": f"{TASK_PORT}/http", - "containerDiskInGb": 10, + "containerDiskInGb": 10 if spec.is_cpu else 30, "terminateAfter": terminate_after, "supportPublicIp": True, } @@ -113,7 +130,7 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] if spec.is_cpu: pod["instanceIds"] = spec.cpu else: - pod["gpuTypeIdList"] = spec.gpu or ["any"] + pod["gpuTypeIdList"] = _device_names(spec.gpu) pod["gpuCount"] = spec.gpu_count return pod diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile index c596dba01..71453ff22 100644 --- a/runpod/runtimes/gpu-base/Dockerfile +++ b/runpod/runtimes/gpu-base/Dockerfile @@ -7,9 +7,13 @@ ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim +# nvidia container runtime keys on these to inject the host driver; +# without them torch sees no gpu even on a gpu machine ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + NVIDIA_VISIBLE_DEVICES=all \ + NVIDIA_DRIVER_CAPABILITIES=compute,utility RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ diff --git a/tests/e2e/matrix/main.py b/tests/e2e/matrix/main.py new file mode 100644 index 000000000..aa4463297 --- /dev/null +++ b/tests/e2e/matrix/main.py @@ -0,0 +1,103 @@ +"""e2e matrix app: every resource kind, cross-calls, deps, custom images. + +deployed by tests/e2e/matrix/run.py; each resource exercises a distinct +permutation of the surface. +""" + +import runpod +from runpod import App + +app = App("e2e-matrix") + + +# -- queue permutations ------------------------------------------------ + +@app.queue(name="q-basic", cpu="cpu3c-1-2", workers=(0, 1)) +def q_basic(x: int): + return {"doubled": x * 2} + + +@app.queue( + name="q-deps", + cpu="cpu3c-2-4", + workers=(0, 1), + dependencies=["pyfiglet"], + system_dependencies=["jq"], +) +def q_deps(word: str): + import shutil + import pyfiglet + + return { + "art": pyfiglet.figlet_format(word).splitlines()[0], + "pyfiglet": pyfiglet.__version__, + "jq": shutil.which("jq") is not None, + } + + +@app.queue( + name="q-custom", + cpu="cpu3c-2-4", + workers=(0, 1), + image="python:3.12-slim", + dependencies=["humanize"], +) +def q_custom(n: int): + import humanize + + return {"human": humanize.intword(n)} + + +@app.queue(name="q-gpu", gpu=runpod.GpuGroup.ADA_24, workers=(0, 1)) +def q_gpu(): + import torch + + return { + "torch": torch.__version__, + "cuda": torch.cuda.is_available(), + "device": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None, + } + + +# -- cross-calls ------------------------------------------------------- + +@app.queue(name="q-caller", cpu="cpu3c-2-4", workers=(0, 1)) +async def q_caller(x: int): + """queue worker fanning out to another queue and a task.""" + doubled = await q_basic.remote.aio(x) + product = await t_mul.remote.aio(x, 10) + return {"from_queue": doubled, "from_task": product} + + +# -- tasks ------------------------------------------------------------- + +@app.task(name="t-mul", cpu="cpu3c-1-2") +def t_mul(a: int, b: int): + return {"product": a * b} + + +@app.task(name="t-gpu", gpu=runpod.GpuGroup.ADA_24) +def t_gpu(size: int): + import torch + + m = torch.rand(size, size, device="cuda") + return {"trace": float(m.trace()), "device": torch.cuda.get_device_name(0)} + + +# -- api (load-balanced) ----------------------------------------------- + +@app.api(name="a-svc", cpu="cpu3c-2-4", workers=(1, 1)) +class Svc: + @runpod.init + def setup(self): + self.counter = 0 + self.ready = True + + @runpod.post("/bump") + def bump(self, body: dict): + self.counter += int(body.get("by", 1)) + return {"counter": self.counter, "ready": self.ready} + + @runpod.get("/stats") + def stats(self): + return {"counter": self.counter} diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py new file mode 100644 index 000000000..fc4ce8356 --- /dev/null +++ b/tests/e2e/matrix/run.py @@ -0,0 +1,119 @@ +"""drive the deployed e2e matrix. run AFTER `rp deploy tests/e2e/matrix`. + + RUNPOD_RUNTIME_TAG=dev python tests/e2e/matrix/run.py [--only q-basic,...] + +each check is independent; failures are collected, not fatal, so one +broken permutation doesn't hide the rest. +""" + +import argparse +import asyncio +import sys +import time +import traceback + +from main import Svc, app, q_basic, q_caller, q_custom, q_deps, q_gpu, t_gpu, t_mul # noqa: E402 + + +async def check_q_basic(): + r = await q_basic.remote.aio(21) + assert r == {"doubled": 42}, r + return r + + +async def check_q_deps(): + r = await q_deps.remote.aio("hi") + assert r["pyfiglet"], r + assert r["jq"] is True, r + return {k: r[k] for k in ("pyfiglet", "jq")} + + +async def check_q_custom(): + r = await q_custom.remote.aio(1_500_000) + assert r == {"human": "1.5 million"}, r + return r + + +async def check_q_gpu(): + r = await q_gpu.remote.aio() + assert r["cuda"] is True, r + return r + + +async def check_q_caller(): + r = await q_caller.remote.aio(3) + assert r["from_queue"] == {"doubled": 6}, r + assert r["from_task"] == {"product": 30}, r + return r + + +async def check_t_mul(): + r = await t_mul.remote.aio(6, 7) + assert r == {"product": 42}, r + return r + + +async def check_t_gpu(): + r = await t_gpu.remote.aio(64) + assert "device" in r, r + return r + + +async def check_api(): + r1 = await Svc.post.aio("/bump", {"by": 5}) + assert r1["counter"] >= 5 and r1["ready"] is True, r1 + r2 = await Svc.get.aio("/stats") + assert r2["counter"] >= 5, r2 + return {"bump": r1, "stats": r2} + + +async def check_spawn(): + job = await q_basic.spawn.aio(4) + r = await job.result.aio() + assert r == {"doubled": 8}, r + return r + + +CHECKS = { + "q-basic": check_q_basic, + "q-deps": check_q_deps, + "q-custom": check_q_custom, + "q-gpu": check_q_gpu, + "q-caller": check_q_caller, + "t-mul": check_t_mul, + "t-gpu": check_t_gpu, + "api": check_api, + "spawn": check_spawn, +} + + +async def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--only", help="comma-separated check names") + args = parser.parse_args() + + names = args.only.split(",") if args.only else list(CHECKS) + results, failures = {}, {} + + for name in names: + started = time.monotonic() + print(f"--- {name} ...", flush=True) + try: + results[name] = await CHECKS[name]() + elapsed = time.monotonic() - started + print(f" PASS {elapsed:.1f}s {results[name]}", flush=True) + except Exception as exc: # noqa: BLE001 - collect and report + elapsed = time.monotonic() - started + failures[name] = exc + print(f" FAIL {elapsed:.1f}s {exc}", flush=True) + traceback.print_exc() + + print(f"\n{len(results)}/{len(names)} passed") + if failures: + print(f"failed: {', '.join(failures)}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index a7fc2f913..a78faba8c 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -272,3 +272,26 @@ def t(): assert isinstance(job, TaskJob) assert job.pod_id == "pod-1" instance.submit.assert_awaited_once() + + +class TestGpuPoolExpansion: + def test_pool_id_expands_to_device_names(self): + from runpod.apps.tasks import _device_names + + assert _device_names(["ADA_24"]) == ["NVIDIA GeForce RTX 4090"] + + def test_device_name_passes_through(self): + from runpod.apps.tasks import _device_names + + assert _device_names(["NVIDIA L40S"]) == ["NVIDIA L40S"] + + def test_none_means_any(self): + from runpod.apps.tasks import _device_names + + assert _device_names(None) == ["any"] + + def test_mixed_pool_and_device(self): + from runpod.apps.tasks import _device_names + + names = _device_names(["AMPERE_48", "NVIDIA L40S"]) + assert "NVIDIA A40" in names and "NVIDIA L40S" in names From c9872cb4485fc3146709018651755b3ce36f812a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:55:53 -0700 Subject: [PATCH 022/120] fix: import List in tasks module --- runpod/apps/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index c1bbf4765..0d0233091 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -20,7 +20,7 @@ import time from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import aiohttp From 4290a0233e80b78153db2265b0e05b52ca7950fe Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:39:25 -0700 Subject: [PATCH 023/120] fix: fall back to RUNPOD_API_KEY env for module api key --- runpod/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runpod/__init__.py b/runpod/__init__.py index 7c410d496..fc74ba060 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -112,7 +112,9 @@ if _credentials is not None: api_key = _credentials["api_key"] # pylint: disable=invalid-name else: - api_key = None # pylint: disable=invalid-name + # workers and ci have no config file; the env var is the standard + # way credentials reach them + api_key = os.environ.get("RUNPOD_API_KEY") # pylint: disable=invalid-name endpoint_url_base = os.environ.get( "RUNPOD_ENDPOINT_BASE_URL", "https://api.runpod.ai/v2" From d3274ee19473f9760de491e1eb4b9edf2fa30909 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:26:52 -0700 Subject: [PATCH 024/120] feat: pod log access via hapi and storage-dc pinning for gpu deploys --- runpod/apps/deploy.py | 16 ++++++- runpod/apps/logs.py | 97 +++++++++++++++++++++++++++++++++++++++++++ runpod/rp_cli/main.py | 33 +++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 runpod/apps/logs.py diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 3e99ba9c7..6171836c9 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -233,7 +233,7 @@ def _deployed_endpoint_input( the environment. binding via flashEnvironmentId is also what makes hosts deliver the build artifact to this endpoint's workers. """ - from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS + from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS, DataCenter from .images import image_for_spec template_env = { @@ -250,6 +250,15 @@ def _deployed_endpoint_input( if api_key and "RUNPOD_API_KEY" not in template_env: template_env["RUNPOD_API_KEY"] = api_key + # workers that spawn tasks must select runtime images from the same + # channel the deploy used (the vendored env supplies the code, so + # only the image channel needs to travel) + import os as _os + + tag = _os.environ.get("RUNPOD_RUNTIME_TAG") + if tag and "RUNPOD_RUNTIME_TAG" not in template_env: + template_env["RUNPOD_RUNTIME_TAG"] = tag + payload: Dict[str, Any] = { "name": spec.name, "flashEnvironmentId": environment_id, @@ -308,6 +317,11 @@ def _deployed_endpoint_input( else: payload["gpuIds"] = ",".join(spec.gpu or ["any"]) payload["gpuCount"] = spec.gpu_count + if not spec.datacenter: + # artifact delivery rides the flash volume (network + # storage); machines outside storage-cluster regions fail + # with "requires network storage" and recycle forever + payload["locations"] = ",".join(dc.value for dc in DataCenter) return payload diff --git a/runpod/apps/logs.py b/runpod/apps/logs.py new file mode 100644 index 000000000..482e0967e --- /dev/null +++ b/runpod/apps/logs.py @@ -0,0 +1,97 @@ +"""pod log access via the host api (hapi). + +hosts expose per-pod container and system logs; this is the fastest +way to see why a worker is stuck (image pull progress, region errors, +crash output) without ssh. snapshot and sse streaming supported. + + GET https://hapi.runpod.net/v1/pod/{podId}/logs + ?type=all|container|system + GET .../logs?stream=true&type=...&tail=N&since=RFC3339 +""" + +import json +import os +from typing import Any, AsyncIterator, Dict, List, Optional + +import aiohttp + +HAPI_BASE = os.environ.get("RUNPOD_HAPI_URL", "https://hapi.runpod.net") + +STREAM_TIMEOUT_SECONDS = 3600.0 + + +def _headers() -> Dict[str, str]: + from .targets import _api_key + + return {"Authorization": f"Bearer {_api_key()}"} + + +async def pod_logs( + pod_id: str, + *, + log_type: str = "all", + timeout: float = 30.0, +) -> Dict[str, List[str]]: + """snapshot of a pod's logs: {"container": [...], "system": [...]}.""" + url = f"{HAPI_BASE}/v1/pod/{pod_id}/logs" + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.get( + url, params={"type": log_type}, headers=_headers() + ) as resp: + resp.raise_for_status() + data = await resp.json() + return {k: v or [] for k, v in data.items()} + + +async def stream_pod_logs( + pod_id: str, + *, + log_type: str = "all", + tail: int = 100, + since: Optional[str] = None, +) -> AsyncIterator[Dict[str, Any]]: + """follow a pod's logs as they arrive. + + yields {"source": "container"|"system", "line": str, "ts": str} + parsed from the host's sse stream. ends when the host closes the + stream (1h server-side cap) or the caller breaks out. + """ + url = f"{HAPI_BASE}/v1/pod/{pod_id}/logs" + params: Dict[str, Any] = { + "stream": "true", + "type": log_type, + "tail": str(tail), + } + if since: + params["since"] = since + + client_timeout = aiohttp.ClientTimeout( + total=STREAM_TIMEOUT_SECONDS, sock_read=60 + ) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.get( + url, params=params, headers=_headers() + ) as resp: + resp.raise_for_status() + async for raw in resp.content: + line = raw.decode("utf-8", "replace").strip() + # sse frames: "data: {...}"; comments (heartbeats) start + # with ":" + if not line.startswith("data:"): + continue + try: + yield json.loads(line[len("data:") :].strip()) + except json.JSONDecodeError: + continue + + +def tail_summary(logs: Dict[str, List[str]], lines: int = 20) -> str: + """human-readable tail of a log snapshot, for error messages.""" + parts = [] + for source in ("system", "container"): + entries = logs.get(source) or [] + if entries: + parts.append(f"--- {source} (last {min(lines, len(entries))}) ---") + parts.extend(entries[-lines:]) + return "\n".join(parts) if parts else "(no logs available)" diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index b83dcc17b..4a896d7b6 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -195,6 +195,39 @@ async def _session() -> int: _echo("\nsession ended.") +@app.command() +def logs( + pod_id: str = typer.Argument(..., help="Pod id to fetch logs for."), + follow: bool = typer.Option( + False, "--follow", "-f", help="Stream logs as they arrive." + ), + log_type: str = typer.Option( + "all", "--type", help="all, container, or system." + ), + tail: int = typer.Option(100, "--tail", help="Lines of backfill."), +): + """show a pod's container and system logs (host api).""" + from runpod.apps.logs import pod_logs, stream_pod_logs + + async def _run(): + if follow: + async for entry in stream_pod_logs( + pod_id, log_type=log_type, tail=tail + ): + source = entry.get("source", "?") + _echo(f"[{source}] {entry.get('line', '')}") + else: + logs_data = await pod_logs(pod_id, log_type=log_type) + for source in ("system", "container"): + for line in logs_data.get(source) or []: + _echo(f"[{source}] {line}") + + try: + asyncio.run(_run()) + except KeyboardInterrupt: + pass + + @app.command() def login(): """authenticate with runpod and store the api key.""" From 5feb5665ef5da094f1a2ac963bbad754d17e7c98 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:33:51 -0700 Subject: [PATCH 025/120] fix: pin cu128 torch wheels in gpu base image --- runpod/runtimes/gpu-base/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile index 71453ff22..9cfd94427 100644 --- a/runpod/runtimes/gpu-base/Dockerfile +++ b/runpod/runtimes/gpu-base/Dockerfile @@ -19,4 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir torch torchvision torchaudio +# cu128 wheels: default pypi torch is cu130, which needs 580+ drivers +# that much of the fleet does not have yet. cu128 runs on 12.8+ drivers. +RUN pip install --no-cache-dir torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/cu128 From 9c222ff07fc30e7e9f5274c46c4ecc6e8ffd55df Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:37:36 -0700 Subject: [PATCH 026/120] fix: pin cp310 torch stack on cu128 index --- runpod/runtimes/gpu-base/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile index 9cfd94427..0fa5a5b82 100644 --- a/runpod/runtimes/gpu-base/Dockerfile +++ b/runpod/runtimes/gpu-base/Dockerfile @@ -21,5 +21,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # cu128 wheels: default pypi torch is cu130, which needs 580+ drivers # that much of the fleet does not have yet. cu128 runs on 12.8+ drivers. -RUN pip install --no-cache-dir torch torchvision torchaudio \ - --index-url https://download.pytorch.org/whl/cu128 +# py3.10 needs an explicit pin: the newest packages on the cu128 index +# no longer resolve together for cp310. +RUN if python -c 'import sys; sys.exit(0 if sys.version_info[:2] == (3, 10) else 1)'; then \ + pip install --no-cache-dir torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1 \ + --index-url https://download.pytorch.org/whl/cu128; \ + else \ + pip install --no-cache-dir torch torchvision torchaudio \ + --index-url https://download.pytorch.org/whl/cu128; \ + fi From a5f89f2994b79ceb442f987f007b5fbf5ff40571 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:40:50 -0700 Subject: [PATCH 027/120] fix: pin cu128 torch stack via extra index across all python versions --- runpod/runtimes/gpu-base/Dockerfile | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile index 0fa5a5b82..5bc0d2c97 100644 --- a/runpod/runtimes/gpu-base/Dockerfile +++ b/runpod/runtimes/gpu-base/Dockerfile @@ -21,12 +21,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # cu128 wheels: default pypi torch is cu130, which needs 580+ drivers # that much of the fleet does not have yet. cu128 runs on 12.8+ drivers. -# py3.10 needs an explicit pin: the newest packages on the cu128 index -# no longer resolve together for cp310. -RUN if python -c 'import sys; sys.exit(0 if sys.version_info[:2] == (3, 10) else 1)'; then \ - pip install --no-cache-dir torch==2.9.1 torchvision==0.24.1 torchaudio==2.9.1 \ - --index-url https://download.pytorch.org/whl/cu128; \ - else \ - pip install --no-cache-dir torch torchvision torchaudio \ - --index-url https://download.pytorch.org/whl/cu128; \ - fi +# pinned + extra-index (not index-url): the cuda index lacks common +# deps like typing-extensions, and the pin prevents drifting back to +# pypi's cu130 build. +RUN pip install --no-cache-dir \ + "torch==2.9.1+cu128" "torchvision==0.24.1+cu128" "torchaudio==2.9.1+cu128" \ + --extra-index-url https://download.pytorch.org/whl/cu128 From 5d0f47cd39d9ce12a151ebee22ef8cd18863d56c Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:19:47 -0700 Subject: [PATCH 028/120] fix: retry proxy 404 propagation races on task execute --- runpod/apps/tasks.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 0d0233091..52edf6cf0 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -176,15 +176,26 @@ async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: ) async def execute(self, request: Dict[str, Any], timeout: float) -> Dict[str, Any]: - """run to completion via /execute.""" + """run to completion via /execute. + + the proxy answers 404 for pods an edge node has not learned + about yet; since this pod demonstrably exists (we created it + and /ping succeeded), 404s here are propagation races and are + retried briefly rather than surfaced. + """ url = f"{_proxy_url(self.pod_id)}/execute" client_timeout = aiohttp.ClientTimeout(total=timeout) + attempts = 6 async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.post( - url, json=request, headers=self._headers - ) as resp: - resp.raise_for_status() - return await resp.json() + for attempt in range(attempts): + async with session.post( + url, json=request, headers=self._headers + ) as resp: + if resp.status == 404 and attempt < attempts - 1: + await asyncio.sleep(2 * (attempt + 1)) + continue + resp.raise_for_status() + return await resp.json() async def submit(self, request: Dict[str, Any]) -> None: """start in the background via /submit.""" From e847a0f6bca1650886ff7209273df4656e0ec676 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:52:17 -0700 Subject: [PATCH 029/120] feat: pretty cli console with dev lifecycle events and deploy phases --- requirements.txt | 1 + runpod/apps/deploy.py | 11 ++ runpod/apps/dev.py | 31 +++++- runpod/rp_cli/console.py | 203 ++++++++++++++++++++++++++++++++++++ runpod/rp_cli/main.py | 107 ++++++++++++++----- tests/test_apps/test_dev.py | 62 +++++++++++ 6 files changed, 386 insertions(+), 29 deletions(-) create mode 100644 runpod/rp_cli/console.py diff --git a/requirements.txt b/requirements.txt index 23500ff14..7d7671502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ psutil >= 5.9.0 py-cpuinfo >= 9.0.0 inquirerpy == 0.3.4 requests >= 2.31.0 +rich >= 13.0.0 tomli >= 2.4.1 tomlkit >= 0.15.0 tqdm-loggable >= 0.2 diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 6171836c9..2c59f7d20 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -371,6 +371,12 @@ async def reconcile_endpoints( return endpoints +def _phase(events, name: str, detail: str = "") -> None: + handler = getattr(events, "phase", None) + if handler is not None: + handler(name, detail) + + async def deploy_app( app: App, project_root: Path, @@ -379,11 +385,13 @@ async def deploy_app( api: Optional[AppsApiClient] = None, python_version: str = DEFAULT_PYTHON_VERSION, exclude: Optional[List[str]] = None, + events: Optional[object] = None, ) -> DeployResult: """run the full deploy pipeline for one app.""" client = api or AppsApiClient() env_name = env_name or app.env + _phase(events, "vendor", f"python {python_version}") build: BuildResult = build_environment( app, project_root, python_version=python_version, exclude=exclude ) @@ -393,6 +401,7 @@ async def deploy_app( python_version=python_version, excluded_packages=build.excluded, ) + _phase(events, "package") tar_path = package_project(project_root, manifest, env_dir=build.env_dir) tar_size = tar_path.stat().st_size size_mb = tar_size / (1024 * 1024) @@ -404,6 +413,7 @@ async def deploy_app( ) log.info("packaged %s (%.1f MB)", app.name, size_mb) + _phase(events, "upload", f"{size_mb:.1f} MB") remote_app = await client.get_app_by_name(app.name) if remote_app is None: remote_app = await client.create_app(app.name) @@ -428,6 +438,7 @@ async def deploy_app( await client.deploy_build(environment["id"], build["id"]) log.info("activated build %s on %s/%s", build["id"], app.name, env_name) + _phase(events, "endpoints") endpoints = await reconcile_endpoints( client, app, environment, build["id"], python_version ) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 7a6852111..6dccfe4a5 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -30,6 +30,11 @@ def dev_endpoint_name(app_name: str, resource_name: str) -> str: return f"{DEV_PREFIX}-{app_name}-{resource_name}" +def _resource_of(endpoint_name: str) -> str: + """resource name from a dev endpoint name (last dash segment).""" + return endpoint_name.rsplit("-", 1)[-1] + + def _image_for(spec: ResourceSpec) -> str: """dev worker image: custom image, env override, or the builtin runtime image matched to the local python (dev requests carry @@ -144,15 +149,30 @@ class DevSession: lifecycle: adopt-or-create by name on start, reconcile on refresh (config updates + a generation bump that recreates workers so every request after a code change runs fully fresh), delete on stop. + + `events`, when provided, receives lifecycle callbacks: + provisioning(name, kind, hardware), adopted(name, id), + ready(name, id), refreshed(name, generation), deleted(name). """ - def __init__(self, apps: List[App], api: Optional[AppsApiClient] = None): + def __init__( + self, + apps: List[App], + api: Optional[AppsApiClient] = None, + events: Optional[object] = None, + ): self.apps = apps self.api = api or AppsApiClient() self.generation = 1 + self.events = events # endpoint name -> id for everything this session owns self._endpoints: Dict[str, str] = {} + def _emit(self, event: str, *args) -> None: + handler = getattr(self.events, event, None) + if handler is not None: + handler(*args) + @property def _endpoint_ids(self) -> List[str]: return list(self._endpoints.values()) @@ -178,19 +198,25 @@ async def start(self) -> None: name = dev_endpoint_name(app.name, spec.name) payload = _endpoint_input(app, spec, self.generation) + hardware = ",".join(spec.cpu or spec.gpu or ["any"]) found = existing.get(name) if found: # adopt: reconcile the leftover endpoint to the # current spec instead of creating a duplicate + self._emit("adopted", spec.name, found["id"]) payload["id"] = found["id"] result = await self.api.save_endpoint(payload) endpoint_id = result["id"] log.info("adopted dev endpoint %s (%s)", name, endpoint_id) else: + self._emit( + "provisioning", spec.name, spec.kind.value, hardware + ) result = await self.api.save_endpoint(payload) endpoint_id = result["id"] log.info("provisioned dev endpoint %s (%s)", name, endpoint_id) + self._emit("ready", spec.name, endpoint_id) self._endpoints[name] = endpoint_id app._dev_targets[spec.name] = LiveTarget(endpoint_id) @@ -217,6 +243,7 @@ async def refresh(self, apps: List[App]) -> None: endpoint_id = self._endpoints.pop(name) try: await self.api.delete_endpoint(endpoint_id) + self._emit("deleted", _resource_of(name)) log.info("deleted removed dev endpoint %s", name) except Exception as exc: log.warning("failed to delete %s: %s", name, exc) @@ -231,6 +258,7 @@ async def refresh(self, apps: List[App]) -> None: endpoint_id = result["id"] self._endpoints[name] = endpoint_id app._dev_targets[handle.spec.name] = LiveTarget(endpoint_id) + self._emit("refreshed", handle.spec.name, self.generation) log.info( "refreshed dev endpoint %s (%s, generation %d)", name, @@ -243,6 +271,7 @@ async def stop(self) -> None: for name, endpoint_id in list(self._endpoints.items()): try: await self.api.delete_endpoint(endpoint_id) + self._emit("deleted", _resource_of(name)) log.info("deleted dev endpoint %s (%s)", name, endpoint_id) except Exception as exc: log.warning( diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py new file mode 100644 index 000000000..e8dcbd2c5 --- /dev/null +++ b/runpod/rp_cli/console.py @@ -0,0 +1,203 @@ +"""console output for the rp cli. + +timestamped, column-aligned lifecycle lines for dev sessions and +deploys, in the spirit of a compact build log: + + 14:02:11 double │ provisioning queue cpu3c-1-2 + 14:02:19 double │ ready qjjwnknp + 14:02:20 POST /double + 14:02:21 ✓ double 0.4s +""" + +import time +from datetime import datetime +from typing import Iterable, Optional + +from rich.console import Console + +console = Console(highlight=False) + +# shared name column width so pipes line up across resources +_name_width: int = 0 + + +def set_name_width(names: Iterable[str]) -> None: + global _name_width + _name_width = max((len(n) for n in names), default=0) + + +def _padded(name: str) -> str: + return f"{name:<{_name_width}}" if _name_width else name + + +def _ts() -> str: + return f"[dim]{datetime.now().strftime('%H:%M:%S')}[/dim]" + + +def _pipe(name: str) -> str: + return f"{_padded(name)} [dim]│[/dim]" + + +# -- generic -- + + +def info(message: str) -> None: + console.print(f"{_ts()} {message}") + + +def success(message: str) -> None: + console.print(f"{_ts()} [green]✓[/green] {message}") + + +def error(message: str) -> None: + console.print(f"{_ts()} [red]✗[/red] {message}") + + +def warn(message: str) -> None: + console.print(f"{_ts()} [yellow]![/yellow] {message}") + + +def rule(title: str = "") -> None: + console.rule(f"[dim]{title}[/dim]" if title else None, style="dim") + + +# -- resource lifecycle -- + + +def resource_provisioning(name: str, kind: str, hardware: str) -> None: + console.print( + f"{_ts()} {_pipe(name)} [blue]provisioning[/blue] " + f"[dim]{kind} {hardware}[/dim]" + ) + + +def resource_adopted(name: str, endpoint_id: str) -> None: + console.print( + f"{_ts()} {_pipe(name)} [blue]adopted[/blue] [dim]{endpoint_id}[/dim]" + ) + + +def resource_ready(name: str, endpoint_id: str) -> None: + console.print( + f"{_ts()} {_pipe(name)} [green]ready[/green] [dim]{endpoint_id}[/dim]" + ) + + +def resource_refreshed(name: str, generation: int) -> None: + console.print( + f"{_ts()} {_pipe(name)} [blue]refreshed[/blue] " + f"[dim]generation {generation}[/dim]" + ) + + +def resource_deleted(name: str) -> None: + console.print(f"{_ts()} {_pipe(name)} [dim]deleted[/dim]") + + +def resource_log(name: str, line: str) -> None: + console.print(f"{_ts()} {_pipe(name)} {line}") + + +# -- request lifecycle -- + + +def request_started(name: str, label: str = "") -> None: + console.print(f"{_ts()} [white]CALL[/white] /{name} [dim]{label}[/dim]") + + +def request_completed(name: str, elapsed_s: float) -> None: + console.print( + f"{_ts()} [green]✓[/green] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + ) + + +def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> None: + console.print( + f"{_ts()} [red]✗[/red] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + ) + if err: + console.print(f" [dim]{err}[/dim]") + + +# -- dev session chrome -- + + +def dev_banner(app_names: Iterable[str], module: str) -> None: + apps = ", ".join(app_names) + console.print() + console.print(f"[green]✓[/green] [bold]rp dev[/bold] [dim]{module}[/dim]") + console.print(f" [dim]apps[/dim] {apps}") + console.print() + + +def dev_resources_table(rows: Iterable[tuple]) -> None: + """rows of (name, kind, hardware, endpoint_id).""" + rows = list(rows) + if not rows: + console.print(" [dim](no queue/api resources; tasks run per-call)[/dim]") + return + w_name = max(len(r[0]) for r in rows) + w_kind = max(len(r[1]) for r in rows) + w_hw = max(len(r[2]) for r in rows) + for name, kind, hardware, endpoint_id in rows: + console.print( + f" [white]{name:<{w_name}}[/white]" + f" [dim]{kind:<{w_kind}}[/dim]" + f" [dim]{hardware:<{w_hw}}[/dim]" + f" [dim]{endpoint_id}[/dim]" + ) + console.print() + + +def dev_hints() -> None: + console.print( + " [dim]enter[/dim] re-run [dim]edit[/dim] reload " + "[dim]ctrl-c[/dim] exit" + ) + console.print() + + +def phase(name: str, detail: str = "") -> None: + suffix = f" [dim]{detail}[/dim]" if detail else "" + console.print(f"{_ts()} [blue]{name}[/blue]{suffix}") + + +class DeployEvents: + """deploy_app event sink rendering phase lines.""" + + def phase(self, name: str, detail: str = "") -> None: + phase(name, detail) + + +class DevEvents: + """DevSession event sink rendering lifecycle lines.""" + + def provisioning(self, name: str, kind: str, hardware: str) -> None: + resource_provisioning(name, kind, hardware) + + def adopted(self, name: str, endpoint_id: str) -> None: + resource_adopted(name, endpoint_id) + + def ready(self, name: str, endpoint_id: str) -> None: + resource_ready(name, endpoint_id) + + def refreshed(self, name: str, generation: int) -> None: + resource_refreshed(name, generation) + + def deleted(self, name: str) -> None: + resource_deleted(name) + + +class Timer: + """context timer for elapsed reporting.""" + + def __enter__(self) -> "Timer": + self._start = time.monotonic() + return self + + def __exit__(self, *exc) -> None: + self.elapsed = time.monotonic() - self._start + + @property + def so_far(self) -> float: + return time.monotonic() - self._start diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 4a896d7b6..d4ee54117 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -52,8 +52,13 @@ def deploy( ), ): """package and deploy all apps found in TARGET.""" + import logging + from runpod.apps.deploy import deploy_app from runpod.apps.discovery import DiscoveryError, discover_apps + from runpod.rp_cli import console as ui + + logging.getLogger("runpod.apps").setLevel(logging.WARNING) target = (target or Path.cwd()).resolve() if not target.exists(): @@ -61,7 +66,6 @@ def deploy( project_root = target if target.is_dir() else target.parent - _echo(f"discovering apps in {target} ...") try: apps = discover_apps(target) except DiscoveryError as exc: @@ -75,22 +79,31 @@ def deploy( ) for found in apps: - names = ", ".join(found.resources) or "(no resources)" - _echo(f"deploying app '{found.name}': {names}") - result = asyncio.run( - deploy_app( - found, - project_root, - env_name=env, - python_version=python_version, - exclude=exclude.split(",") if exclude else None, - ) + resource_names = list(found.resources) + ui.set_name_width(resource_names) + ui.console.print() + ui.console.print( + f"[bold]{found.name}[/bold] [dim]-> {env or found.env}[/dim]" ) - _echo( - f" build {result.build_id} active on " - f"{result.app_name}/{env or found.env}" + with ui.Timer() as t: + result = asyncio.run( + deploy_app( + found, + project_root, + env_name=env, + python_version=python_version, + exclude=exclude.split(",") if exclude else None, + events=ui.DeployEvents(), + ) + ) + for name, endpoint_id in sorted(result.endpoints.items()): + ui.resource_ready(name, endpoint_id) + ui.success( + f"build [white]{result.build_id}[/white] live on " + f"[white]{result.app_name}/{env or found.env}[/white] " + f"[dim]{t.elapsed:.1f}s[/dim]" ) - _echo("done.") + ui.console.print() @app.command() @@ -113,6 +126,7 @@ def dev( from runpod.apps.discovery import DiscoveryError, discover_apps from runpod.apps.entrypoint import get_entrypoint, run_entrypoint from runpod.apps.watch import FileWatcher + from runpod.rp_cli import console as ui module = module.resolve() if not module.is_file(): @@ -120,6 +134,12 @@ def dev( os.environ["RUNPOD_DEV_SESSION"] = "1" + # the console renders lifecycle lines; keep the module loggers to + # warnings so output isn't duplicated + import logging + + logging.getLogger("runpod.apps").setLevel(logging.WARNING) + def _scan(): try: apps = discover_apps(module) @@ -138,6 +158,25 @@ def _scan(): apps, entrypoint = _scan() + def _all_resource_names(app_list) -> list: + return [ + handle.spec.name + for a in app_list + for handle in a.resources.values() + ] + + def _table_rows(session: DevSession) -> list: + rows = [] + for a in session.apps: + for handle in a.resources.values(): + spec = handle.spec + hardware = ",".join(spec.cpu or spec.gpu or ["any"]) + endpoint_id = session._endpoints.get( + f"dev-{a.name}-{spec.name}", "" + ) or ("per-call" if spec.kind.value == "task" else "") + rows.append((spec.name, spec.kind.value, hardware, endpoint_id)) + return rows + async def _wait_for_rerun(watcher: FileWatcher) -> str: """race the enter key against a file change.""" loop = asyncio.get_event_loop() @@ -164,35 +203,47 @@ async def _wait_for_change(watcher: FileWatcher) -> str: async def _session() -> int: nonlocal apps, entrypoint - session = DevSession(apps) + ui.set_name_width(_all_resource_names(apps)) + session = DevSession(apps, events=ui.DevEvents()) watcher = FileWatcher([module.parent]) - _echo("provisioning dev endpoints ...") + + ui.dev_banner([a.name for a in apps], str(module.name)) await session.start() + ui.console.print() + ui.dev_resources_table(_table_rows(session)) + try: while True: - _echo("--- running entrypoint ---") - try: - run_entrypoint(entrypoint) - except Exception as exc: # noqa: BLE001 - dev loop survives user errors - typer.secho(f"entrypoint error: {exc}", fg=typer.colors.RED) - if once: - return 1 + ui.rule("entrypoint") + with ui.Timer() as t: + try: + run_entrypoint(entrypoint) + ui.success(f"entrypoint [dim]{t.so_far:.1f}s[/dim]") + except Exception as exc: # noqa: BLE001 - dev loop survives user errors + ui.error(f"entrypoint failed [dim]{t.so_far:.1f}s[/dim]") + ui.console.print(f" [red]{exc}[/red]") + if once: + return 1 if once: return 0 - _echo("--- press enter to re-run, edit files to reload, ctrl-c to exit ---") + ui.rule() + ui.dev_hints() reason = await _wait_for_rerun(watcher) if reason == "changed": - _echo("--- change detected: re-scanning and refreshing endpoints ---") + ui.info("[blue]change detected[/blue] reloading ...") apps, entrypoint = _scan() + ui.set_name_width(_all_resource_names(apps)) await session.refresh(apps) finally: - _echo("cleaning up dev endpoints ...") + ui.console.print() + ui.info("cleaning up dev endpoints ...") await session.stop() try: raise typer.Exit(asyncio.run(_session())) except (KeyboardInterrupt, EOFError): - _echo("\nsession ended.") + ui.console.print() + ui.info("session ended.") @app.command() diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index ee8f701db..9b25b60c4 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -279,3 +279,65 @@ def q2(): await session.refresh([app2]) api.delete_endpoint.assert_awaited_once_with("ep-gone") + + +class TestDevEvents: + async def test_lifecycle_events_emitted(self): + events = [] + + class Sink: + def provisioning(self, name, kind, hardware): + events.append(("provisioning", name, kind, hardware)) + + def adopted(self, name, endpoint_id): + events.append(("adopted", name, endpoint_id)) + + def ready(self, name, endpoint_id): + events.append(("ready", name, endpoint_id)) + + def refreshed(self, name, generation): + events.append(("refreshed", name, generation)) + + def deleted(self, name): + events.append(("deleted", name)) + + app = App("ev-app") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.return_value = {"id": "ep-1"} + api.delete_endpoint.return_value = True + + session = DevSession([app], api=api, events=Sink()) + await session.start() + await session.refresh([app]) + await session.stop() + + kinds = [e[0] for e in events] + assert "provisioning" in kinds + assert "ready" in kinds + assert "refreshed" in kinds + assert "deleted" in kinds + assert ("provisioning", "q", "queue", "cpu3c-1-2") in events + # deleted reports the resource name, not the endpoint name + assert ("deleted", "q") in events + + async def test_missing_event_methods_ignored(self): + app = App("ev-app2") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.return_value = {"id": "ep-1"} + + # events object with no handlers at all must not break anything + session = DevSession([app], api=api, events=object()) + await session.start() + await session.stop() From 264b984414bbb80f061a889b89478f36e667d8d0 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 20:55:43 -0700 Subject: [PATCH 030/120] feat: unify runpod and rp entry points with grouped help panels --- pyproject.toml | 2 +- runpod/apps/api.py | 38 +++- runpod/apps/deploy.py | 9 +- runpod/apps/dev.py | 3 + runpod/rp_cli/console.py | 378 ++++++++++++++++++++++++++++++--------- runpod/rp_cli/main.py | 72 +++++--- setup.py | 2 +- 7 files changed, 379 insertions(+), 125 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5e8da034..4a8c870c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ local_scheme = "no-local-version" [project.scripts] -runpod = "runpod.cli.entry:runpod_cli" +runpod = "runpod.rp_cli.main:run" rp = "runpod.rp_cli.main:run" diff --git a/runpod/apps/api.py b/runpod/apps/api.py index ae3d97159..f5c59df94 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -213,15 +213,41 @@ async def deploy_build( ) return data["deployBuildToEnvironment"] - async def upload_tarball(self, upload_url: str, tar_path: str) -> None: - """put the build tarball to the presigned url.""" + async def upload_tarball( + self, + upload_url: str, + tar_path: str, + progress: Optional[Any] = None, + ) -> None: + """put the build tarball to the presigned url. + + progress, when given, is called with (bytes_sent, total_bytes) + as chunks go out. + """ + import os + timeout = aiohttp.ClientTimeout(total=600) - with open(tar_path, "rb") as f: - body = f.read() + total = os.path.getsize(tar_path) + + def _reader(): + sent = 0 + with open(tar_path, "rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + sent += len(chunk) + if progress is not None: + progress(sent, total) + yield chunk + async with aiohttp.ClientSession(timeout=timeout) as session: async with session.put( upload_url, - data=body, - headers={"Content-Type": "application/gzip"}, + data=_reader(), + headers={ + "Content-Type": "application/gzip", + "Content-Length": str(total), + }, ) as resp: resp.raise_for_status() diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 2c59f7d20..fa775416d 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -414,6 +414,7 @@ async def deploy_app( log.info("packaged %s (%.1f MB)", app.name, size_mb) _phase(events, "upload", f"{size_mb:.1f} MB") + upload_progress = getattr(events, "upload_progress", None) remote_app = await client.get_app_by_name(app.name) if remote_app is None: remote_app = await client.create_app(app.name) @@ -429,7 +430,9 @@ async def deploy_app( log.info("created environment %s (%s)", env_name, environment["id"]) upload = await client.prepare_artifact_upload(app_id, tar_size) - await client.upload_tarball(upload["uploadUrl"], str(tar_path)) + await client.upload_tarball( + upload["uploadUrl"], str(tar_path), progress=upload_progress + ) build = await client.finalize_artifact_upload( app_id, upload["objectKey"], manifest ) @@ -442,6 +445,10 @@ async def deploy_app( endpoints = await reconcile_endpoints( client, app, environment, build["id"], python_version ) + endpoint_ready = getattr(events, "endpoint_ready", None) + if endpoint_ready is not None: + for name, endpoint_id in sorted(endpoints.items()): + endpoint_ready(name, endpoint_id) return DeployResult( app_name=app.name, diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 6dccfe4a5..c71e93e1c 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -190,6 +190,7 @@ def _provisionable( async def start(self) -> None: """provision (or adopt) a live endpoint per queue/api resource and register the targets on each app.""" + self._emit("session_starting") existing = {e["name"]: e for e in await self.api.list_my_endpoints()} for app in self.apps: @@ -220,6 +221,8 @@ async def start(self) -> None: self._endpoints[name] = endpoint_id app._dev_targets[spec.name] = LiveTarget(endpoint_id) + self._emit("session_started") + async def refresh(self, apps: List[App]) -> None: """reconcile endpoints against a re-scanned set of apps. diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index e8dcbd2c5..7c19958cf 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -1,23 +1,58 @@ -"""console output for the rp cli. +"""console rendering for the rp cli. -timestamped, column-aligned lifecycle lines for dev sessions and -deploys, in the spirit of a compact build log: - - 14:02:11 double │ provisioning queue cpu3c-1-2 - 14:02:19 double │ ready qjjwnknp - 14:02:20 POST /double - 14:02:21 ✓ double 0.4s +live, animated lifecycle output: spinner-driven phase tracking for +deploys (with byte-level upload progress), per-resource provisioning +animation for dev sessions, and color-coded resource badges. engine +modules stay renderer-free; they emit duck-typed events and this +module turns them into pixels. """ import time from datetime import datetime -from typing import Iterable, Optional - -from rich.console import Console - -console = Console(highlight=False) +from typing import Dict, Iterable, Optional, Tuple + +from rich import box +from rich.console import Console, Group +from rich.panel import Panel +from rich.progress import ( + Progress, + SpinnerColumn, + TaskID, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table +from rich.text import Text +from rich.theme import Theme + +# -- brand ---------------------------------------------------------------- + +ACCENT = "#673de6" # runpod purple +ACCENT_LIGHT = "#a78bfa" +OK = "#34d399" +ERR = "#f87171" +WARN = "#fbbf24" +DIM = "grey58" + +KIND_STYLES: Dict[str, str] = { + "queue": "bold cyan", + "api": "bold magenta", + "task": "bold yellow", +} + +theme = Theme( + { + "accent": ACCENT, + "accent.light": ACCENT_LIGHT, + "ok": OK, + "err": ERR, + "warn": WARN, + "dim": DIM, + } +) + +console = Console(highlight=False, theme=theme) -# shared name column width so pipes line up across resources _name_width: int = 0 @@ -35,10 +70,15 @@ def _ts() -> str: def _pipe(name: str) -> str: - return f"{_padded(name)} [dim]│[/dim]" + return f"[accent.light]{_padded(name)}[/accent.light] [dim]│[/dim]" + + +def kind_badge(kind: str) -> str: + style = KIND_STYLES.get(kind, "bold white") + return f"[{style}]{kind:<5}[/{style}]" -# -- generic -- +# -- generic lines --------------------------------------------------------- def info(message: str) -> None: @@ -46,146 +86,275 @@ def info(message: str) -> None: def success(message: str) -> None: - console.print(f"{_ts()} [green]✓[/green] {message}") + console.print(f"{_ts()} [ok]✓[/ok] {message}") def error(message: str) -> None: - console.print(f"{_ts()} [red]✗[/red] {message}") + console.print(f"{_ts()} [err]✗[/err] {message}") def warn(message: str) -> None: - console.print(f"{_ts()} [yellow]![/yellow] {message}") + console.print(f"{_ts()} [warn]![/warn] {message}") def rule(title: str = "") -> None: - console.rule(f"[dim]{title}[/dim]" if title else None, style="dim") + console.rule(f"[accent]{title}[/accent]" if title else None, style="dim") -# -- resource lifecycle -- - - -def resource_provisioning(name: str, kind: str, hardware: str) -> None: - console.print( - f"{_ts()} {_pipe(name)} [blue]provisioning[/blue] " - f"[dim]{kind} {hardware}[/dim]" - ) +# -- request lifecycle ----------------------------------------------------- -def resource_adopted(name: str, endpoint_id: str) -> None: +def request_started(name: str, label: str = "") -> None: console.print( - f"{_ts()} {_pipe(name)} [blue]adopted[/blue] [dim]{endpoint_id}[/dim]" + f"{_ts()} [bold white]CALL[/bold white] [accent.light]/{name}[/accent.light]" + f" [dim]{label}[/dim]" ) -def resource_ready(name: str, endpoint_id: str) -> None: +def request_completed(name: str, elapsed_s: float) -> None: console.print( - f"{_ts()} {_pipe(name)} [green]ready[/green] [dim]{endpoint_id}[/dim]" + f"{_ts()} [ok]✓[/ok] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" ) -def resource_refreshed(name: str, generation: int) -> None: +def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> None: console.print( - f"{_ts()} {_pipe(name)} [blue]refreshed[/blue] " - f"[dim]generation {generation}[/dim]" + f"{_ts()} [err]✗[/err] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" ) + if err: + console.print(f" [err dim]{err}[/err dim]") -def resource_deleted(name: str) -> None: - console.print(f"{_ts()} {_pipe(name)} [dim]deleted[/dim]") - - -def resource_log(name: str, line: str) -> None: - console.print(f"{_ts()} {_pipe(name)} {line}") - +# -- banners --------------------------------------------------------------- -# -- request lifecycle -- +def _brand_title(subtitle: str) -> Text: + title = Text() + title.append("◤ ", style=ACCENT) + title.append("runpod", style=f"bold {ACCENT_LIGHT}") + title.append(" ▸ ", style=DIM) + title.append(subtitle, style="bold white") + return title -def request_started(name: str, label: str = "") -> None: - console.print(f"{_ts()} [white]CALL[/white] /{name} [dim]{label}[/dim]") - -def request_completed(name: str, elapsed_s: float) -> None: +def dev_banner(app_names: Iterable[str], module: str) -> None: + body = Text() + body.append("apps ", style=DIM) + body.append(", ".join(app_names), style="white") + body.append("\nmodule ", style=DIM) + body.append(module, style="white") + console.print() console.print( - f"{_ts()} [green]✓[/green] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + Panel( + body, + title=_brand_title("dev"), + title_align="left", + border_style=ACCENT, + box=box.ROUNDED, + padding=(0, 2), + expand=False, + ) ) -def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> None: +def deploy_banner(app_name: str, env: str, resources: Iterable[Tuple[str, str]]) -> None: + """resources: (name, kind) pairs.""" + body = Text() + first = True + for name, kind in resources: + if not first: + body.append("\n") + body.append(f"{kind:<6}", style=KIND_STYLES.get(kind, "white")) + body.append(f" {name}", style="white") + first = False + if first: + body.append("(no resources)", style=DIM) + console.print() console.print( - f"{_ts()} [red]✗[/red] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + Panel( + body, + title=_brand_title(f"deploy · {app_name} → {env}"), + title_align="left", + border_style=ACCENT, + box=box.ROUNDED, + padding=(0, 2), + expand=False, + ) ) - if err: - console.print(f" [dim]{err}[/dim]") - - -# -- dev session chrome -- - - -def dev_banner(app_names: Iterable[str], module: str) -> None: - apps = ", ".join(app_names) - console.print() - console.print(f"[green]✓[/green] [bold]rp dev[/bold] [dim]{module}[/dim]") - console.print(f" [dim]apps[/dim] {apps}") - console.print() -def dev_resources_table(rows: Iterable[tuple]) -> None: +def resources_table(rows: Iterable[tuple]) -> None: """rows of (name, kind, hardware, endpoint_id).""" rows = list(rows) if not rows: - console.print(" [dim](no queue/api resources; tasks run per-call)[/dim]") + console.print(" [dim](no resources)[/dim]") return - w_name = max(len(r[0]) for r in rows) - w_kind = max(len(r[1]) for r in rows) - w_hw = max(len(r[2]) for r in rows) + table = Table( + box=box.SIMPLE_HEAD, + border_style="dim", + header_style=f"bold {ACCENT_LIGHT}", + pad_edge=False, + padding=(0, 2), + ) + table.add_column("resource", style="bold white") + table.add_column("kind") + table.add_column("hardware", style="dim") + table.add_column("endpoint", style=ACCENT_LIGHT) for name, kind, hardware, endpoint_id in rows: - console.print( - f" [white]{name:<{w_name}}[/white]" - f" [dim]{kind:<{w_kind}}[/dim]" - f" [dim]{hardware:<{w_hw}}[/dim]" - f" [dim]{endpoint_id}[/dim]" + table.add_row( + name, + kind_badge(kind), + hardware, + endpoint_id or "[dim]—[/dim]", ) - console.print() + console.print(table) def dev_hints() -> None: console.print( - " [dim]enter[/dim] re-run [dim]edit[/dim] reload " - "[dim]ctrl-c[/dim] exit" + f" [{ACCENT_LIGHT}]⏎[/{ACCENT_LIGHT}] [dim]re-run[/dim] " + f"[{ACCENT_LIGHT}]✎[/{ACCENT_LIGHT}] [dim]edit to reload[/dim] " + f"[{ACCENT_LIGHT}]^C[/{ACCENT_LIGHT}] [dim]exit[/dim]" ) console.print() -def phase(name: str, detail: str = "") -> None: - suffix = f" [dim]{detail}[/dim]" if detail else "" - console.print(f"{_ts()} [blue]{name}[/blue]{suffix}") +def reload_flash() -> None: + console.print() + console.rule(f"[{WARN}]⚡ reload[/{WARN}]", style=WARN) + + +# -- live deploy phases ------------------------------------------------------ + +_PHASE_LABELS = { + "vendor": "resolving dependencies", + "package": "packaging artifact", + "upload": "uploading build", + "endpoints": "reconciling endpoints", +} class DeployEvents: - """deploy_app event sink rendering phase lines.""" + """deploy_app event sink: animated phase list with live progress. + + each phase renders a spinner while active and collapses to a ✓ with + elapsed time when the next phase begins. upload progress (bytes) + animates in-line when the transport reports it. + """ + + def __init__(self) -> None: + self._progress = Progress( + SpinnerColumn("dots", style=ACCENT, finished_text=f"[ok]✓[/ok]"), + TextColumn("[white]{task.description}[/white]"), + TextColumn("[dim]{task.fields[detail]}[/dim]"), + TimeElapsedColumn(), + console=console, + transient=False, + ) + self._current: Optional[TaskID] = None + self._started = False + + def _ensure_started(self) -> None: + if not self._started: + self._progress.start() + self._started = True def phase(self, name: str, detail: str = "") -> None: - phase(name, detail) + self._ensure_started() + if self._current is not None: + self._progress.update(self._current, total=1, completed=1) + self._current = self._progress.add_task( + _PHASE_LABELS.get(name, name), detail=detail, total=None + ) + + def upload_progress(self, sent: int, total: int) -> None: + if self._current is None: + return + mb = 1024 * 1024 + self._progress.update( + self._current, + detail=f"{sent / mb:.1f} / {total / mb:.1f} MB", + ) + + def endpoint_ready(self, name: str, endpoint_id: str) -> None: + self._progress.console.print( + f" {_pipe(name)} [ok]ready[/ok] [dim]{endpoint_id}[/dim]" + ) + + def close(self) -> None: + if self._current is not None: + self._progress.update(self._current, total=1, completed=1) + self._current = None + if self._started: + self._progress.stop() + self._started = False + + +# -- live dev provisioning ---------------------------------------------------- class DevEvents: - """DevSession event sink rendering lifecycle lines.""" + """DevSession event sink: per-resource spinner rows during start, + plain lifecycle lines afterwards. + """ + + def __init__(self) -> None: + self._progress: Optional[Progress] = None + self._tasks: Dict[str, TaskID] = {} + + def session_starting(self) -> None: + self._progress = Progress( + SpinnerColumn("dots", style=ACCENT, finished_text="[ok]✓[/ok]"), + TextColumn("[accent.light]{task.description}[/accent.light]"), + TextColumn("[dim]{task.fields[detail]}[/dim]"), + console=console, + ) + self._progress.start() + + def session_started(self) -> None: + if self._progress is not None: + self._progress.stop() + self._progress = None + self._tasks.clear() + + def _row(self, name: str, detail: str) -> None: + if self._progress is None: + info(f"{_pipe(name)} {detail}") + return + if name not in self._tasks: + self._tasks[name] = self._progress.add_task( + _padded(name), detail=detail, total=None + ) + else: + self._progress.update(self._tasks[name], detail=detail) def provisioning(self, name: str, kind: str, hardware: str) -> None: - resource_provisioning(name, kind, hardware) + self._row(name, f"provisioning {kind} · {hardware}") def adopted(self, name: str, endpoint_id: str) -> None: - resource_adopted(name, endpoint_id) + self._row(name, f"adopted {endpoint_id}") def ready(self, name: str, endpoint_id: str) -> None: - resource_ready(name, endpoint_id) + if self._progress is not None and name in self._tasks: + self._progress.update( + self._tasks[name], + detail=f"ready · {endpoint_id}", + total=1, + completed=1, + ) + else: + console.print( + f"{_ts()} {_pipe(name)} [ok]ready[/ok] [dim]{endpoint_id}[/dim]" + ) def refreshed(self, name: str, generation: int) -> None: - resource_refreshed(name, generation) + console.print( + f"{_ts()} {_pipe(name)} [accent]refreshed[/accent] " + f"[dim]generation {generation}[/dim]" + ) def deleted(self, name: str) -> None: - resource_deleted(name) + console.print(f"{_ts()} {_pipe(name)} [dim]deleted[/dim]") class Timer: @@ -201,3 +370,38 @@ def __exit__(self, *exc) -> None: @property def so_far(self) -> float: return time.monotonic() - self._start + + +def entrypoint_header() -> None: + console.print() + console.rule(f"[{ACCENT_LIGHT}]▶ entrypoint[/{ACCENT_LIGHT}]", style="dim") + + +def entrypoint_success(elapsed: float) -> None: + console.rule(f"[ok]✓ {elapsed:.1f}s[/ok]", style="dim") + + +def entrypoint_failure(elapsed: float, err: str) -> None: + console.rule(f"[err]✗ {elapsed:.1f}s[/err]", style="dim") + console.print(f" [err]{err}[/err]") + + +def session_summary(runs: int, reloads: int, elapsed: float) -> None: + minutes, seconds = divmod(int(elapsed), 60) + console.print() + console.print( + Panel( + Text.assemble( + ("runs ", DIM), + (str(runs), "white"), + (" reloads ", DIM), + (str(reloads), "white"), + (" session ", DIM), + (f"{minutes}m {seconds}s" if minutes else f"{seconds}s", "white"), + ), + border_style="dim", + box=box.ROUNDED, + padding=(0, 2), + expand=False, + ) + ) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index d4ee54117..20be7db12 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -17,8 +17,13 @@ help="Runpod CLI: deploy and manage apps, endpoints, and pods.", no_args_is_help=True, pretty_exceptions_show_locals=False, + rich_markup_mode="rich", ) +APPS_PANEL = "Apps" +OBSERVE_PANEL = "Observability" +ACCOUNT_PANEL = "Account" + def _echo(message: str) -> None: typer.echo(message) @@ -29,7 +34,7 @@ def _fail(message: str) -> None: raise typer.Exit(1) -@app.command() +@app.command(rich_help_panel=APPS_PANEL) def deploy( target: Optional[Path] = typer.Argument( None, @@ -81,32 +86,38 @@ def deploy( for found in apps: resource_names = list(found.resources) ui.set_name_width(resource_names) - ui.console.print() - ui.console.print( - f"[bold]{found.name}[/bold] [dim]-> {env or found.env}[/dim]" + ui.deploy_banner( + found.name, + env or found.env, + [ + (h.spec.name, h.spec.kind.value) + for h in found.resources.values() + ], ) + events = ui.DeployEvents() with ui.Timer() as t: - result = asyncio.run( - deploy_app( - found, - project_root, - env_name=env, - python_version=python_version, - exclude=exclude.split(",") if exclude else None, - events=ui.DeployEvents(), + try: + result = asyncio.run( + deploy_app( + found, + project_root, + env_name=env, + python_version=python_version, + exclude=exclude.split(",") if exclude else None, + events=events, + ) ) - ) - for name, endpoint_id in sorted(result.endpoints.items()): - ui.resource_ready(name, endpoint_id) + finally: + events.close() ui.success( - f"build [white]{result.build_id}[/white] live on " - f"[white]{result.app_name}/{env or found.env}[/white] " + f"build [accent.light]{result.build_id}[/accent.light] live on " + f"[bold white]{result.app_name}/{env or found.env}[/bold white] " f"[dim]{t.elapsed:.1f}s[/dim]" ) ui.console.print() -@app.command() +@app.command(rich_help_panel=APPS_PANEL) def dev( module: Path = typer.Argument(..., help="Python module to run."), once: bool = typer.Option( @@ -209,33 +220,36 @@ async def _session() -> int: ui.dev_banner([a.name for a in apps], str(module.name)) await session.start() - ui.console.print() - ui.dev_resources_table(_table_rows(session)) + ui.resources_table(_table_rows(session)) + runs = 0 + reloads = 0 + session_timer = ui.Timer().__enter__() try: while True: - ui.rule("entrypoint") + ui.entrypoint_header() + runs += 1 with ui.Timer() as t: try: run_entrypoint(entrypoint) - ui.success(f"entrypoint [dim]{t.so_far:.1f}s[/dim]") + ui.entrypoint_success(t.so_far) except Exception as exc: # noqa: BLE001 - dev loop survives user errors - ui.error(f"entrypoint failed [dim]{t.so_far:.1f}s[/dim]") - ui.console.print(f" [red]{exc}[/red]") + ui.entrypoint_failure(t.so_far, str(exc)) if once: return 1 if once: return 0 - ui.rule() ui.dev_hints() reason = await _wait_for_rerun(watcher) if reason == "changed": - ui.info("[blue]change detected[/blue] reloading ...") + reloads += 1 + ui.reload_flash() apps, entrypoint = _scan() ui.set_name_width(_all_resource_names(apps)) await session.refresh(apps) finally: - ui.console.print() + session_timer.__exit__() + ui.session_summary(runs, reloads, session_timer.elapsed) ui.info("cleaning up dev endpoints ...") await session.stop() @@ -246,7 +260,7 @@ async def _session() -> int: ui.info("session ended.") -@app.command() +@app.command(rich_help_panel=OBSERVE_PANEL) def logs( pod_id: str = typer.Argument(..., help="Pod id to fetch logs for."), follow: bool = typer.Option( @@ -279,7 +293,7 @@ async def _run(): pass -@app.command() +@app.command(rich_help_panel=ACCOUNT_PANEL) def login(): """authenticate with runpod and store the api key.""" from runpod.cli.groups.config.functions import set_credentials diff --git a/setup.py b/setup.py index a98c375eb..7f8a076e8 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ }, entry_points={ "console_scripts": [ - "runpod = runpod.cli.entry:runpod_cli", + "runpod = runpod.rp_cli.main:run", "rp = runpod.rp_cli.main:run", ] }, From db7d26f2bcf05fa13f26133c13217e48d546ca14 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:11:47 -0700 Subject: [PATCH 031/120] feat: rich help rendering for legacy click subcommands --- requirements.txt | 1 + runpod/rp_cli/main.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7d7671502..16604c2b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ py-cpuinfo >= 9.0.0 inquirerpy == 0.3.4 requests >= 2.31.0 rich >= 13.0.0 +rich-click >= 1.8.0 tomli >= 2.4.1 tomlkit >= 0.15.0 tqdm-loggable >= 0.2 diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 20be7db12..e4ccad201 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -308,6 +308,12 @@ def login(): def _mount_legacy_groups() -> None: """mount the existing click groups so `rp pod ...` etc work.""" + # patch click before the legacy modules import it so their groups + # and subcommands render with the same rich help styling + from rich_click.patch import patch as _rich_click_patch + + _rich_click_patch() + import typer.main as typer_main from runpod.cli.groups.config.commands import config_wizard @@ -326,7 +332,14 @@ def _mount_legacy_groups() -> None: def run() -> None: """console-script entry point.""" - cli() # noqa: F821 - bound by _mount_legacy_groups + import click.exceptions + + try: + cli() # noqa: F821 - bound by _mount_legacy_groups + except click.exceptions.Exit as exc: + # rich-click's eager --help callback exits via the click Exit + # exception, which escapes the mixed typer/click stack here + sys.exit(exc.exit_code) if __name__ == "__main__": From fcd4a6ae5f953724ffb02a999f13aac027e5b0f8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:14:12 -0700 Subject: [PATCH 032/120] fix: render click usage errors instead of tracebacks --- runpod/rp_cli/main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index e4ccad201..8df548e1e 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -340,6 +340,12 @@ def run() -> None: # rich-click's eager --help callback exits via the click Exit # exception, which escapes the mixed typer/click stack here sys.exit(exc.exit_code) + except click.exceptions.ClickException as exc: + # usage errors from the mounted click groups (bad subcommand, + # missing arg) escape typer's handler; render them the click + # way instead of a traceback + exc.show() + sys.exit(exc.exit_code) if __name__ == "__main__": From 0be0f2ff7d00a26f59562574e6fa7a0b3d76106a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:19:04 -0700 Subject: [PATCH 033/120] refactor: single plain-click command tree without help panels --- requirements.txt | 1 - runpod/rp_cli/console.py | 112 +++++++-------------- runpod/rp_cli/main.py | 207 +++++++++++++++++---------------------- 3 files changed, 123 insertions(+), 197 deletions(-) diff --git a/requirements.txt b/requirements.txt index 16604c2b7..7d7671502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ py-cpuinfo >= 9.0.0 inquirerpy == 0.3.4 requests >= 2.31.0 rich >= 13.0.0 -rich-click >= 1.8.0 tomli >= 2.4.1 tomlkit >= 0.15.0 tqdm-loggable >= 0.2 diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 7c19958cf..d34bb7d79 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -11,9 +11,7 @@ from datetime import datetime from typing import Dict, Iterable, Optional, Tuple -from rich import box -from rich.console import Console, Group -from rich.panel import Panel +from rich.console import Console from rich.progress import ( Progress, SpinnerColumn, @@ -21,7 +19,6 @@ TextColumn, TimeElapsedColumn, ) -from rich.table import Table from rich.text import Text from rich.theme import Theme @@ -128,59 +125,31 @@ def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> No # -- banners --------------------------------------------------------------- -def _brand_title(subtitle: str) -> Text: - title = Text() - title.append("◤ ", style=ACCENT) - title.append("runpod", style=f"bold {ACCENT_LIGHT}") - title.append(" ▸ ", style=DIM) - title.append(subtitle, style="bold white") - return title +def _brand(subtitle: str, detail: str = "") -> None: + line = Text() + line.append("◤ ", style=ACCENT) + line.append("runpod", style=f"bold {ACCENT_LIGHT}") + line.append(" ▸ ", style=DIM) + line.append(subtitle, style="bold white") + if detail: + line.append(f" {detail}", style=DIM) + console.print() + console.print(line) def dev_banner(app_names: Iterable[str], module: str) -> None: - body = Text() - body.append("apps ", style=DIM) - body.append(", ".join(app_names), style="white") - body.append("\nmodule ", style=DIM) - body.append(module, style="white") + _brand("dev", f"{', '.join(app_names)} · {module}") console.print() - console.print( - Panel( - body, - title=_brand_title("dev"), - title_align="left", - border_style=ACCENT, - box=box.ROUNDED, - padding=(0, 2), - expand=False, - ) - ) def deploy_banner(app_name: str, env: str, resources: Iterable[Tuple[str, str]]) -> None: """resources: (name, kind) pairs.""" - body = Text() - first = True + _brand("deploy", f"{app_name} → {env}") for name, kind in resources: - if not first: - body.append("\n") - body.append(f"{kind:<6}", style=KIND_STYLES.get(kind, "white")) - body.append(f" {name}", style="white") - first = False - if first: - body.append("(no resources)", style=DIM) - console.print() - console.print( - Panel( - body, - title=_brand_title(f"deploy · {app_name} → {env}"), - title_align="left", - border_style=ACCENT, - box=box.ROUNDED, - padding=(0, 2), - expand=False, + console.print( + f" {kind_badge(kind)} [white]{name}[/white]" ) - ) + console.print() def resources_table(rows: Iterable[tuple]) -> None: @@ -189,25 +158,22 @@ def resources_table(rows: Iterable[tuple]) -> None: if not rows: console.print(" [dim](no resources)[/dim]") return - table = Table( - box=box.SIMPLE_HEAD, - border_style="dim", - header_style=f"bold {ACCENT_LIGHT}", - pad_edge=False, - padding=(0, 2), - ) - table.add_column("resource", style="bold white") - table.add_column("kind") - table.add_column("hardware", style="dim") - table.add_column("endpoint", style=ACCENT_LIGHT) + w_name = max(len(r[0]) for r in rows) + w_hw = max(len(r[2]) for r in rows) + console.print() for name, kind, hardware, endpoint_id in rows: - table.add_row( - name, - kind_badge(kind), - hardware, - endpoint_id or "[dim]—[/dim]", + endpoint = ( + f"[accent.light]{endpoint_id}[/accent.light]" + if endpoint_id + else "[dim]—[/dim]" ) - console.print(table) + console.print( + f" [bold white]{name:<{w_name}}[/bold white]" + f" {kind_badge(kind)}" + f" [dim]{hardware:<{w_hw}}[/dim]" + f" {endpoint}" + ) + console.print() def dev_hints() -> None: @@ -388,20 +354,10 @@ def entrypoint_failure(elapsed: float, err: str) -> None: def session_summary(runs: int, reloads: int, elapsed: float) -> None: minutes, seconds = divmod(int(elapsed), 60) + duration = f"{minutes}m {seconds}s" if minutes else f"{seconds}s" console.print() console.print( - Panel( - Text.assemble( - ("runs ", DIM), - (str(runs), "white"), - (" reloads ", DIM), - (str(reloads), "white"), - (" session ", DIM), - (f"{minutes}m {seconds}s" if minutes else f"{seconds}s", "white"), - ), - border_style="dim", - box=box.ROUNDED, - padding=(0, 2), - expand=False, - ) + f" [dim]runs[/dim] [white]{runs}[/white]" + f" [dim]reloads[/dim] [white]{reloads}[/white]" + f" [dim]session[/dim] [white]{duration}[/white]" ) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 8df548e1e..8949bd5b5 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -1,62 +1,50 @@ -"""the `rp` cli: deploy, dev, login, plus mounted legacy groups. +"""the `rp` / `runpod` cli. -typer is the root; the existing click groups from runpod.cli mount -underneath so `rp pod ...`, `rp ssh ...`, etc keep working. +a single plain-click command tree: app commands (deploy, dev, logs, +login) and the pod/ssh/exec/config groups all hang off one root, so +help, errors, and exit codes behave identically at every level. rich +is used only for runtime output (progress, lifecycle lines), never for +help rendering. """ import asyncio import os import sys from pathlib import Path -from typing import Optional -import typer +import click -app = typer.Typer( - name="rp", - help="Runpod CLI: deploy and manage apps, endpoints, and pods.", - no_args_is_help=True, - pretty_exceptions_show_locals=False, - rich_markup_mode="rich", -) -APPS_PANEL = "Apps" -OBSERVE_PANEL = "Observability" -ACCOUNT_PANEL = "Account" +@click.group() +def cli(): + """Runpod CLI: deploy and manage apps, endpoints, and pods.""" + +def _fail(message: str) -> None: + raise click.ClickException(message) -def _echo(message: str) -> None: - typer.echo(message) +# ---------------------------------------------------------------- deploy -def _fail(message: str) -> None: - typer.secho(message, fg=typer.colors.RED, err=True) - raise typer.Exit(1) - - -@app.command(rich_help_panel=APPS_PANEL) -def deploy( - target: Optional[Path] = typer.Argument( - None, - help="Directory or file to deploy (defaults to current directory).", - ), - env: Optional[str] = typer.Option( - None, "--env", "-e", help="Target environment name." - ), - python_version: str = typer.Option( - "3.12", - "--python-version", - help="Worker python version for dependency wheels (3.10-3.14).", - ), - exclude: Optional[str] = typer.Option( - None, - "--exclude", - help="Comma-separated packages to exclude from the artifact " - "(they must then come from the worker image). torch packages " - "are always excluded.", - ), -): - """package and deploy all apps found in TARGET.""" + +@cli.command() +@click.argument("target", required=False, type=click.Path(path_type=Path)) +@click.option("--env", "-e", "env", default=None, help="Target environment name.") +@click.option( + "--python-version", + default="3.12", + show_default=True, + help="Worker python version for dependency wheels (3.10-3.14).", +) +@click.option( + "--exclude", + default=None, + help="Comma-separated packages to exclude from the artifact " + "(they must then come from the worker image). Torch packages are " + "excluded automatically on builtin gpu images.", +) +def deploy(target, env, python_version, exclude): + """Package and deploy all apps found in TARGET (default: cwd).""" import logging from runpod.apps.deploy import deploy_app @@ -80,19 +68,15 @@ def deploy( _fail( "no runpod.App found. define one with:\n\n" " from runpod import App\n" - ' app = App("my-app")\n' + ' app = App("my-app")' ) for found in apps: - resource_names = list(found.resources) - ui.set_name_width(resource_names) + ui.set_name_width(list(found.resources)) ui.deploy_banner( found.name, env or found.env, - [ - (h.spec.name, h.spec.kind.value) - for h in found.resources.values() - ], + [(h.spec.name, h.spec.kind.value) for h in found.resources.values()], ) events = ui.DeployEvents() with ui.Timer() as t: @@ -117,40 +101,40 @@ def deploy( ui.console.print() -@app.command(rich_help_panel=APPS_PANEL) -def dev( - module: Path = typer.Argument(..., help="Python module to run."), - once: bool = typer.Option( - False, - "--once", - help="Run the entrypoint once, tear down, and exit (for scripts/CI).", - ), -): - """start an interactive dev session for MODULE. +# ------------------------------------------------------------------- dev - provisions temporary live endpoints, runs the module's + +@cli.command() +@click.argument("module", type=click.Path(exists=True, path_type=Path)) +@click.option( + "--once", + is_flag=True, + help="Run the entrypoint once, tear down, and exit (for scripts/CI).", +) +def dev(module, once): + """Start an interactive dev session for MODULE. + + Provisions temporary live endpoints, runs the module's @runpod.local_entrypoint, watches for file changes (re-scanning and refreshing endpoints so requests run fresh code), and deletes the endpoints on exit. """ + import logging + from runpod.apps.dev import DevSession from runpod.apps.discovery import DiscoveryError, discover_apps from runpod.apps.entrypoint import get_entrypoint, run_entrypoint from runpod.apps.watch import FileWatcher from runpod.rp_cli import console as ui + logging.getLogger("runpod.apps").setLevel(logging.WARNING) + module = module.resolve() if not module.is_file(): _fail(f"{module} is not a file") os.environ["RUNPOD_DEV_SESSION"] = "1" - # the console renders lifecycle lines; keep the module loggers to - # warnings so output isn't duplicated - import logging - - logging.getLogger("runpod.apps").setLevel(logging.WARNING) - def _scan(): try: apps = discover_apps(module) @@ -163,7 +147,7 @@ def _scan(): _fail( "no @runpod.local_entrypoint found. define one:\n\n" " @runpod.local_entrypoint\n" - " async def main(): ...\n" + " async def main(): ..." ) return apps, entrypoint @@ -176,7 +160,7 @@ def _all_resource_names(app_list) -> list: for handle in a.resources.values() ] - def _table_rows(session: DevSession) -> list: + def _table_rows(session: "DevSession") -> list: rows = [] for a in session.apps: for handle in a.resources.values(): @@ -188,7 +172,7 @@ def _table_rows(session: DevSession) -> list: rows.append((spec.name, spec.kind.value, hardware, endpoint_id)) return rows - async def _wait_for_rerun(watcher: FileWatcher) -> str: + async def _wait_for_rerun(watcher: "FileWatcher") -> str: """race the enter key against a file change.""" loop = asyncio.get_event_loop() stdin_task = loop.run_in_executor(None, sys.stdin.readline) @@ -206,7 +190,7 @@ async def _wait_for_rerun(watcher: FileWatcher) -> str: finally: stdin_task.cancel() - async def _wait_for_change(watcher: FileWatcher) -> str: + async def _wait_for_change(watcher: "FileWatcher") -> str: while True: if watcher.changed(): return "changed" @@ -254,24 +238,29 @@ async def _session() -> int: await session.stop() try: - raise typer.Exit(asyncio.run(_session())) + sys.exit(asyncio.run(_session())) except (KeyboardInterrupt, EOFError): ui.console.print() ui.info("session ended.") -@app.command(rich_help_panel=OBSERVE_PANEL) -def logs( - pod_id: str = typer.Argument(..., help="Pod id to fetch logs for."), - follow: bool = typer.Option( - False, "--follow", "-f", help="Stream logs as they arrive." - ), - log_type: str = typer.Option( - "all", "--type", help="all, container, or system." - ), - tail: int = typer.Option(100, "--tail", help="Lines of backfill."), -): - """show a pod's container and system logs (host api).""" +# ------------------------------------------------------------------ logs + + +@cli.command() +@click.argument("pod_id") +@click.option("--follow", "-f", is_flag=True, help="Stream logs as they arrive.") +@click.option( + "--type", + "log_type", + default="all", + show_default=True, + type=click.Choice(["all", "container", "system"]), + help="Log source.", +) +@click.option("--tail", default=100, show_default=True, help="Lines of backfill.") +def logs(pod_id, follow, log_type, tail): + """Show a pod's container and system logs.""" from runpod.apps.logs import pod_logs, stream_pod_logs async def _run(): @@ -280,12 +269,12 @@ async def _run(): pod_id, log_type=log_type, tail=tail ): source = entry.get("source", "?") - _echo(f"[{source}] {entry.get('line', '')}") + click.echo(f"[{source}] {entry.get('line', '')}") else: logs_data = await pod_logs(pod_id, log_type=log_type) for source in ("system", "container"): for line in logs_data.get(source) or []: - _echo(f"[{source}] {line}") + click.echo(f"[{source}] {line}") try: asyncio.run(_run()) @@ -293,59 +282,41 @@ async def _run(): pass -@app.command(rich_help_panel=ACCOUNT_PANEL) +# ----------------------------------------------------------------- login + + +@cli.command() def login(): - """authenticate with runpod and store the api key.""" + """Authenticate with Runpod and store the API key.""" from runpod.cli.groups.config.functions import set_credentials - api_key = typer.prompt("Runpod API key", hide_input=True) + api_key = click.prompt("Runpod API key", hide_input=True) try: set_credentials(api_key, overwrite=True) except ValueError as exc: _fail(str(exc)) - _echo("credentials saved to ~/.runpod/config.toml") + click.echo("credentials saved to ~/.runpod/config.toml") -def _mount_legacy_groups() -> None: - """mount the existing click groups so `rp pod ...` etc work.""" - # patch click before the legacy modules import it so their groups - # and subcommands render with the same rich help styling - from rich_click.patch import patch as _rich_click_patch +# -------------------------------------------------------- legacy groups - _rich_click_patch() - - import typer.main as typer_main +def _mount_groups() -> None: from runpod.cli.groups.config.commands import config_wizard from runpod.cli.groups.exec.commands import exec_cli from runpod.cli.groups.pod.commands import pod_cli from runpod.cli.groups.ssh.commands import ssh_cli - click_app = typer_main.get_command(app) for command in (config_wizard, ssh_cli, pod_cli, exec_cli): - click_app.add_command(command) - globals()["cli"] = click_app + cli.add_command(command) -_mount_legacy_groups() +_mount_groups() def run() -> None: """console-script entry point.""" - import click.exceptions - - try: - cli() # noqa: F821 - bound by _mount_legacy_groups - except click.exceptions.Exit as exc: - # rich-click's eager --help callback exits via the click Exit - # exception, which escapes the mixed typer/click stack here - sys.exit(exc.exit_code) - except click.exceptions.ClickException as exc: - # usage errors from the mounted click groups (bad subcommand, - # missing arg) escape typer's handler; render them the click - # way instead of a traceback - exc.show() - sys.exit(exc.exit_code) + cli() if __name__ == "__main__": From abc0024b28310115df794a74ff32dcc96a862fb1 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:36:56 -0700 Subject: [PATCH 034/120] fix: expand gpu any to pool ids and quiet the dev loop ui --- .github/workflows/runtimes.yml | 5 +-- runpod/apps/deploy.py | 4 ++- runpod/apps/dev.py | 4 ++- runpod/apps/gpu.py | 11 ++++++ runpod/rp_cli/console.py | 65 ++++++++-------------------------- runpod/rp_cli/main.py | 16 ++++----- tests/test_apps/test_dev.py | 26 ++++++++++++++ 7 files changed, 66 insertions(+), 65 deletions(-) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 662426ce2..4f02bc7c4 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -14,8 +14,9 @@ on: env: REGISTRY: docker.io - # dev-suffixed tags for non-main branches so testing never clobbers latest - TAG_SUFFIX: ${{ github.ref == 'refs/heads/main' && 'latest' || 'dev' }} + # the apps sdk branch publishes latest while the feature is prerelease: + # the runtime image names are new, so nothing released depends on them + TAG_SUFFIX: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/apps-sdk') && 'latest' || 'dev' }} jobs: test-runner: diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index fa775416d..bb48c7c44 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -315,7 +315,9 @@ def _deployed_endpoint_input( dc.value for dc in CPU3_DATACENTERS ) else: - payload["gpuIds"] = ",".join(spec.gpu or ["any"]) + from .gpu import gpu_ids_value + + payload["gpuIds"] = gpu_ids_value(spec.gpu) payload["gpuCount"] = spec.gpu_count if not spec.datacenter: # artifact delivery rides the flash volume (network diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index c71e93e1c..5fd9fdd7d 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -138,7 +138,9 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: if not spec.datacenter: payload["locations"] = _cpu_locations(spec.cpu or []) else: - payload["gpuIds"] = ",".join(spec.gpu or ["any"]) + from .gpu import gpu_ids_value + + payload["gpuIds"] = gpu_ids_value(spec.gpu) payload["gpuCount"] = spec.gpu_count return payload diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py index 251d61de0..ddddec307 100644 --- a/runpod/apps/gpu.py +++ b/runpod/apps/gpu.py @@ -154,3 +154,14 @@ class CpuInstanceType(str, Enum): GpuLike = Union[GpuGroup, GpuType, str] + + +def gpu_ids_value(gpu: Optional[List[str]]) -> str: + """the gpuIds string for an endpoint payload. + + "any gpu" (no selection, or the ANY sentinel) means every pool id: + the api has no wildcard and rejects "any". + """ + if not gpu or any(str(g).lower() == "any" for g in gpu): + return ",".join(g.value for g in GpuGroup.all()) + return ",".join(str(g) for g in gpu) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index d34bb7d79..6959310df 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -125,69 +125,43 @@ def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> No # -- banners --------------------------------------------------------------- -def _brand(subtitle: str, detail: str = "") -> None: - line = Text() - line.append("◤ ", style=ACCENT) - line.append("runpod", style=f"bold {ACCENT_LIGHT}") - line.append(" ▸ ", style=DIM) - line.append(subtitle, style="bold white") - if detail: - line.append(f" {detail}", style=DIM) - console.print() - console.print(line) - - def dev_banner(app_names: Iterable[str], module: str) -> None: - _brand("dev", f"{', '.join(app_names)} · {module}") - console.print() + console.print( + f"[bold white]dev[/bold white] [dim]{', '.join(app_names)} · {module}[/dim]" + ) def deploy_banner(app_name: str, env: str, resources: Iterable[Tuple[str, str]]) -> None: """resources: (name, kind) pairs.""" - _brand("deploy", f"{app_name} → {env}") - for name, kind in resources: - console.print( - f" {kind_badge(kind)} [white]{name}[/white]" - ) - console.print() + names = ", ".join(name for name, _ in resources) or "(no resources)" + console.print( + f"[bold white]deploy[/bold white] [dim]{app_name} → {env} · {names}[/dim]" + ) def resources_table(rows: Iterable[tuple]) -> None: """rows of (name, kind, hardware, endpoint_id).""" rows = list(rows) if not rows: - console.print(" [dim](no resources)[/dim]") return w_name = max(len(r[0]) for r in rows) w_hw = max(len(r[2]) for r in rows) - console.print() for name, kind, hardware, endpoint_id in rows: endpoint = ( f"[accent.light]{endpoint_id}[/accent.light]" if endpoint_id - else "[dim]—[/dim]" + else "[dim]per-call[/dim]" ) console.print( - f" [bold white]{name:<{w_name}}[/bold white]" + f" [white]{name:<{w_name}}[/white]" f" {kind_badge(kind)}" f" [dim]{hardware:<{w_hw}}[/dim]" f" {endpoint}" ) - console.print() - - -def dev_hints() -> None: - console.print( - f" [{ACCENT_LIGHT}]⏎[/{ACCENT_LIGHT}] [dim]re-run[/dim] " - f"[{ACCENT_LIGHT}]✎[/{ACCENT_LIGHT}] [dim]edit to reload[/dim] " - f"[{ACCENT_LIGHT}]^C[/{ACCENT_LIGHT}] [dim]exit[/dim]" - ) - console.print() def reload_flash() -> None: - console.print() - console.rule(f"[{WARN}]⚡ reload[/{WARN}]", style=WARN) + console.print(f"[warn]⚡ reload[/warn]") # -- live deploy phases ------------------------------------------------------ @@ -340,24 +314,13 @@ def so_far(self) -> float: def entrypoint_header() -> None: console.print() - console.rule(f"[{ACCENT_LIGHT}]▶ entrypoint[/{ACCENT_LIGHT}]", style="dim") def entrypoint_success(elapsed: float) -> None: - console.rule(f"[ok]✓ {elapsed:.1f}s[/ok]", style="dim") + console.print( + f"[ok]✓[/ok] [dim]{elapsed:.1f}s · enter to re-run · edit to reload · ^C to quit[/dim]" + ) def entrypoint_failure(elapsed: float, err: str) -> None: - console.rule(f"[err]✗ {elapsed:.1f}s[/err]", style="dim") - console.print(f" [err]{err}[/err]") - - -def session_summary(runs: int, reloads: int, elapsed: float) -> None: - minutes, seconds = divmod(int(elapsed), 60) - duration = f"{minutes}m {seconds}s" if minutes else f"{seconds}s" - console.print() - console.print( - f" [dim]runs[/dim] [white]{runs}[/white]" - f" [dim]reloads[/dim] [white]{reloads}[/white]" - f" [dim]session[/dim] [white]{duration}[/white]" - ) + console.print(f"[err]✗ {err}[/err] [dim]{elapsed:.1f}s[/dim]") diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 8949bd5b5..77a038a67 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -91,6 +91,8 @@ def deploy(target, env, python_version, exclude): events=events, ) ) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc finally: events.close() ui.success( @@ -206,13 +208,9 @@ async def _session() -> int: await session.start() ui.resources_table(_table_rows(session)) - runs = 0 - reloads = 0 - session_timer = ui.Timer().__enter__() try: while True: ui.entrypoint_header() - runs += 1 with ui.Timer() as t: try: run_entrypoint(entrypoint) @@ -223,25 +221,23 @@ async def _session() -> int: return 1 if once: return 0 - ui.dev_hints() reason = await _wait_for_rerun(watcher) if reason == "changed": - reloads += 1 ui.reload_flash() apps, entrypoint = _scan() ui.set_name_width(_all_resource_names(apps)) await session.refresh(apps) finally: - session_timer.__exit__() - ui.session_summary(runs, reloads, session_timer.elapsed) - ui.info("cleaning up dev endpoints ...") + ui.console.print() + ui.info("cleaning up ...") await session.stop() try: sys.exit(asyncio.run(_session())) except (KeyboardInterrupt, EOFError): ui.console.print() - ui.info("session ended.") + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc # ------------------------------------------------------------------ logs diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 9b25b60c4..f66e0af4c 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -341,3 +341,29 @@ def q(): session = DevSession([app], api=api, events=object()) await session.start() await session.stop() + + +class TestGpuIdsPayload: + def test_no_gpu_selection_expands_to_all_pools(self): + # the api rejects "any"; an unconstrained gpu queue asks for + # every pool id instead + app = App("gpu-any") + + @app.queue(name="q") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + gpu_ids = payload["gpuIds"].split(",") + assert "any" not in gpu_ids + assert "ADA_24" in gpu_ids and "AMPERE_80" in gpu_ids + + def test_explicit_pools_pass_through(self): + app = App("gpu-explicit") + + @app.queue(name="q", gpu="ADA_24") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["gpuIds"] == "ADA_24" From b70f898688850f74e4b0ec2b6cd74391e99af2e5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:25:21 -0700 Subject: [PATCH 035/120] fix: async-safe awaitable driving, console links, quieter dev ui --- runpod/apps/api.py | 2 +- runpod/apps/build.py | 35 ++++++++++++++++++++++++--- runpod/apps/deploy.py | 12 +++++++++- runpod/rp_cli/console.py | 39 ++++++++++++++++-------------- runpod/rp_cli/main.py | 44 +++++++++++++++++++++------------- runpod/runtimes/task/runner.py | 25 ++++++++++++++++--- tests/test_apps/test_build.py | 19 +++++++++++++++ tests/test_apps/test_deploy.py | 18 +++++++------- 8 files changed, 142 insertions(+), 52 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index f5c59df94..9436b3065 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -229,7 +229,7 @@ async def upload_tarball( timeout = aiohttp.ClientTimeout(total=600) total = os.path.getsize(tar_path) - def _reader(): + async def _reader(): sent = 0 with open(tar_path, "rb") as f: while True: diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 198bebf18..b5604e1ff 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -19,6 +19,7 @@ import logging import os import re +import shutil import subprocess import sys import tempfile @@ -143,9 +144,14 @@ def runtime_requirement(scratch_dir: Path) -> str: """the runpod runtime spec to vendor into the artifact. RUNPOD_PACKAGE_SPEC overrides the published package (version pins, - git refs, tarball urls, local checkouts). non-index specs are built - into a wheel first so the platform-targeted vendoring install can - stay binary-only. + git refs, tarball urls, local checkouts). non-index specs are + built into a wheel first so the platform-targeted vendoring + install can stay binary-only. + + without an override the published package is vendored for its + dependency closure, then sync_running_package overlays the + client's own package tree so the worker always runs exactly the + client's code regardless of what pypi has. """ spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") if _is_pinnable_name(spec): @@ -153,6 +159,27 @@ def runtime_requirement(scratch_dir: Path) -> str: return str(_build_wheel(spec, scratch_dir)) +def sync_running_package(env_dir: Path) -> None: + """overlay the running runpod package onto the vendored env. + + the package is pure python (py3-none-any), so the client's + installed tree is valid on the worker platform. this pins the + vendored runtime to the exact client version: wire protocols, + manifest handling, and worker code can never drift, and prerelease + clients work before their version reaches pypi. + """ + import runpod as _runpod + + src = Path(_runpod.__file__).resolve().parent + dest = env_dir / "runpod" + shutil.copytree( + src, + dest, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + dirs_exist_ok=True, + ) + + def _build_wheel(spec: str, scratch_dir: Path) -> Path: """build a wheel from a url/path spec for binary-only vendoring.""" wheel_dir = scratch_dir / "wheels" @@ -288,6 +315,8 @@ def build_environment( ", ".join(excluded) or "none", ) vendor(env_dir, all_requirements, python_version) + if not os.environ.get("RUNPOD_PACKAGE_SPEC"): + sync_running_package(env_dir) _verify_runtime(env_dir) return BuildResult( diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index bb48c7c44..a099e22c7 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -37,7 +37,7 @@ BuildResult, build_environment, ) -from .errors import ScheduleNotSupported +from .errors import InvalidResourceError, ScheduleNotSupported from .schedule import SCHEDULES_ENABLED from .spec import ResourceKind @@ -102,6 +102,16 @@ def build_manifest( spec = handle.spec if spec.schedule and not SCHEDULES_ENABLED: raise ScheduleNotSupported() + if ( + spec.kind in (ResourceKind.QUEUE, ResourceKind.API) + and len(spec.name) < 3 + ): + # deployed queue/api resources become endpoints named + # exactly the resource name; the api floor is 3 chars + raise InvalidResourceError( + f"resource name '{spec.name}' is too short to deploy; " + f"queue and api names must be at least 3 characters" + ) entry = spec.to_manifest() fn = ( getattr(handle, "_fn", None) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 6959310df..5d7841e5e 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -8,7 +8,6 @@ """ import time -from datetime import datetime from typing import Dict, Iterable, Optional, Tuple from rich.console import Console @@ -50,6 +49,14 @@ console = Console(highlight=False, theme=theme) +CONSOLE_URL = "https://console.runpod.io/serverless/user/endpoint" + + +def endpoint_link(endpoint_id: str) -> str: + """clickable console url for an endpoint.""" + url = f"{CONSOLE_URL}/{endpoint_id}?tab=overview" + return f"[link={url}][accent.light]{endpoint_id}[/accent.light][/link]" + _name_width: int = 0 @@ -62,10 +69,6 @@ def _padded(name: str) -> str: return f"{name:<{_name_width}}" if _name_width else name -def _ts() -> str: - return f"[dim]{datetime.now().strftime('%H:%M:%S')}[/dim]" - - def _pipe(name: str) -> str: return f"[accent.light]{_padded(name)}[/accent.light] [dim]│[/dim]" @@ -79,19 +82,19 @@ def kind_badge(kind: str) -> str: def info(message: str) -> None: - console.print(f"{_ts()} {message}") + console.print(f"{message}") def success(message: str) -> None: - console.print(f"{_ts()} [ok]✓[/ok] {message}") + console.print(f"[ok]✓[/ok] {message}") def error(message: str) -> None: - console.print(f"{_ts()} [err]✗[/err] {message}") + console.print(f"[err]✗[/err] {message}") def warn(message: str) -> None: - console.print(f"{_ts()} [warn]![/warn] {message}") + console.print(f"[warn]![/warn] {message}") def rule(title: str = "") -> None: @@ -103,20 +106,20 @@ def rule(title: str = "") -> None: def request_started(name: str, label: str = "") -> None: console.print( - f"{_ts()} [bold white]CALL[/bold white] [accent.light]/{name}[/accent.light]" + f"[bold white]CALL[/bold white] [accent.light]/{name}[/accent.light]" f" [dim]{label}[/dim]" ) def request_completed(name: str, elapsed_s: float) -> None: console.print( - f"{_ts()} [ok]✓[/ok] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + f"[ok]✓[/ok] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" ) def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> None: console.print( - f"{_ts()} [err]✗[/err] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" + f"[err]✗[/err] {_padded(name)} [dim]{elapsed_s:.1f}s[/dim]" ) if err: console.print(f" [err dim]{err}[/err dim]") @@ -148,8 +151,8 @@ def resources_table(rows: Iterable[tuple]) -> None: w_hw = max(len(r[2]) for r in rows) for name, kind, hardware, endpoint_id in rows: endpoint = ( - f"[accent.light]{endpoint_id}[/accent.light]" - if endpoint_id + endpoint_link(endpoint_id) + if endpoint_id and endpoint_id != "per-call" else "[dim]per-call[/dim]" ) console.print( @@ -218,7 +221,7 @@ def upload_progress(self, sent: int, total: int) -> None: def endpoint_ready(self, name: str, endpoint_id: str) -> None: self._progress.console.print( - f" {_pipe(name)} [ok]ready[/ok] [dim]{endpoint_id}[/dim]" + f" {_pipe(name)} [ok]ready[/ok] {endpoint_link(endpoint_id)}" ) def close(self) -> None: @@ -284,17 +287,17 @@ def ready(self, name: str, endpoint_id: str) -> None: ) else: console.print( - f"{_ts()} {_pipe(name)} [ok]ready[/ok] [dim]{endpoint_id}[/dim]" + f"{_pipe(name)} [ok]ready[/ok] {endpoint_link(endpoint_id)}" ) def refreshed(self, name: str, generation: int) -> None: console.print( - f"{_ts()} {_pipe(name)} [accent]refreshed[/accent] " + f"{_pipe(name)} [accent]refreshed[/accent] " f"[dim]generation {generation}[/dim]" ) def deleted(self, name: str) -> None: - console.print(f"{_ts()} {_pipe(name)} [dim]deleted[/dim]") + console.print(f"{_pipe(name)} [dim]deleted[/dim]") class Timer: diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 77a038a67..46428a5db 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -174,28 +174,38 @@ def _table_rows(session: "DevSession") -> list: rows.append((spec.name, spec.kind.value, hardware, endpoint_id)) return rows + # daemon stdin reader: a blocked readline must never keep the + # process alive after ctrl-c (executor threads are joined at + # loop shutdown; a daemon thread is not) + import queue as _queue + import threading + + stdin_lines: "_queue.Queue[str]" = _queue.Queue() + + def _stdin_reader() -> None: + for line in sys.stdin: + stdin_lines.put(line) + stdin_lines.put("") # eof marker + + threading.Thread(target=_stdin_reader, daemon=True).start() + async def _wait_for_rerun(watcher: "FileWatcher") -> str: """race the enter key against a file change.""" - loop = asyncio.get_event_loop() - stdin_task = loop.run_in_executor(None, sys.stdin.readline) - try: - while True: - if watcher.changed(): - return "changed" - done, _ = await asyncio.wait({stdin_task}, timeout=0.5) - if done: - # eof (piped stdin, ci): interactive re-run is - # impossible, wait on file changes only - if not stdin_task.result(): - return await _wait_for_change(watcher) - return "enter" - finally: - stdin_task.cancel() - - async def _wait_for_change(watcher: "FileWatcher") -> str: + eof = False while True: if watcher.changed(): return "changed" + if not eof: + try: + line = stdin_lines.get_nowait() + if line == "": + # piped stdin / ci: no interactive re-runs, + # keep watching files only + eof = True + else: + return "enter" + except _queue.Empty: + pass await asyncio.sleep(0.5) async def _session() -> int: diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index fa295420e..1efae73a6 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -163,9 +163,7 @@ def execute_request(request): with redirect_stdout(stdout_io): result = fn(*args, **kwargs) if hasattr(result, "__await__"): - import asyncio - - result = asyncio.run(_await(result)) + result = _run_awaitable(result) response = {"success": True, "stdout": stdout_io.getvalue()} response.update( @@ -182,6 +180,27 @@ def execute_request(request): } +def _run_awaitable(awaitable): + """drive an awaitable to completion from sync code. + + the task runner runs this on plain threads, but the queue worker's + live mode calls execute_request from inside the serverless event + loop, where asyncio.run would raise; a private loop on a helper + thread covers both. + """ + import asyncio + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(_await(awaitable)) + + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, _await(awaitable)).result() + + async def _await(awaitable): return await awaitable diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index 60a3c23c2..13abd7fda 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -110,6 +110,25 @@ def test_default_is_published_package(self, tmp_path, monkeypatch): monkeypatch.delenv("RUNPOD_PACKAGE_SPEC", raising=False) assert runtime_requirement(tmp_path) == "runpod" + def test_running_package_overlays_env(self, tmp_path): + from runpod.apps.build import sync_running_package + + env_dir = tmp_path / "env" + # stale copy from the pypi install: must be overwritten + (env_dir / "runpod").mkdir(parents=True) + (env_dir / "runpod" / "stale.py").write_text("old = True") + + sync_running_package(env_dir) + + # the client's own tree (which has the runtimes) wins + assert (env_dir / "runpod" / "runtimes" / "bootstrap.py").is_file() + assert (env_dir / "runpod" / "apps" / "build.py").is_file() + assert not any( + p.name == "__pycache__" + for p in (env_dir / "runpod").rglob("*") + if p.is_dir() + ) + def test_version_pin_passes_through(self, tmp_path, monkeypatch): monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==1.8.0") assert runtime_requirement(tmp_path) == "runpod==1.8.0" diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index 240274f2c..f8223061c 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -38,8 +38,8 @@ def _write_project(tmp_path: Path) -> Path: app = App("demo-app") - @app.queue(name="q1", cpu="cpu5c-2-4") - def q1(x: int): + @app.queue(name="que1", cpu="cpu5c-2-4") + def que1(x: int): return x * 2 if __name__ == "__main__": @@ -56,7 +56,7 @@ def test_discovers_app(self, tmp_path): apps = discover_apps(tmp_path) assert len(apps) == 1 assert apps[0].name == "demo-app" - assert "q1" in apps[0].resources + assert "que1" in apps[0].resources def test_single_file_target(self, tmp_path): _write_project(tmp_path) @@ -92,8 +92,8 @@ class TestManifest: def test_manifest_shape(self, tmp_path): app = App("m-app") - @app.queue(name="q", cpu="cpu5c-2-4", dependencies=["numpy"]) - def q(x): + @app.queue(name="qee", cpu="cpu5c-2-4", dependencies=["numpy"]) + def qee(x): return x manifest = build_manifest(app, tmp_path) @@ -101,7 +101,7 @@ def q(x): assert manifest["version"] == 1 (resource,) = manifest["resources"] assert resource["kind"] == "queue" - assert resource["name"] == "q" + assert resource["name"] == "qee" assert resource["dependencies"] == ["numpy"] assert resource["qualname"] @@ -195,7 +195,7 @@ async def test_deploy_app_full_flow(self, tmp_path): assert isinstance(result, DeployResult) assert result.build_id == "build-1" - assert result.resources == ["q1"] + assert result.resources == ["que1"] api.upload_tarball.assert_awaited_once() api.deploy_build.assert_awaited_once_with("env-1", "build-1") @@ -242,7 +242,7 @@ def test_no_apps_and_failures_raises_with_causes(self, tmp_path): def test_import_time_invocation_diagnosed(self, tmp_path): _write_project(tmp_path) (tmp_path / "client.py").write_text( - "from main import q1\nq1.remote(1)\n" + "from main import que1\nque1.remote(1)\n" ) # directory walk: the client file fails with the precise # diagnosis but the app still discovers @@ -252,6 +252,6 @@ def test_import_time_invocation_diagnosed(self, tmp_path): def test_import_time_invocation_message(self, tmp_path): _write_project(tmp_path) client = tmp_path / "client.py" - client.write_text("from main import q1\nq1.remote(1)\n") + client.write_text("from main import que1\nque1.remote(1)\n") with pytest.raises(DiscoveryError, match="invoked at import time"): discover_apps(client) From 51d7ca50a7dafd21a3cfc506524e5d951e21719f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:38:57 -0700 Subject: [PATCH 036/120] feat: full console urls in deploy and dev summaries --- runpod/rp_cli/console.py | 39 +++++++++++++++++++++++---------------- runpod/rp_cli/main.py | 11 +++++++++-- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 5d7841e5e..78096194e 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -44,6 +44,7 @@ "err": ERR, "warn": WARN, "dim": DIM, + "progress.elapsed": DIM, } ) @@ -52,10 +53,11 @@ CONSOLE_URL = "https://console.runpod.io/serverless/user/endpoint" -def endpoint_link(endpoint_id: str) -> str: - """clickable console url for an endpoint.""" - url = f"{CONSOLE_URL}/{endpoint_id}?tab=overview" - return f"[link={url}][accent.light]{endpoint_id}[/accent.light][/link]" +def endpoint_url(endpoint_id: str) -> str: + return f"{CONSOLE_URL}/{endpoint_id}?tab=overview" + + + _name_width: int = 0 @@ -149,18 +151,24 @@ def resources_table(rows: Iterable[tuple]) -> None: return w_name = max(len(r[0]) for r in rows) w_hw = max(len(r[2]) for r in rows) + console.print() for name, kind, hardware, endpoint_id in rows: - endpoint = ( - endpoint_link(endpoint_id) - if endpoint_id and endpoint_id != "per-call" - else "[dim]per-call[/dim]" + tail = ( + "[dim]per-call[/dim]" + if not endpoint_id or endpoint_id == "per-call" + else "" ) console.print( f" [white]{name:<{w_name}}[/white]" f" {kind_badge(kind)}" f" [dim]{hardware:<{w_hw}}[/dim]" - f" {endpoint}" + f" {tail}" ) + if endpoint_id and endpoint_id != "per-call": + console.print( + f" [accent.light]{endpoint_url(endpoint_id)}[/accent.light]" + ) + console.print() def reload_flash() -> None: @@ -220,9 +228,10 @@ def upload_progress(self, sent: int, total: int) -> None: ) def endpoint_ready(self, name: str, endpoint_id: str) -> None: - self._progress.console.print( - f" {_pipe(name)} [ok]ready[/ok] {endpoint_link(endpoint_id)}" - ) + # collected, not printed: endpoint urls render in the final + # summary after the progress display stops + self.endpoints: Dict[str, str] = getattr(self, "endpoints", {}) + self.endpoints[name] = endpoint_id def close(self) -> None: if self._current is not None: @@ -281,14 +290,12 @@ def ready(self, name: str, endpoint_id: str) -> None: if self._progress is not None and name in self._tasks: self._progress.update( self._tasks[name], - detail=f"ready · {endpoint_id}", + detail="ready", total=1, completed=1, ) else: - console.print( - f"{_pipe(name)} [ok]ready[/ok] {endpoint_link(endpoint_id)}" - ) + console.print(f"{_pipe(name)} [ok]ready[/ok]") def refreshed(self, name: str, generation: int) -> None: console.print( diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 46428a5db..6255c5be7 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -96,10 +96,17 @@ def deploy(target, env, python_version, exclude): finally: events.close() ui.success( - f"build [accent.light]{result.build_id}[/accent.light] live on " f"[bold white]{result.app_name}/{env or found.env}[/bold white] " - f"[dim]{t.elapsed:.1f}s[/dim]" + f"is live [dim]{t.elapsed:.1f}s[/dim]" ) + if result.endpoints: + ui.console.print() + w = max(len(n) for n in result.endpoints) + for name, endpoint_id in sorted(result.endpoints.items()): + ui.console.print( + f" [white]{name:<{w}}[/white] " + f"[accent.light]{ui.endpoint_url(endpoint_id)}[/accent.light]" + ) ui.console.print() From dc06b01229bccaac818a0518a27fcd9cb2810662 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:46:01 -0700 Subject: [PATCH 037/120] feat: live vendor package feed, upload bar, and upload retries in deploy --- runpod/apps/api.py | 18 ++++++++- runpod/apps/build.py | 39 ++++++++++++++++--- runpod/apps/deploy.py | 6 ++- runpod/rp_cli/console.py | 71 ++++++++++++++++++++++++++++------- tests/test_apps/test_build.py | 71 +++++++++++++++++++++++++---------- 5 files changed, 163 insertions(+), 42 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 9436b3065..7a04ba4dd 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -222,8 +222,11 @@ async def upload_tarball( """put the build tarball to the presigned url. progress, when given, is called with (bytes_sent, total_bytes) - as chunks go out. + as chunks go out. transient resets (broken pipe, connection + reset, 5xx) are retried with backoff; the payload is re-read + from disk on each attempt. """ + import asyncio import os timeout = aiohttp.ClientTimeout(total=600) @@ -241,10 +244,21 @@ async def _reader(): progress(sent, total) yield chunk + attempts = 4 + for attempt in range(attempts): + try: + await self._put_tarball(upload_url, _reader(), total, timeout) + return + except (aiohttp.ClientError, OSError) as exc: + if attempt == attempts - 1: + raise + await asyncio.sleep(2**attempt) + + async def _put_tarball(self, upload_url, reader, total, timeout) -> None: async with aiohttp.ClientSession(timeout=timeout) as session: async with session.put( upload_url, - data=_reader(), + data=reader, headers={ "Content-Type": "application/gzip", "Content-Length": str(total), diff --git a/runpod/apps/build.py b/runpod/apps/build.py index b5604e1ff..7e7b4c068 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -224,8 +224,13 @@ def vendor( python_version: str, *, no_deps: bool = False, + progress: Optional[callable] = None, ) -> None: - """resolve requirements into env_dir for the worker platform.""" + """resolve requirements into env_dir for the worker platform. + + progress, if given, is called as progress(count, package) each time + pip starts resolving another distribution. + """ if not requirements: return cmd = [ @@ -233,8 +238,8 @@ def vendor( "-m", "pip", "install", - "-q", "--upgrade", + "--no-color", "--target", str(env_dir), "--python-version", @@ -250,10 +255,26 @@ def vendor( cmd.append("--no-deps") cmd.extend(requirements) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + count = 0 + for line in proc.stdout: + line = line.strip() + # pip emits one "Collecting " per resolved distribution + if line.startswith("Collecting "): + count += 1 + if progress is not None: + name = re.split(r"[><=!~\[ (]", line[11:], maxsplit=1)[0] + progress(count, name) + stderr = proc.stderr.read() + proc.wait() + if proc.returncode != 0: raise BuildError( - f"dependency vendoring failed: {result.stderr[-3000:]}\n" + f"dependency vendoring failed: {stderr[-3000:]}\n" f"packages without linux wheels cannot be vendored; " f"exclude them (they must then come from the image)" ) @@ -276,6 +297,7 @@ def build_environment( python_version: str = DEFAULT_PYTHON_VERSION, exclude: Optional[List[str]] = None, scratch_dir: Optional[Path] = None, + events: Optional[object] = None, ) -> BuildResult: """vendor the app's full runtime environment into a directory tree.""" if python_version not in SUPPORTED_PYTHON_VERSIONS: @@ -314,7 +336,12 @@ def build_environment( len(excluded), ", ".join(excluded) or "none", ) - vendor(env_dir, all_requirements, python_version) + vendor( + env_dir, + all_requirements, + python_version, + progress=getattr(events, "vendor_progress", None), + ) if not os.environ.get("RUNPOD_PACKAGE_SPEC"): sync_running_package(env_dir) _verify_runtime(env_dir) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index a099e22c7..8c5648cc1 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -405,7 +405,11 @@ async def deploy_app( _phase(events, "vendor", f"python {python_version}") build: BuildResult = build_environment( - app, project_root, python_version=python_version, exclude=exclude + app, + project_root, + python_version=python_version, + exclude=exclude, + events=events, ) manifest = build_manifest( app, diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 78096194e..507666e7a 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -16,7 +16,6 @@ SpinnerColumn, TaskID, TextColumn, - TimeElapsedColumn, ) from rich.text import Text from rich.theme import Theme @@ -185,46 +184,94 @@ def reload_flash() -> None: } +_BAR_WIDTH = 22 + + +def _bar(fraction: float) -> str: + """slim one-line progress bar.""" + fraction = min(max(fraction, 0.0), 1.0) + filled = int(fraction * _BAR_WIDTH) + return ( + f"[accent]{'━' * filled}[/accent]" + f"[grey30]{'━' * (_BAR_WIDTH - filled)}[/grey30]" + ) + + class DeployEvents: """deploy_app event sink: animated phase list with live progress. each phase renders a spinner while active and collapses to a ✓ with - elapsed time when the next phase begins. upload progress (bytes) - animates in-line when the transport reports it. + a short summary when the next begins. vendoring streams package + names as pip resolves them; upload renders a slim in-line bar. """ def __init__(self) -> None: self._progress = Progress( SpinnerColumn("dots", style=ACCENT, finished_text=f"[ok]✓[/ok]"), TextColumn("[white]{task.description}[/white]"), - TextColumn("[dim]{task.fields[detail]}[/dim]"), - TimeElapsedColumn(), + TextColumn("{task.fields[detail]}"), console=console, transient=False, ) self._current: Optional[TaskID] = None + self._current_name = "" self._started = False + self._package_count = 0 + self._upload_total = 0 def _ensure_started(self) -> None: if not self._started: self._progress.start() self._started = True + def _finish_current(self) -> None: + if self._current is None: + return + # collapse the live detail into a short summary + if self._current_name == "vendor" and self._package_count: + summary = f"{self._package_count} packages" + elif self._current_name == "upload" and self._upload_total: + summary = f"{self._upload_total / 1048576:.1f} MB" + else: + summary = "" + self._progress.update( + self._current, + detail=f"[dim]{summary}[/dim]", + total=1, + completed=1, + ) + self._current = None + def phase(self, name: str, detail: str = "") -> None: self._ensure_started() - if self._current is not None: - self._progress.update(self._current, total=1, completed=1) + self._finish_current() + self._current_name = name self._current = self._progress.add_task( - _PHASE_LABELS.get(name, name), detail=detail, total=None + _PHASE_LABELS.get(name, name), + detail=f"[dim]{detail}[/dim]" if detail else "", + total=None, + ) + + def vendor_progress(self, count: int, package: str) -> None: + if self._current is None: + return + self._package_count = count + self._progress.update( + self._current, + detail=f"[accent.light]{package}[/accent.light] [dim]· {count}[/dim]", ) def upload_progress(self, sent: int, total: int) -> None: if self._current is None: return - mb = 1024 * 1024 + self._upload_total = total + mb = 1048576 self._progress.update( self._current, - detail=f"{sent / mb:.1f} / {total / mb:.1f} MB", + detail=( + f"{_bar(sent / total)} " + f"[dim]{sent / mb:.1f} / {total / mb:.1f} MB[/dim]" + ), ) def endpoint_ready(self, name: str, endpoint_id: str) -> None: @@ -234,9 +281,7 @@ def endpoint_ready(self, name: str, endpoint_id: str) -> None: self.endpoints[name] = endpoint_id def close(self) -> None: - if self._current is not None: - self._progress.update(self._current, total=1, completed=1) - self._current = None + self._finish_current() if self._started: self._progress.stop() self._started = False diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index 13abd7fda..30af8ece3 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -153,20 +153,33 @@ class R: assert runtime_requirement(tmp_path) == str(wheel) -class TestVendor: - def test_targets_worker_platform(self, tmp_path): - captured = {} +def _fake_popen(captured, *, returncode=0, stdout="", stderr=""): + import io - def fake_run(cmd, **kwargs): - captured["cmd"] = cmd + def fake(cmd, **kwargs): + captured["cmd"] = cmd - class R: - returncode = 0 - stderr = "" + class P: + def __init__(self): + self.stdout = io.StringIO(stdout) + self.stderr = io.StringIO(stderr) + self.returncode = returncode - return R() + def wait(self): + return returncode - with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): + return P() + + return fake + + +class TestVendor: + def test_targets_worker_platform(self, tmp_path): + captured = {} + with patch( + "runpod.apps.build.subprocess.Popen", + side_effect=_fake_popen(captured), + ): vendor(tmp_path, ["numpy"], "3.12") cmd = captured["cmd"] @@ -176,18 +189,36 @@ class R: assert "manylinux_2_28_x86_64" in cmd def test_empty_requirements_no_op(self, tmp_path): - with patch("runpod.apps.build.subprocess.run") as run: + with patch("runpod.apps.build.subprocess.Popen") as popen: vendor(tmp_path, [], "3.12") - run.assert_not_called() + popen.assert_not_called() def test_failure_raises_build_error(self, tmp_path): - def fake_run(cmd, **kwargs): - class R: - returncode = 1 - stderr = "no matching distribution" - - return R() - - with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): + with patch( + "runpod.apps.build.subprocess.Popen", + side_effect=_fake_popen( + {}, returncode=1, stderr="no matching distribution" + ), + ): with pytest.raises(BuildError, match="no matching distribution"): vendor(tmp_path, ["nonexistent-xyz"], "3.12") + + def test_progress_reports_collected_packages(self, tmp_path): + seen = [] + stdout = ( + "Collecting numpy>=1.0\n" + " Downloading numpy-2.0-cp312.whl\n" + "Collecting requests\n" + "Installing collected packages\n" + ) + with patch( + "runpod.apps.build.subprocess.Popen", + side_effect=_fake_popen({}, stdout=stdout), + ): + vendor( + tmp_path, + ["numpy"], + "3.12", + progress=lambda c, n: seen.append((c, n)), + ) + assert seen == [(1, "numpy"), (2, "requests")] From 1191bdfd5019e1d8416599b8e97c5c34c1c0c6c1 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:40:57 -0700 Subject: [PATCH 038/120] feat: ship full module source in dev, worker monitor feed, sleeker dev ui --- runpod/apps/api.py | 1 + runpod/apps/dev.py | 14 ++- runpod/apps/monitor.py | 147 +++++++++++++++++++++++++++++++ runpod/apps/serialization.py | 34 +++---- runpod/apps/targets.py | 114 ++++++++++++++++++++---- runpod/rp_cli/console.py | 83 ++++++++++++----- runpod/rp_cli/main.py | 2 +- runpod/runtimes/task/runner.py | 7 +- tests/test_apps/test_dispatch.py | 65 ++++++++++++++ tests/test_apps/test_monitor.py | 105 ++++++++++++++++++++++ 10 files changed, 509 insertions(+), 63 deletions(-) create mode 100644 runpod/apps/monitor.py create mode 100644 tests/test_apps/test_monitor.py diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 7a04ba4dd..84d339bb2 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -43,6 +43,7 @@ async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: workersMin workersMax idleTimeout + aiKey } } """ diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 5fd9fdd7d..ff807967e 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -221,7 +221,12 @@ async def start(self) -> None: self._emit("ready", spec.name, endpoint_id) self._endpoints[name] = endpoint_id - app._dev_targets[spec.name] = LiveTarget(endpoint_id) + app._dev_targets[spec.name] = LiveTarget( + endpoint_id, + spec.name, + events=self.events, + metrics_key=result.get("aiKey"), + ) self._emit("session_started") @@ -262,7 +267,12 @@ async def refresh(self, apps: List[App]) -> None: result = await self.api.save_endpoint(payload) endpoint_id = result["id"] self._endpoints[name] = endpoint_id - app._dev_targets[handle.spec.name] = LiveTarget(endpoint_id) + app._dev_targets[handle.spec.name] = LiveTarget( + endpoint_id, + handle.spec.name, + events=self.events, + metrics_key=result.get("aiKey"), + ) self._emit("refreshed", handle.spec.name, self.generation) log.info( "refreshed dev endpoint %s (%s, generation %d)", diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py new file mode 100644 index 000000000..db8baaf9d --- /dev/null +++ b/runpod/apps/monitor.py @@ -0,0 +1,147 @@ +"""request-time worker observability for dev sessions. + +while a dev call is in flight, a WorkerMonitor watches the endpoint's +worker metrics for state transitions (initializing, throttled, ready) +and, once the job is assigned a worker, follows that worker's container +logs over the hapi sse stream. everything surfaces through a duck-typed +event sink; missing handlers are silently skipped. +""" + +import asyncio +import logging +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +log = logging.getLogger(__name__) + +METRICS_POLL_INTERVAL = 2.0 + +# worker states worth reporting, in display order +_TRACKED_STATES = ("initializing", "ready", "running", "throttled", "unhealthy") + +# leading iso timestamp on hapi log lines +_TS_PREFIX = re.compile( + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?\s*" +) + + +def emit(sink: Optional[object], event: str, *args: Any) -> None: + """invoke a handler on the sink if it exists.""" + handler = getattr(sink, event, None) + if handler is not None: + try: + handler(*args) + except Exception: # noqa: BLE001 - rendering must never break calls + log.debug("event sink %s failed", event, exc_info=True) + + +class WorkerMonitor: + """observes one in-flight request against a live endpoint. + + start() launches a metrics poller; on_status(payload) should be fed + every job-status payload so the assigned worker's log stream can + attach as soon as a workerId appears. stop() tears everything down. + """ + + def __init__( + self, + endpoint_id: str, + resource_name: str, + events: object, + metrics_key: Optional[str] = None, + ): + self.endpoint_id = endpoint_id + self.name = resource_name + self.events = events + # /metrics is served on the data plane behind the endpoint's own + # ai key; the user api key is rejected there + self.metrics_key = metrics_key + self._tasks: List[asyncio.Task] = [] + self._streamed_workers: set = set() + self._last_counts: Optional[Dict[str, int]] = None + self._since = datetime.now(timezone.utc).isoformat() + + async def start(self) -> None: + if self.metrics_key: + self._tasks.append(asyncio.ensure_future(self._poll_metrics())) + + def on_status(self, data: Dict[str, Any]) -> None: + """inspect a job-status payload for the assigned worker.""" + worker_id = data.get("workerId") + if worker_id and worker_id not in self._streamed_workers: + self._streamed_workers.add(worker_id) + emit(self.events, "worker_ready", self.name, str(worker_id)) + self._tasks.append( + asyncio.ensure_future(self._stream_logs(str(worker_id))) + ) + + async def stop(self) -> None: + for task in self._tasks: + task.cancel() + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + self._tasks.clear() + + async def _poll_metrics(self) -> None: + from .targets import _endpoint_url_base, _get_json + + url = f"{_endpoint_url_base()}/{self.endpoint_id}/metrics" + headers = {"Authorization": f"Bearer {self.metrics_key}"} + while True: + try: + data = await _get_json(url, headers, 10.0) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - observability is best-effort + await asyncio.sleep(METRICS_POLL_INTERVAL) + continue + self._report_counts(data) + await asyncio.sleep(METRICS_POLL_INTERVAL) + + def _report_counts(self, data: Dict[str, Any]) -> None: + workers = data.get("workers") + if not isinstance(workers, dict): + return + counts = { + state: workers.get(state, 0) or 0 for state in _TRACKED_STATES + } + if counts == self._last_counts: + return + first = self._last_counts is None + self._last_counts = counts + # the first snapshot is only interesting when workers are still + # coming up (or wedged); a steady ready/running pool is implied + if first and not ( + counts["initializing"] or counts["throttled"] or counts["unhealthy"] + ): + return + emit(self.events, "worker_status", self.name, counts) + + async def _stream_logs(self, worker_id: str) -> None: + from .logs import stream_pod_logs + + try: + async for event in stream_pod_logs( + worker_id, + log_type="container", + tail=0, + since=self._since, + ): + line = _TS_PREFIX.sub("", (event.get("line") or "").rstrip()) + if line: + emit(self.events, "worker_log", self.name, line) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - observability is best-effort + log.debug("log stream for %s ended", worker_id, exc_info=True) + + +def format_worker_counts(counts: Dict[str, int]) -> str: + """human summary like '1 initializing · 2 ready'.""" + parts = [ + f"{count} {state}" + for state, count in counts.items() + if count + ] + return " · ".join(parts) if parts else "no workers" diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py index 216bb1189..5b0db0ad2 100644 --- a/runpod/apps/serialization.py +++ b/runpod/apps/serialization.py @@ -31,27 +31,19 @@ def deserialize_result(result_b64: str) -> Any: def get_function_source(fn: Callable) -> str: - """extract a function's source with decorators stripped. + """the full source of the function's module. - the worker execs this source into a fresh namespace and calls the - function by name, so the emitted code must be a bare def. + the worker execs this and resolves the function by name, exactly + mirroring deployed mode (which imports the user's module from the + build artifact): module-level imports, globals, and decorators + behave identically in dev and deploy. """ fn = inspect.unwrap(fn) - source = textwrap.dedent(inspect.getsource(fn)) - - module = ast.parse(source) - fn_def = None - for node in ast.walk(module): - if ( - isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and node.name == fn.__name__ - ): - fn_def = node - break - if fn_def is None: - raise ValueError(f"could not find function definition for {fn.__name__}") - - # slice from the def line, dropping decorator lines above it - lines = source.split("\n") - fn_lines = lines[fn_def.lineno - 1 :] - return textwrap.dedent("\n".join(fn_lines)) + module = inspect.getmodule(fn) + if module is not None: + try: + return inspect.getsource(module) + except (OSError, TypeError): + pass + # no module file (repl, exec): fall back to the bare function body + return textwrap.dedent(inspect.getsource(fn)) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 1330fd6eb..2078fbc83 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -218,15 +218,19 @@ async def _wait_terminal( data: Dict[str, Any], headers: Dict[str, str], timeout: float, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Dict[str, Any]: """poll /status until the job reaches a terminal state. runsync returns early (e.g. IN_QUEUE) when the job outlives the sync - window, typically on cold starts; polling covers the rest. + window, typically on cold starts; polling covers the rest. on_status, + when given, sees every payload (observability hooks read workerId). """ import asyncio import time + if on_status is not None: + on_status(data) deadline = time.monotonic() + timeout interval = 0.5 while data.get("status") not in FINAL_STATUSES: @@ -241,6 +245,8 @@ async def _wait_terminal( await asyncio.sleep(interval) interval = min(interval * 1.5, 5.0) data = await _get_json(f"{base_url}/status/{job_id}", headers, 30.0) + if on_status is not None: + on_status(data) return data @@ -335,10 +341,21 @@ class LiveTarget(InvocationTarget): provisioning of the live endpoint itself is owned by the dev session (see runpod.apps.dev); this target only speaks to an endpoint id. + when an event sink is attached, each call gets a dispatch line, + live worker status/log feed, and a completion line. """ - def __init__(self, endpoint_id: str): + def __init__( + self, + endpoint_id: str, + resource_name: str = "", + events: Optional[object] = None, + metrics_key: Optional[str] = None, + ): self.endpoint_id = endpoint_id + self.resource_name = resource_name + self.events = events + self.metrics_key = metrics_key def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict @@ -374,14 +391,60 @@ def unwrap(self, data: Dict[str, Any]) -> Any: return response.json_result return output + def _monitor(self): + if self.events is None: + return None + from .monitor import WorkerMonitor + + return WorkerMonitor( + self.endpoint_id, + self.resource_name, + self.events, + metrics_key=self.metrics_key, + ) + async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: + import time + + from .monitor import emit + base = f"{_endpoint_url_base()}/{self.endpoint_id}" headers = _headers() - data = await _post_json(f"{base}/runsync", payload, headers, timeout) - data = await _wait_terminal(base, data, headers, timeout) - return self.unwrap(data) + monitor = self._monitor() + emit(self.events, "dispatch", self.resource_name) + start = time.monotonic() + if monitor is not None: + await monitor.start() + try: + data = await _post_json(f"{base}/runsync", payload, headers, timeout) + data = await _wait_terminal( + base, + data, + headers, + timeout, + on_status=monitor.on_status if monitor else None, + ) + result = self.unwrap(data) + except Exception: + emit( + self.events, + "request_failed", + self.resource_name, + time.monotonic() - start, + ) + raise + finally: + if monitor is not None: + await monitor.stop() + emit( + self.events, + "request_completed", + self.resource_name, + time.monotonic() - start, + ) + return result async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: url = f"{_endpoint_url_base()}/{self.endpoint_id}/run" @@ -405,18 +468,39 @@ async def request( *, timeout: float = DEFAULT_TIMEOUT_SECONDS, ) -> Any: + import time + + from .monitor import emit + + emit(self.events, "dispatch", self.resource_name, f"{method} {path}") + start = time.monotonic() url = f"https://{self.endpoint_id}.{_lb_domain()}{path}" client_timeout = aiohttp.ClientTimeout(total=timeout) - async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.request( - method, url, json=body, headers=_headers() - ) as resp: - resp.raise_for_status() - text = await resp.text() - try: - return json.loads(text) - except json.JSONDecodeError: - return text + try: + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.request( + method, url, json=body, headers=_headers() + ) as resp: + resp.raise_for_status() + text = await resp.text() + except Exception: + emit( + self.events, + "request_failed", + self.resource_name, + time.monotonic() - start, + ) + raise + emit( + self.events, + "request_completed", + self.resource_name, + time.monotonic() - start, + ) + try: + return json.loads(text) + except json.JSONDecodeError: + return text class PodTarget(InvocationTarget): diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 507666e7a..96b47e9fe 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -56,6 +56,14 @@ def endpoint_url(endpoint_id: str) -> str: return f"{CONSOLE_URL}/{endpoint_id}?tab=overview" +def endpoint_link(endpoint_id: str) -> str: + """short clickable endpoint reference (osc8 hyperlink).""" + return ( + f"[link={endpoint_url(endpoint_id)}]" + f"[accent.light]{endpoint_id} ↗[/accent.light][/link]" + ) + + _name_width: int = 0 @@ -150,12 +158,11 @@ def resources_table(rows: Iterable[tuple]) -> None: return w_name = max(len(r[0]) for r in rows) w_hw = max(len(r[2]) for r in rows) - console.print() for name, kind, hardware, endpoint_id in rows: tail = ( "[dim]per-call[/dim]" if not endpoint_id or endpoint_id == "per-call" - else "" + else endpoint_link(endpoint_id) ) console.print( f" [white]{name:<{w_name}}[/white]" @@ -163,15 +170,10 @@ def resources_table(rows: Iterable[tuple]) -> None: f" [dim]{hardware:<{w_hw}}[/dim]" f" {tail}" ) - if endpoint_id and endpoint_id != "per-call": - console.print( - f" [accent.light]{endpoint_url(endpoint_id)}[/accent.light]" - ) - console.print() def reload_flash() -> None: - console.print(f"[warn]⚡ reload[/warn]") + console.print() # -- live deploy phases ------------------------------------------------------ @@ -291,8 +293,12 @@ def close(self) -> None: class DevEvents: - """DevSession event sink: per-resource spinner rows during start, - plain lifecycle lines afterwards. + """DevSession event sink. + + provisioning renders transient spinner rows that vanish once the + session is up (the resource table is the durable record). request + lifecycle renders as a block: an accent dispatch line, a dim + indented worker feed, and a check/cross verdict. """ def __init__(self) -> None: @@ -302,9 +308,10 @@ def __init__(self) -> None: def session_starting(self) -> None: self._progress = Progress( SpinnerColumn("dots", style=ACCENT, finished_text="[ok]✓[/ok]"), - TextColumn("[accent.light]{task.description}[/accent.light]"), + TextColumn("[white]{task.description}[/white]"), TextColumn("[dim]{task.fields[detail]}[/dim]"), console=console, + transient=True, ) self._progress.start() @@ -329,27 +336,54 @@ def provisioning(self, name: str, kind: str, hardware: str) -> None: self._row(name, f"provisioning {kind} · {hardware}") def adopted(self, name: str, endpoint_id: str) -> None: - self._row(name, f"adopted {endpoint_id}") + self._row(name, "adopting") def ready(self, name: str, endpoint_id: str) -> None: if self._progress is not None and name in self._tasks: self._progress.update( - self._tasks[name], - detail="ready", - total=1, - completed=1, + self._tasks[name], detail="ready", total=1, completed=1 ) - else: - console.print(f"{_pipe(name)} [ok]ready[/ok]") def refreshed(self, name: str, generation: int) -> None: console.print( - f"{_pipe(name)} [accent]refreshed[/accent] " - f"[dim]generation {generation}[/dim]" + f"[warn]⚡[/warn] {_padded(name)} [dim]reloaded[/dim]" ) def deleted(self, name: str) -> None: - console.print(f"{_pipe(name)} [dim]deleted[/dim]") + console.print(f" [dim]{_padded(name)} deleted[/dim]") + + # -- request lifecycle (LiveTarget events) -- + + def dispatch(self, name: str, label: str = "") -> None: + suffix = f" [dim]{label}[/dim]" if label else "" + console.print( + f"[accent]→[/accent] [bold white]{_padded(name)}[/bold white]{suffix}" + ) + + def worker_status(self, name: str, counts: Dict[str, int]) -> None: + from runpod.apps.monitor import format_worker_counts + + console.print( + f" [dim]│ {format_worker_counts(counts)}[/dim]" + ) + + def worker_ready(self, name: str, worker_id: str) -> None: + console.print(f" [dim]│ worker {worker_id[:12]}[/dim]") + + def worker_log(self, name: str, line: str) -> None: + from rich.markup import escape + + console.print(f" [dim]│ {escape(line)}[/dim]") + + def request_completed(self, name: str, elapsed_s: float) -> None: + console.print( + f"[ok]✓[/ok] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" + ) + + def request_failed(self, name: str, elapsed_s: float) -> None: + console.print( + f"[err]✗[/err] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" + ) class Timer: @@ -372,10 +406,13 @@ def entrypoint_header() -> None: def entrypoint_success(elapsed: float) -> None: + console.print() console.print( - f"[ok]✓[/ok] [dim]{elapsed:.1f}s · enter to re-run · edit to reload · ^C to quit[/dim]" + f"[dim]── {elapsed:.1f}s · enter re-run · edit reload · ^C quit ──[/dim]" ) def entrypoint_failure(elapsed: float, err: str) -> None: - console.print(f"[err]✗ {err}[/err] [dim]{elapsed:.1f}s[/dim]") + console.print() + console.print(f"[err]✗ {err}[/err]") + console.print(f"[dim]── {elapsed:.1f}s · enter re-run · edit reload · ^C quit ──[/dim]") diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 6255c5be7..ea40996db 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -105,7 +105,7 @@ def deploy(target, env, python_version, exclude): for name, endpoint_id in sorted(result.endpoints.items()): ui.console.print( f" [white]{name:<{w}}[/white] " - f"[accent.light]{ui.endpoint_url(endpoint_id)}[/accent.light]" + f"{ui.endpoint_link(endpoint_id)}" ) ui.console.print() diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index 1efae73a6..f2b9aede3 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -147,7 +147,11 @@ def execute_request(request): "error": "function_name and function_code are required", } - namespace = {} + # exec mirrors deployed-mode module import: the code is the + # user's full module, so __name__ is set like an import would + # (main guards stay inert) and decorated handles resolve to + # their wrapped functions + namespace = {"__name__": "__runpod_live__"} exec(function_code, namespace) # noqa: S102 - that is the job if function_name not in namespace: return { @@ -155,6 +159,7 @@ def execute_request(request): "error": f"function '{function_name}' not found in provided code", } fn = namespace[function_name] + fn = getattr(fn, "_fn", fn) args, kwargs = _deserialize_args(request) diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index 889c8128c..dc52ca4e5 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -239,3 +239,68 @@ async def caller(): result = asyncio.run(caller()) assert result == "bridged" + + +class TestModuleSourceShipping: + def test_whole_module_ships(self, tmp_path): + import importlib.util + import sys + + mod_file = tmp_path / "shipmod.py" + mod_file.write_text( + "import asyncio\n" + "from pathlib import Path\n" + "\n" + "GREETING = 'hello'\n" + "\n" + "async def waiter():\n" + " await asyncio.sleep(0)\n" + " return f'{GREETING} {Path(\"x\")}'\n" + "\n" + "if __name__ == '__main__':\n" + " raise SystemExit('main guard must not run')\n" + ) + spec = importlib.util.spec_from_file_location("shipmod", mod_file) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + sys.modules["shipmod"] = mod + try: + from runpod.apps.serialization import get_function_source + + source = get_function_source(mod.waiter) + # the entire module ships: imports and globals included + assert "import asyncio" in source + assert "GREETING = 'hello'" in source + + # executes like deployed-mode module import: main guard inert + namespace = {"__name__": "__runpod_live__"} + exec(source, namespace) + import asyncio as aio + + assert aio.run(namespace["waiter"]()) == "hello x" + finally: + sys.modules.pop("shipmod", None) + + def test_execute_request_unwraps_handles(self): + # a shipped module defines decorated handles; the runner must + # call the wrapped function, not the handle + from runpod.runtimes.task.runner import execute_request + + code = ( + "import runpod\n" + "app = runpod.App('t')\n" + "@app.queue()\n" + "def double(x):\n" + " return x * 2\n" + ) + response = execute_request( + { + "function_name": "double", + "function_code": code, + "args": [], + "kwargs": {"x": "AAA"}, + "serialization_format": "json", + } + ) + assert response["success"], response.get("error") + assert response["json_result"] == "AAAAAA" diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py new file mode 100644 index 000000000..d05445055 --- /dev/null +++ b/tests/test_apps/test_monitor.py @@ -0,0 +1,105 @@ +"""worker monitor: metrics transitions, log attach, event emission.""" + +import asyncio +from unittest.mock import patch + +import pytest + +from runpod.apps.monitor import WorkerMonitor, emit, format_worker_counts + + +class Sink: + def __init__(self): + self.events = [] + + def worker_status(self, name, counts): + self.events.append(("worker_status", name, counts)) + + def worker_ready(self, name, worker_id): + self.events.append(("worker_ready", name, worker_id)) + + def worker_log(self, name, line): + self.events.append(("worker_log", name, line)) + + +class TestEmit: + def test_missing_handler_skipped(self): + emit(object(), "nope", 1) + + def test_none_sink_skipped(self): + emit(None, "anything") + + def test_handler_called(self): + sink = Sink() + emit(sink, "worker_ready", "calc", "w1") + assert sink.events == [("worker_ready", "calc", "w1")] + + def test_handler_errors_swallowed(self): + class Bad: + def worker_ready(self, *a): + raise RuntimeError("render bug") + + emit(Bad(), "worker_ready", "calc", "w1") + + +class TestFormatWorkerCounts: + def test_only_nonzero(self): + s = format_worker_counts({"initializing": 1, "ready": 2, "throttled": 0}) + assert s == "1 initializing · 2 ready" + + def test_empty(self): + assert format_worker_counts({"ready": 0}) == "no workers" + + +class TestOnStatus: + def test_worker_id_triggers_ready_and_stream(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + + async def run(): + with patch.object( + monitor, "_stream_logs", return_value=asyncio.sleep(0) + ): + monitor.on_status({"workerId": "w123", "status": "IN_PROGRESS"}) + monitor.on_status({"workerId": "w123", "status": "IN_PROGRESS"}) + await monitor.stop() + + asyncio.run(run()) + assert sink.events == [("worker_ready", "calc", "w123")] + + def test_no_worker_id_no_events(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + monitor.on_status({"status": "IN_QUEUE"}) + assert sink.events == [] + + +class TestReportCounts: + def test_first_steady_snapshot_suppressed(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + monitor._report_counts({"workers": {"ready": 2, "running": 1}}) + assert sink.events == [] + + def test_first_snapshot_with_initializing_reported(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + monitor._report_counts({"workers": {"initializing": 1}}) + assert len(sink.events) == 1 + assert sink.events[0][2]["initializing"] == 1 + + def test_transition_reported_once(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + monitor._report_counts({"workers": {"initializing": 1}}) + monitor._report_counts({"workers": {"initializing": 1}}) + monitor._report_counts({"workers": {"ready": 1}}) + assert len(sink.events) == 2 + assert sink.events[1][2]["ready"] == 1 + + def test_malformed_payload_ignored(self): + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink) + monitor._report_counts({"workers": None}) + monitor._report_counts({}) + assert sink.events == [] From 142d03b9ce6bf06fa0b1b6205575c2c4b15af646 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:02:14 -0700 Subject: [PATCH 039/120] fix: tee function stdout to container logs and name interleaved dev feeds --- runpod/rp_cli/console.py | 27 ++++++++++++++----- runpod/runtimes/task/runner.py | 33 +++++++++++++++++++++--- tests/test_apps/test_task_runner_http.py | 20 ++++++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 96b47e9fe..0b22732d0 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -8,7 +8,7 @@ """ import time -from typing import Dict, Iterable, Optional, Tuple +from typing import Dict, Iterable, List, Optional, Tuple from rich.console import Console from rich.progress import ( @@ -304,6 +304,9 @@ class DevEvents: def __init__(self) -> None: self._progress: Optional[Progress] = None self._tasks: Dict[str, TaskID] = {} + # in-flight request names; feed lines carry the name whenever + # calls overlap so interleaved output stays attributable + self._inflight: List[str] = [] def session_starting(self) -> None: self._progress = Progress( @@ -354,7 +357,15 @@ def deleted(self, name: str) -> None: # -- request lifecycle (LiveTarget events) -- + def _feed(self, name: str, body: str) -> None: + """a dim indented line under a dispatch; named when calls overlap.""" + if len(self._inflight) > 1: + console.print(f" [dim]│ {_padded(name)} {body}[/dim]") + else: + console.print(f" [dim]│ {body}[/dim]") + def dispatch(self, name: str, label: str = "") -> None: + self._inflight.append(name) suffix = f" [dim]{label}[/dim]" if label else "" console.print( f"[accent]→[/accent] [bold white]{_padded(name)}[/bold white]{suffix}" @@ -363,24 +374,28 @@ def dispatch(self, name: str, label: str = "") -> None: def worker_status(self, name: str, counts: Dict[str, int]) -> None: from runpod.apps.monitor import format_worker_counts - console.print( - f" [dim]│ {format_worker_counts(counts)}[/dim]" - ) + self._feed(name, format_worker_counts(counts)) def worker_ready(self, name: str, worker_id: str) -> None: - console.print(f" [dim]│ worker {worker_id[:12]}[/dim]") + self._feed(name, f"worker {worker_id[:12]}") def worker_log(self, name: str, line: str) -> None: from rich.markup import escape - console.print(f" [dim]│ {escape(line)}[/dim]") + self._feed(name, escape(line)) + + def _done(self, name: str) -> None: + if name in self._inflight: + self._inflight.remove(name) def request_completed(self, name: str, elapsed_s: float) -> None: + self._done(name) console.print( f"[ok]✓[/ok] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" ) def request_failed(self, name: str, elapsed_s: float) -> None: + self._done(name) console.print( f"[err]✗[/err] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" ) diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index f2b9aede3..fc42473a9 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -163,9 +163,12 @@ def execute_request(request): args, kwargs = _deserialize_args(request) - # stdout only: sys.stderr carries the runner's own request logs - # from other threads, which must not leak into job output - with redirect_stdout(stdout_io): + # stdout is teed: captured for the job response and written + # through to the real stdout so container logs (and live log + # streaming in dev sessions) carry the function's prints. + # sys.stderr carries the runner's own request logs from other + # threads, which must not leak into job output + with redirect_stdout(_Tee(stdout_io, sys.__stdout__)): result = fn(*args, **kwargs) if hasattr(result, "__await__"): result = _run_awaitable(result) @@ -185,6 +188,30 @@ def execute_request(request): } +class _Tee(io.TextIOBase): + """write-through to multiple streams, flushing eagerly so log + followers see lines as they happen.""" + + def __init__(self, *streams): + self._streams = [s for s in streams if s is not None] + + def write(self, s): + for stream in self._streams: + try: + stream.write(s) + stream.flush() + except (ValueError, OSError): + pass + return len(s) + + def flush(self): + for stream in self._streams: + try: + stream.flush() + except (ValueError, OSError): + pass + + def _run_awaitable(awaitable): """drive an awaitable to completion from sync code. diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index 7f42c4fa6..30ec46c6f 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -190,3 +190,23 @@ class R: task_runner._install_system(["sox"]) updates = [c for c in calls if c[:2] == ["apt-get", "update"]] assert len(updates) == 1 + + +class TestStdoutTee: + def test_prints_reach_real_stdout_and_response(self, capfd): + from runpod.runtimes.task.runner import execute_request + + response = execute_request( + { + "function_name": "speak", + "function_code": "def speak():\n print('live line')\n return 1\n", + "args": [], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"] + # captured for the job response + assert "live line" in response["stdout"] + # and written through to the container's stdout for log streams + assert "live line" in capfd.readouterr().out From 6ebd36a89d3f9de30019f75518f48c2aeb02e386 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:22:34 -0700 Subject: [PATCH 040/120] feat: stream worker stdout into rp dev with request-style lifecycle lines --- runpod/apps/monitor.py | 139 ++++++++++++++++++++++++++++---- runpod/apps/targets.py | 10 ++- runpod/rp_cli/console.py | 69 +++++++++------- runpod/rp_cli/main.py | 2 +- tests/test_apps/test_monitor.py | 34 ++++++++ 5 files changed, 208 insertions(+), 46 deletions(-) diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index db8baaf9d..528ae2259 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -8,6 +8,7 @@ """ import asyncio +import json import logging import re from datetime import datetime, timezone @@ -26,6 +27,36 @@ ) +def _worker_frame(line: str) -> Optional[Dict[str, Any]]: + """parse the serverless sdk's structured json log lines.""" + if not line.startswith("{"): + return None + try: + frame = json.loads(line) + except json.JSONDecodeError: + return None + if isinstance(frame, dict) and "message" in frame and "level" in frame: + return frame + return None + + +def _filter_line(line: str) -> Optional[str]: + """what to show from a raw container log line. + + the serverless sdk's own frames (fitness checks, queue counts, + started/finished) are runtime noise; user prints pass through + verbatim, and sdk error/warn frames surface as their message. + """ + if line.startswith("--- Starting Serverless Worker"): + return None + frame = _worker_frame(line) + if frame is None: + return line + if frame.get("level", "").upper() in ("ERROR", "WARN", "WARNING"): + return str(frame.get("message", "")) + return None + + def emit(sink: Optional[object], event: str, *args: Any) -> None: """invoke a handler on the sink if it exists.""" handler = getattr(sink, event, None) @@ -61,6 +92,7 @@ def __init__( self._streamed_workers: set = set() self._last_counts: Optional[Dict[str, int]] = None self._since = datetime.now(timezone.utc).isoformat() + self._lines_emitted = 0 async def start(self) -> None: if self.metrics_key: @@ -82,6 +114,28 @@ async def stop(self) -> None: if self._tasks: await asyncio.gather(*self._tasks, return_exceptions=True) self._tasks.clear() + # the stream often cannot attach within a short job's lifetime + # (fresh pods 403 on the logs endpoint until the gateway can + # resolve them); a final snapshot recovers the output + if self._streamed_workers and not self._lines_emitted: + await self._snapshot_logs() + + async def _snapshot_logs(self) -> None: + from .logs import pod_logs + + for worker_id in self._streamed_workers: + try: + logs = await pod_logs(worker_id, log_type="container") + except Exception: # noqa: BLE001 - observability is best-effort + log.debug("log snapshot for %s failed", worker_id, exc_info=True) + continue + for raw in logs.get("container") or []: + if _line_before(raw, self._since): + continue + line = _TS_PREFIX.sub("", raw.rstrip()) + shown = _filter_line(line) if line else None + if shown: + emit(self.events, "worker_log", self.name, shown) async def _poll_metrics(self) -> None: from .targets import _endpoint_url_base, _get_json @@ -100,6 +154,9 @@ async def _poll_metrics(self) -> None: await asyncio.sleep(METRICS_POLL_INTERVAL) def _report_counts(self, data: Dict[str, Any]) -> None: + # once a worker picked the job up, pool churn is noise + if self._streamed_workers: + return workers = data.get("workers") if not isinstance(workers, dict): return @@ -119,22 +176,76 @@ def _report_counts(self, data: Dict[str, Any]) -> None: emit(self.events, "worker_status", self.name, counts) async def _stream_logs(self, worker_id: str) -> None: + """follow the worker's container logs, retrying the attach. + + freshly provisioned workers 403 on the logs endpoint until their + pod row is resolvable by the gateway, and the stream itself can + drop mid-request; the attach retries with backoff for as long + as the request is in flight (stop() cancels the task). + """ from .logs import stream_pod_logs - try: - async for event in stream_pod_logs( - worker_id, - log_type="container", - tail=0, - since=self._since, - ): - line = _TS_PREFIX.sub("", (event.get("line") or "").rstrip()) - if line: - emit(self.events, "worker_log", self.name, line) - except asyncio.CancelledError: - raise - except Exception: # noqa: BLE001 - observability is best-effort - log.debug("log stream for %s ended", worker_id, exc_info=True) + attempt = 0 + while True: + streamed = False + try: + # backfill (tail) + since: lines printed between request + # start and the attach finally succeeding (fresh workers + # 403 until the gateway can resolve them) are recovered + # from the backfill instead of dropped + async for event in stream_pod_logs( + worker_id, + log_type="container", + tail=1000, + since=self._since, + ): + streamed = True + attempt = 0 + line = _TS_PREFIX.sub( + "", (event.get("line") or "").rstrip() + ) + if line: + self._lines_emitted += 1 + shown = _filter_line(line) + if shown: + emit( + self.events, "worker_log", self.name, shown + ) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - observability is best-effort + log.debug( + "log stream attach for %s failed (attempt %d)", + worker_id, + attempt + 1, + exc_info=True, + ) + if streamed: + # the server closed a healthy stream (1h cap or worker + # teardown); resume immediately from where we left off + self._since = datetime.now(timezone.utc).isoformat() + continue + attempt += 1 + # attach latency matters more than politeness: fresh pods + # 403 for tens of seconds and short jobs end quickly, so + # ramp linearly instead of exponentially + await asyncio.sleep(min(attempt, 30)) + + +def _line_before(raw: str, since_iso: str) -> bool: + """true when the log line's leading timestamp predates since.""" + match = _TS_PREFIX.match(raw) + if not match: + return False + ts = match.group(0).strip().replace("Z", "+00:00") + # docker timestamps carry nanoseconds; fromisoformat caps at micro + ts = re.sub(r"(\.\d{6})\d+", r"\1", ts) + try: + line_ts = datetime.fromisoformat(ts) + since = datetime.fromisoformat(since_iso) + except ValueError: + return False + return line_ts <= since def format_worker_counts(counts: Dict[str, int]) -> str: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 2078fbc83..79f7e829d 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -418,7 +418,15 @@ async def invoke( if monitor is not None: await monitor.start() try: - data = await _post_json(f"{base}/runsync", payload, headers, timeout) + if monitor is not None: + # async submit + status polling: runsync would hold the + # connection, hiding the workerId until completion and + # starving the live worker feed + data = await _post_json(f"{base}/run", payload, headers, timeout) + else: + data = await _post_json( + f"{base}/runsync", payload, headers, timeout + ) data = await _wait_terminal( base, data, diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 0b22732d0..0ca63185e 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -304,9 +304,6 @@ class DevEvents: def __init__(self) -> None: self._progress: Optional[Progress] = None self._tasks: Dict[str, TaskID] = {} - # in-flight request names; feed lines carry the name whenever - # calls overlap so interleaved output stays attributable - self._inflight: List[str] = [] def session_starting(self) -> None: self._progress = Progress( @@ -356,48 +353,43 @@ def deleted(self, name: str) -> None: console.print(f" [dim]{_padded(name)} deleted[/dim]") # -- request lifecycle (LiveTarget events) -- - - def _feed(self, name: str, body: str) -> None: - """a dim indented line under a dispatch; named when calls overlap.""" - if len(self._inflight) > 1: - console.print(f" [dim]│ {_padded(name)} {body}[/dim]") - else: - console.print(f" [dim]│ {body}[/dim]") + # next.js/wrangler-style event lines: leading status glyph, the + # function call in normal weight, everything else dim, timing at + # the end. function stdout renders as `name │ line` so concurrent + # calls interleave without losing attribution. def dispatch(self, name: str, label: str = "") -> None: - self._inflight.append(name) - suffix = f" [dim]{label}[/dim]" if label else "" - console.print( - f"[accent]→[/accent] [bold white]{_padded(name)}[/bold white]{suffix}" - ) + detail = f" [dim]{label}[/dim]" if label else "" + console.print(f" [accent]○[/accent] [white]{name}()[/white]{detail}") def worker_status(self, name: str, counts: Dict[str, int]) -> None: from runpod.apps.monitor import format_worker_counts - self._feed(name, format_worker_counts(counts)) + console.print( + f" [dim]○ {name}() waiting · {format_worker_counts(counts)}[/dim]" + ) def worker_ready(self, name: str, worker_id: str) -> None: - self._feed(name, f"worker {worker_id[:12]}") + console.print( + f" [accent]●[/accent] [white]{name}()[/white] [dim]running on " + f"worker {worker_id[:12]}[/dim]" + ) def worker_log(self, name: str, line: str) -> None: from rich.markup import escape - self._feed(name, escape(line)) - - def _done(self, name: str) -> None: - if name in self._inflight: - self._inflight.remove(name) + console.print(f" {_pipe(name)} {escape(line)}") def request_completed(self, name: str, elapsed_s: float) -> None: - self._done(name) console.print( - f"[ok]✓[/ok] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" + f" [ok]✓[/ok] [white]{name}()[/white] " + f"[dim]in {_fmt_elapsed(elapsed_s)}[/dim]" ) def request_failed(self, name: str, elapsed_s: float) -> None: - self._done(name) console.print( - f"[err]✗[/err] [white]{_padded(name)}[/white] [dim]{elapsed_s:.1f}s[/dim]" + f" [err]✗[/err] [white]{name}()[/white] " + f"[dim]failed in {_fmt_elapsed(elapsed_s)}[/dim]" ) @@ -416,18 +408,35 @@ def so_far(self) -> float: return time.monotonic() - self._start -def entrypoint_header() -> None: +def _fmt_elapsed(elapsed: float) -> str: + if elapsed < 1: + return f"{elapsed * 1000:.0f}ms" + if elapsed < 120: + return f"{elapsed:.1f}s" + return f"{int(elapsed // 60)}m{int(elapsed % 60):02d}s" + + +def entrypoint_header(fn_name: str = "") -> None: console.print() + if fn_name: + console.print( + f" [accent]▸[/accent] [white]{fn_name}()[/white] " + f"[dim]local entrypoint[/dim]" + ) def entrypoint_success(elapsed: float) -> None: console.print() console.print( - f"[dim]── {elapsed:.1f}s · enter re-run · edit reload · ^C quit ──[/dim]" + f" [ok]✓[/ok] [white]done[/white] [dim]in {_fmt_elapsed(elapsed)} " + f"· enter re-run · edit reload · ^C quit[/dim]" ) def entrypoint_failure(elapsed: float, err: str) -> None: console.print() - console.print(f"[err]✗ {err}[/err]") - console.print(f"[dim]── {elapsed:.1f}s · enter re-run · edit reload · ^C quit ──[/dim]") + console.print(f" [err]✗ {err}[/err]") + console.print( + f" [err]✗[/err] [white]failed[/white] [dim]in {_fmt_elapsed(elapsed)} " + f"· enter re-run · edit reload · ^C quit[/dim]" + ) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index ea40996db..f6a95c1b7 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -227,7 +227,7 @@ async def _session() -> int: try: while True: - ui.entrypoint_header() + ui.entrypoint_header(getattr(entrypoint, "__name__", "")) with ui.Timer() as t: try: run_entrypoint(entrypoint) diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index d05445055..7900e75dc 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -103,3 +103,37 @@ def test_malformed_payload_ignored(self): monitor._report_counts({"workers": None}) monitor._report_counts({}) assert sink.events == [] + + +class TestLineFiltering: + def test_user_prints_pass_through(self): + from runpod.apps.monitor import _filter_line + + assert _filter_line("hello world") == "hello world" + assert _filter_line("42") == "42" + + def test_sdk_info_frames_hidden(self): + from runpod.apps.monitor import _filter_line + + assert ( + _filter_line('{"requestId": null, "message": "Jobs in queue: 1", "level": "INFO"}') + is None + ) + assert ( + _filter_line('{"requestId": "abc", "message": "Started.", "level": "INFO"}') + is None + ) + + def test_sdk_error_frames_surface_message(self): + from runpod.apps.monitor import _filter_line + + assert ( + _filter_line('{"requestId": "abc", "message": "boom", "level": "ERROR"}') + == "boom" + ) + + def test_non_frame_json_passes_through(self): + from runpod.apps.monitor import _filter_line + + # user code printing json without the sdk frame shape + assert _filter_line('{"result": 5}') == '{"result": 5}' From 07e7feeaa223a547615ed955c7ba0adf0eedfb48 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:02:35 -0700 Subject: [PATCH 041/120] feat: nested dev calls, realtime log attach, refresh diffs --- runpod/apps/app.py | 25 +++++++++++++ runpod/apps/dev.py | 68 +++++++++++++++++++++++++++++++++- runpod/apps/monitor.py | 31 ++++++++++++---- runpod/apps/targets.py | 5 ++- runpod/rp_cli/console.py | 24 +++++++++--- runpod/rp_cli/main.py | 2 +- tests/test_apps/test_dev.py | 74 ++++++++++++++++++++++++++++++++++--- 7 files changed, 206 insertions(+), 23 deletions(-) diff --git a/runpod/apps/app.py b/runpod/apps/app.py index fc5ae3249..244a0d9c3 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -245,9 +245,34 @@ async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: # startup; a miss means the resource was added mid-session raise EndpointNotFound(self.name, spec.name) + if ctx is Context.WORKER and os.getenv("RUNPOD_DEV_APP") == self.name: + # nested call inside a dev worker: siblings live on dev + # endpoints (named dev-{app}-{resource}), not the sentinel + return await self._resolve_dev_sibling(spec) + env_name = os.getenv("FLASH_ENVIRONMENT") or self.env return SentinelTarget(self.name, env_name, spec.name) + async def _resolve_dev_sibling(self, spec: ResourceSpec) -> InvocationTarget: + """find the dev endpoint for a sibling resource by name.""" + from .targets import LiveTarget + + target = self._dev_targets.get(spec.name) + if target is not None: + return target + + from .api import AppsApiClient + from .dev import dev_endpoint_name + + wanted = dev_endpoint_name(self.name, spec.name) + endpoints = await AppsApiClient().list_my_endpoints() + for endpoint in endpoints: + if endpoint["name"] == wanted: + target = LiveTarget(endpoint["id"], spec.name) + self._dev_targets[spec.name] = target + return target + raise EndpointNotFound(self.name, spec.name) + def __repr__(self) -> str: return f"" diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index ff807967e..db39b885d 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -35,6 +35,41 @@ def _resource_of(endpoint_name: str) -> str: return endpoint_name.rsplit("-", 1)[-1] +def _comparable(payload: Dict) -> Dict: + """payload normalized for change detection. + + the generation env is bumped on every refresh by design; strip it so + only real configuration differences count. + """ + import copy + + clean = copy.deepcopy(payload) + clean.pop("id", None) + template = clean.get("template") or {} + template["env"] = [ + e + for e in template.get("env") or [] + if e.get("key") != GENERATION_ENV + ] + return clean + + +def _changed_fields(old: Dict, new: Dict) -> List[str]: + """human-readable names of top-level payload fields that differ.""" + fields = [] + for key in sorted(set(old) | set(new)): + if old.get(key) == new.get(key): + continue + if key == "template": + t_old, t_new = old.get(key) or {}, new.get(key) or {} + for sub in sorted(set(t_old) | set(t_new)): + if t_old.get(sub) != t_new.get(sub): + fields.append("image" if sub == "imageName" else sub) + else: + fields.append(key) + return fields + + def _image_for(spec: ResourceSpec) -> str: """dev worker image: custom image, env override, or the builtin runtime image matched to the local python (dev requests carry @@ -63,6 +98,12 @@ def _bootstrap_docker_args() -> str: return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") +def _client_api_key() -> str: + from .targets import _api_key + + return _api_key() + + def _cpu_locations(instance_ids: List[str]) -> str: """locations covering every requested cpu flavor's stock, limited to datacenters with storage and S3 support. the flash images are @@ -102,6 +143,11 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: "dockerArgs": "", "env": [ {"key": GENERATION_ENV, "value": str(generation)}, + # nested .remote() calls from inside a dev worker need + # credentials and the dev-session marker to resolve + # sibling dev endpoints by name + {"key": "RUNPOD_API_KEY", "value": _client_api_key()}, + {"key": "RUNPOD_DEV_APP", "value": app.name}, *({"key": k, "value": v} for k, v in (spec.env or {}).items()), ], }, @@ -169,6 +215,8 @@ def __init__( self.events = events # endpoint name -> id for everything this session owns self._endpoints: Dict[str, str] = {} + # endpoint name -> comparable payload, for refresh diffing + self._payloads: Dict[str, Dict] = {} def _emit(self, event: str, *args) -> None: handler = getattr(self.events, event, None) @@ -221,6 +269,7 @@ async def start(self) -> None: self._emit("ready", spec.name, endpoint_id) self._endpoints[name] = endpoint_id + self._payloads[name] = _comparable(payload) app._dev_targets[spec.name] = LiveTarget( endpoint_id, spec.name, @@ -251,9 +300,10 @@ async def refresh(self, apps: List[App]) -> None: for name in list(self._endpoints): if name not in desired: endpoint_id = self._endpoints.pop(name) + self._payloads.pop(name, None) try: await self.api.delete_endpoint(endpoint_id) - self._emit("deleted", _resource_of(name)) + self._emit("resource_removed", _resource_of(name)) log.info("deleted removed dev endpoint %s", name) except Exception as exc: log.warning("failed to delete %s: %s", name, exc) @@ -262,18 +312,32 @@ async def refresh(self, apps: List[App]) -> None: for name, (app, handle) in desired.items(): payload = _endpoint_input(app, handle.spec, self.generation) existing_id = self._endpoints.get(name) + comparable = _comparable(payload) + previous = self._payloads.get(name) if existing_id: payload["id"] = existing_id result = await self.api.save_endpoint(payload) endpoint_id = result["id"] self._endpoints[name] = endpoint_id + self._payloads[name] = comparable app._dev_targets[handle.spec.name] = LiveTarget( endpoint_id, handle.spec.name, events=self.events, metrics_key=result.get("aiKey"), ) - self._emit("refreshed", handle.spec.name, self.generation) + spec = handle.spec + hardware = ",".join(spec.cpu or spec.gpu or ["any"]) + if previous is None: + self._emit( + "resource_added", spec.name, spec.kind.value, hardware + ) + elif previous != comparable: + self._emit( + "resource_changed", + spec.name, + _changed_fields(previous, comparable), + ) log.info( "refreshed dev endpoint %s (%s, generation %d)", name, diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index 528ae2259..f11aa2440 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -93,6 +93,9 @@ def __init__( self._last_counts: Optional[Dict[str, int]] = None self._since = datetime.now(timezone.utc).isoformat() self._lines_emitted = 0 + # (ts, line) pairs already shown; reconnect backfill overlaps + # the previous window and must not replay lines + self._seen: set = set() async def start(self) -> None: if self.metrics_key: @@ -133,7 +136,13 @@ async def _snapshot_logs(self) -> None: if _line_before(raw, self._since): continue line = _TS_PREFIX.sub("", raw.rstrip()) - shown = _filter_line(line) if line else None + if not line: + continue + key = (raw.rstrip(), line) + if key in self._seen: + continue + self._seen.add(key) + shown = _filter_line(line) if shown: emit(self.events, "worker_log", self.name, shown) @@ -201,10 +210,13 @@ async def _stream_logs(self, worker_id: str) -> None: ): streamed = True attempt = 0 - line = _TS_PREFIX.sub( - "", (event.get("line") or "").rstrip() - ) + raw = (event.get("line") or "").rstrip() + line = _TS_PREFIX.sub("", raw) if line: + key = (event.get("ts") or raw, line) + if key in self._seen: + continue + self._seen.add(key) self._lines_emitted += 1 shown = _filter_line(line) if shown: @@ -226,10 +238,13 @@ async def _stream_logs(self, worker_id: str) -> None: self._since = datetime.now(timezone.utc).isoformat() continue attempt += 1 - # attach latency matters more than politeness: fresh pods - # 403 for tens of seconds and short jobs end quickly, so - # ramp linearly instead of exponentially - await asyncio.sleep(min(attempt, 30)) + # attach latency is the product: retry aggressively while + # the gateway warms up to the fresh pod (sub-second polls), + # then ease off if it stays unreachable + if attempt <= 20: + await asyncio.sleep(0.5) + else: + await asyncio.sleep(min(attempt - 20, 15)) def _line_before(raw: str, since_iso: str) -> bool: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 79f7e829d..047718663 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -233,6 +233,9 @@ async def _wait_terminal( on_status(data) deadline = time.monotonic() + timeout interval = 0.5 + # observed polls (dev sessions) stay tight so the worker id + # surfaces fast enough for log streams to attach in realtime + max_interval = 1.0 if on_status is not None else 5.0 while data.get("status") not in FINAL_STATUSES: job_id = data.get("id") if not job_id: @@ -243,7 +246,7 @@ async def _wait_terminal( f"(last status: {data.get('status', 'UNKNOWN')})" ) await asyncio.sleep(interval) - interval = min(interval * 1.5, 5.0) + interval = min(interval * 1.5, max_interval) data = await _get_json(f"{base_url}/status/{job_id}", headers, 30.0) if on_status is not None: on_status(data) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 0ca63185e..c83e1544c 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -172,8 +172,10 @@ def resources_table(rows: Iterable[tuple]) -> None: ) -def reload_flash() -> None: +def reload_banner(module: str = "") -> None: console.print() + detail = f" [dim]{module} changed[/dim]" if module else "" + console.print(f" [accent]↻[/accent] [white]reloading[/white]{detail}") # -- live deploy phases ------------------------------------------------------ @@ -344,13 +346,25 @@ def ready(self, name: str, endpoint_id: str) -> None: self._tasks[name], detail="ready", total=1, completed=1 ) - def refreshed(self, name: str, generation: int) -> None: + def deleted(self, name: str) -> None: + console.print(f" [dim]− {name}[/dim]") + + # -- refresh diff -- + + def resource_added(self, name: str, kind: str, hardware: str) -> None: console.print( - f"[warn]⚡[/warn] {_padded(name)} [dim]reloaded[/dim]" + f" [ok]+[/ok] [white]{name}[/white] " + f"[dim]{kind} · {hardware}[/dim]" ) - def deleted(self, name: str) -> None: - console.print(f" [dim]{_padded(name)} deleted[/dim]") + def resource_changed(self, name: str, fields: List[str]) -> None: + what = ", ".join(fields) if fields else "config" + console.print( + f" [warn]~[/warn] [white]{name}[/white] [dim]{what}[/dim]" + ) + + def resource_removed(self, name: str) -> None: + console.print(f" [err]−[/err] [white]{name}[/white]") # -- request lifecycle (LiveTarget events) -- # next.js/wrangler-style event lines: leading status glyph, the diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index f6a95c1b7..1acd2f25f 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -240,7 +240,7 @@ async def _session() -> int: return 0 reason = await _wait_for_rerun(watcher) if reason == "changed": - ui.reload_flash() + ui.reload_banner(str(module.name)) apps, entrypoint = _scan() ui.set_name_width(_all_resource_names(apps)) await session.refresh(apps) diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index f66e0af4c..bcad27204 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -67,9 +67,11 @@ def q(): assert loc in valid template = payload["template"] assert template["dockerArgs"] == "" - assert template["env"] == [ - {"key": "RUNPOD_DEV_GENERATION", "value": "1"} - ] + env = {e["key"]: e["value"] for e in template["env"]} + assert env["RUNPOD_DEV_GENERATION"] == "1" + # nested .remote() support: credentials + dev-session marker + assert "RUNPOD_API_KEY" in env + assert env["RUNPOD_DEV_APP"] == app.name assert "gpuIds" not in payload def test_gpu_queue_payload(self): @@ -295,8 +297,14 @@ def adopted(self, name, endpoint_id): def ready(self, name, endpoint_id): events.append(("ready", name, endpoint_id)) - def refreshed(self, name, generation): - events.append(("refreshed", name, generation)) + def resource_changed(self, name, fields): + events.append(("resource_changed", name, fields)) + + def resource_added(self, name, kind, hardware): + events.append(("resource_added", name, kind, hardware)) + + def resource_removed(self, name): + events.append(("resource_removed", name)) def deleted(self, name): events.append(("deleted", name)) @@ -320,12 +328,66 @@ def q(): kinds = [e[0] for e in events] assert "provisioning" in kinds assert "ready" in kinds - assert "refreshed" in kinds assert "deleted" in kinds + # unchanged resource refreshes silently: no diff events + assert "resource_changed" not in kinds + assert "resource_added" not in kinds assert ("provisioning", "q", "queue", "cpu3c-1-2") in events # deleted reports the resource name, not the endpoint name assert ("deleted", "q") in events + async def test_refresh_diff_events(self): + events = [] + + class Sink: + def resource_added(self, name, kind, hardware): + events.append(("added", name, kind, hardware)) + + def resource_changed(self, name, fields): + events.append(("changed", name, tuple(fields))) + + def resource_removed(self, name): + events.append(("removed", name)) + + app = App("diff-app") + + @app.queue(name="stays", cpu="cpu3c-1-2") + def stays(): + pass + + @app.queue(name="goes", cpu="cpu3c-1-2") + def goes(): + pass + + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.side_effect = lambda p: {"id": f"ep-{p['name']}"} + api.delete_endpoint.return_value = True + + session = DevSession([app], api=api, events=Sink()) + await session.start() + + # rescan: 'goes' removed, 'stays' reconfigured, 'fresh' added + app2 = App("diff-app") + + @app2.queue(name="stays", cpu="cpu3c-1-2", workers=(1, 4)) + def stays2(): + pass + + @app2.queue(name="fresh", cpu="cpu5c-2-4") + def fresh(): + pass + + await session.refresh([app2]) + + assert ("removed", "goes") in events + added = [e for e in events if e[0] == "added"] + assert ("added", "fresh", "queue", "cpu5c-2-4") in added + changed = [e for e in events if e[0] == "changed"] + assert len(changed) == 1 + assert changed[0][1] == "stays" + assert "workersMin" in changed[0][2] or "workersMax" in changed[0][2] + async def test_missing_event_methods_ignored(self): app = App("ev-app2") From 789e471f7c33d99050ef71f1542d7660de49b14a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:05:32 -0700 Subject: [PATCH 042/120] fix: install runpod package in task runtime images --- runpod/runtimes/task/Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/runpod/runtimes/task/Dockerfile b/runpod/runtimes/task/Dockerfile index 0ab5be41d..a1a9f18d5 100644 --- a/runpod/runtimes/task/Dockerfile +++ b/runpod/runtimes/task/Dockerfile @@ -7,9 +7,14 @@ ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ PYTHONUNBUFFERED=1 -RUN pip install --no-cache-dir cloudpickle +RUN apt-get update && apt-get install -y --no-install-recommends \ + git ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -COPY runpod/runtimes/task/runner.py /runner.py +# install the runpod package: task payloads carry the user's full module +# source, which imports runpod (decorators, nested .remote calls) +COPY . /src +RUN pip install --no-cache-dir /src cloudpickle && rm -rf /src EXPOSE 8080 -CMD ["python", "/runner.py"] +CMD ["python", "-m", "runpod.runtimes.task.runner"] From 8d350c157a1e4b80141789f9e842593b0ce08b16 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:08:20 -0700 Subject: [PATCH 043/120] fix: keyless dev payload construction --- runpod/apps/dev.py | 13 +++++++++++-- tests/test_apps/test_dev.py | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index db39b885d..de6480b9a 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -101,7 +101,12 @@ def _bootstrap_docker_args() -> str: def _client_api_key() -> str: from .targets import _api_key - return _api_key() + try: + return _api_key() + except RuntimeError: + # payload construction must not require credentials (tests, + # dry runs); the session itself fails loudly on its first call + return "" def _cpu_locations(instance_ids: List[str]) -> str: @@ -146,7 +151,11 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: # nested .remote() calls from inside a dev worker need # credentials and the dev-session marker to resolve # sibling dev endpoints by name - {"key": "RUNPOD_API_KEY", "value": _client_api_key()}, + *( + [{"key": "RUNPOD_API_KEY", "value": key}] + if (key := _client_api_key()) + else [] + ), {"key": "RUNPOD_DEV_APP", "value": app.name}, *({"key": k, "value": v} for k, v in (spec.env or {}).items()), ], diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index bcad27204..59d2d422b 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -69,9 +69,20 @@ def q(): assert template["dockerArgs"] == "" env = {e["key"]: e["value"] for e in template["env"]} assert env["RUNPOD_DEV_GENERATION"] == "1" - # nested .remote() support: credentials + dev-session marker - assert "RUNPOD_API_KEY" in env + # nested .remote() support: dev-session marker always present assert env["RUNPOD_DEV_APP"] == app.name + + def test_api_key_forwarded_when_configured(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "rpa_test123") + app = App("keyed") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + env = {e["key"]: e["value"] for e in payload["template"]["env"]} + assert env["RUNPOD_API_KEY"] == "rpa_test123" assert "gpuIds" not in payload def test_gpu_queue_payload(self): From ab4c10c4f7aa9c3a624deec2cf056cc42fc60347 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:14:24 -0700 Subject: [PATCH 044/120] ci: disable fail-fast on runtime image matrix --- .github/workflows/runtimes.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 4f02bc7c4..34e825a43 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -40,6 +40,9 @@ jobs: runs-on: ubuntu-latest needs: [test-runner] strategy: + # a transient registry/mirror failure on one leg must not cancel + # the other python versions + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: @@ -78,6 +81,7 @@ jobs: runs-on: ubuntu-latest needs: [test-runner, build-gpu-base] strategy: + fail-fast: false matrix: kind: [task, queue, api] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] From e1dfc83a80c45f01047af890f663ed96c049a51f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:38:41 -0700 Subject: [PATCH 045/120] ci: build runtime images even when gpu-base rebuild fails --- .github/workflows/runtimes.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 34e825a43..a36f97da4 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -79,7 +79,11 @@ jobs: build-runtimes: runs-on: ubuntu-latest + # gpu legs build FROM the registry's gpu-base tag, so a failed + # gpu-base rebuild (e.g. torch mirror outage) must not block + # runtime images: the previously published base keeps working needs: [test-runner, build-gpu-base] + if: always() && needs.test-runner.result == 'success' strategy: fail-fast: false matrix: From ff7c1b39f20718568cd2f0028f173d34cf6cc8dd Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:08:38 -0700 Subject: [PATCH 046/120] fix: make shipped module source inspectable for nested remote calls --- runpod/runtimes/task/runner.py | 25 ++++++++++++++++++++++--- tests/test_apps/test_dispatch.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index fc42473a9..1d53d1faa 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -150,9 +150,13 @@ def execute_request(request): # exec mirrors deployed-mode module import: the code is the # user's full module, so __name__ is set like an import would # (main guards stay inert) and decorated handles resolve to - # their wrapped functions - namespace = {"__name__": "__runpod_live__"} - exec(function_code, namespace) # noqa: S102 - that is the job + # their wrapped functions. the source is written to a real + # file first so inspect.getsource works inside the function + # (nested .remote() calls re-extract sibling source) + source_path = _materialize_source(function_code) + namespace = {"__name__": "__runpod_live__", "__file__": source_path} + code_obj = compile(function_code, source_path, "exec") + exec(code_obj, namespace) # noqa: S102 - that is the job if function_name not in namespace: return { "success": False, @@ -188,6 +192,21 @@ def execute_request(request): } +def _materialize_source(function_code): + """persist request source to a stable path for inspect/linecache.""" + import hashlib + import tempfile + + digest = hashlib.sha256(function_code.encode()).hexdigest()[:16] + path = os.path.join( + tempfile.gettempdir(), f"runpod_live_{digest}.py" + ) + if not os.path.exists(path): + with open(path, "w") as f: + f.write(function_code) + return path + + class _Tee(io.TextIOBase): """write-through to multiple streams, flushing eagerly so log followers see lines as they happen.""" diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index dc52ca4e5..b60689f49 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -304,3 +304,33 @@ def test_execute_request_unwraps_handles(self): ) assert response["success"], response.get("error") assert response["json_result"] == "AAAAAA" + + def test_nested_source_extraction_from_shipped_module(self): + # inside a live worker, a function calling sibling.remote() + # re-extracts that sibling's source via inspect; the shipped + # module must be inspectable after exec + from runpod.runtimes.task.runner import execute_request + + code = ( + "import runpod\n" + "app = runpod.App('t2')\n" + "@app.queue()\n" + "def sibling(x):\n" + " return x\n" + "@app.queue()\n" + "def caller():\n" + " import inspect\n" + " from runpod.apps.serialization import get_function_source\n" + " return get_function_source(sibling._fn)\n" + ) + response = execute_request( + { + "function_name": "caller", + "function_code": code, + "args": [], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"], response.get("error") + assert "def sibling" in response["json_result"] From 03c6cb36f1b21638cc15d72176f953c924b2aeed Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:36:53 -0700 Subject: [PATCH 047/120] fix: re-ship materialized module source on nested dev hops --- runpod/apps/serialization.py | 27 +++++++++++++++++++++++++-- tests/test_apps/test_dispatch.py | 4 +++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py index 5b0db0ad2..351abecf1 100644 --- a/runpod/apps/serialization.py +++ b/runpod/apps/serialization.py @@ -8,6 +8,7 @@ import ast import base64 import inspect +import os import textwrap from typing import Any, Callable, Dict, List @@ -45,5 +46,27 @@ def get_function_source(fn: Callable) -> str: return inspect.getsource(module) except (OSError, TypeError): pass - # no module file (repl, exec): fall back to the bare function body - return textwrap.dedent(inspect.getsource(fn)) + # exec'd module (nested hop inside a live worker): the runner + # materialized the shipped module to a real file; re-ship it whole + # so decorators and globals keep working on the next worker + filename = getattr(getattr(fn, "__code__", None), "co_filename", "") + if filename and os.path.isfile(filename): + with open(filename) as f: + return f.read() + # last resort (repl): the bare function body with decorators + # stripped, since their context cannot travel + return _bare_function_source(fn) + + +def _bare_function_source(fn: Callable) -> str: + """a function's own def, decorators removed.""" + source = textwrap.dedent(inspect.getsource(fn)) + tree = ast.parse(source) + for node in ast.walk(tree): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == fn.__name__ + ): + lines = source.split("\n") + return textwrap.dedent("\n".join(lines[node.lineno - 1 :])) + return source diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index b60689f49..8f4ce2815 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -319,7 +319,6 @@ def test_nested_source_extraction_from_shipped_module(self): " return x\n" "@app.queue()\n" "def caller():\n" - " import inspect\n" " from runpod.apps.serialization import get_function_source\n" " return get_function_source(sibling._fn)\n" ) @@ -333,4 +332,7 @@ def test_nested_source_extraction_from_shipped_module(self): } ) assert response["success"], response.get("error") + # the whole module re-ships (decorators intact), matching what + # the first hop sent assert "def sibling" in response["json_result"] + assert "@app.queue()" in response["json_result"] From 16ff3e85dd3d8b0900bc0bcca9c188309b22800c Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:46:21 -0700 Subject: [PATCH 048/120] feat: live api routes via source sync endpoint --- runpod/apps/handles.py | 5 ++ runpod/apps/serialization.py | 12 +++- runpod/apps/targets.py | 36 ++++++++++ runpod/apps/tasks.py | 11 +-- runpod/runtimes/api/server.py | 110 +++++++++++++++++++++++++---- tests/test_apps/test_api_server.py | 63 +++++++++++++++++ tests/test_apps/test_dispatch.py | 17 +++++ tests/test_apps/test_tasks.py | 12 +++- 8 files changed, 244 insertions(+), 22 deletions(-) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index a7b23ad27..8e6c4c57e 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -177,6 +177,11 @@ async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: if in_discovery(): raise DiscoveryInvocationError(self._handle.spec.name) target = await self._handle._app._resolve(self._handle.spec) + # live targets need the module source to materialize the api + # server-side; other targets ignore the reference + setter = getattr(target, "attach_source", None) + if setter is not None: + setter(self._handle.__wrapped__, self._handle.spec.name) return await target.request(self._method, path, body, **kwargs) diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py index 351abecf1..40bcae55e 100644 --- a/runpod/apps/serialization.py +++ b/runpod/apps/serialization.py @@ -28,7 +28,17 @@ def serialize_kwargs(kwargs: dict) -> Dict[str, str]: def deserialize_result(result_b64: str) -> Any: - return cloudpickle.loads(base64.b64decode(result_b64)) + try: + return cloudpickle.loads(base64.b64decode(result_b64)) + except ModuleNotFoundError as exc: + from .errors import RemoteExecutionError + + raise RemoteExecutionError( + f"the remote result contains an object from the " + f"'{exc.name}' package, which is not installed locally. " + f"return plain python types (e.g. str(...) it) or install " + f"'{exc.name}' on this machine." + ) from exc def get_function_source(fn: Callable) -> str: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 047718663..39a836cb5 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -359,6 +359,41 @@ def __init__( self.resource_name = resource_name self.events = events self.metrics_key = metrics_key + self._source_target: Optional[Any] = None + self._source_resource: str = "" + # source hash the worker last confirmed; None forces a sync + self._synced_hash: Optional[str] = None + + def attach_source(self, target: Any, resource: str) -> None: + """register the object whose module backs this api resource. + + live api endpoints materialize their routes from module source + pushed over /_runpod/sync (once per source change), keeping dev + behavior identical to deployed. + """ + self._source_target = target + self._source_resource = resource + + async def _sync_source(self, timeout: float) -> None: + """push the current module source to the worker if it changed.""" + if self._source_target is None: + return + import hashlib + + from .serialization import get_function_source + + source = get_function_source(self._source_target) + digest = hashlib.sha256(source.encode()).hexdigest() + if digest == self._synced_hash: + return + url = f"https://{self.endpoint_id}.{_lb_domain()}/_runpod/sync" + await _post_json( + url, + {"source": source, "resource": self._source_resource}, + _headers(), + timeout, + ) + self._synced_hash = digest def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict @@ -488,6 +523,7 @@ async def request( url = f"https://{self.endpoint_id}.{_lb_domain()}{path}" client_timeout = aiohttp.ClientTimeout(total=timeout) try: + await self._sync_source(timeout) async with aiohttp.ClientSession(timeout=client_timeout) as session: async with session.request( method, url, json=body, headers=_headers() diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 52edf6cf0..662dc54e6 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -56,13 +56,16 @@ def _bootstrap_command() -> str: def _device_names(gpu: Optional[List[str]]) -> List[str]: - """pods take exact device names, not pool ids; expand any pool ids + """pods take exact device names, not pool ids; expand pool ids (e.g. ADA_24 -> NVIDIA GeForce RTX 4090) and pass device names - through.""" - from .gpu import GpuGroup + through. no gpu constraint means any device, which the pod api + only understands as the full device list.""" + from .gpu import POOLS_TO_TYPES, GpuGroup if not gpu: - return ["any"] + return sorted( + {t.value for types in POOLS_TO_TYPES.values() for t in types} + ) names: List[str] = [] for entry in gpu: try: diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py index c4e258676..411494dac 100644 --- a/runpod/runtimes/api/server.py +++ b/runpod/runtimes/api/server.py @@ -14,8 +14,11 @@ - asgi factory: call the factory, serve what it returns live mode (rp dev): - no artifact. serves /execute, which runs FunctionRequest payloads - (source per request) via the task runner's execute_request. + no artifact. the client pushes the user's module source to + /_runpod/sync (once per source change); the matching api class is + materialized from it and serves the real routes, so dev requests + hit the same handlers a deployed worker would. /execute remains + for FunctionRequest payloads. """ import importlib @@ -112,6 +115,14 @@ async def ping(): return JSONResponse({"status": "initializing"}, status_code=204) return {"status": "healthy"} + _mount_routes(app, handle, instance) + return app + + +def _mount_routes(app: Any, handle, instance) -> None: + """add each @get/@post route of an ApiHandle's class to a fastapi app.""" + from fastapi import Request + for route in handle.spec.routes: method = getattr(route, "method", None) or route["method"] path = getattr(route, "path", None) or route["path"] @@ -138,8 +149,6 @@ async def endpoint(request: Request): path, make_endpoint(bound), methods=[method], name=handler_name ) - return app - def _build_factory_app(handle) -> Any: """call the user's asgi factory and ensure /ping exists.""" @@ -155,23 +164,94 @@ async def ping(): return app -def _build_live_app() -> Any: - """generic /execute server for dev sessions (source per request).""" +async def _materialize_live_api(source: str, resource: str) -> Any: + """exec shipped module source and build the api app for a resource. + + @init runs here, before the app is swapped in, so the first + routed request already sees initialized state. + """ from fastapi import FastAPI - app = FastAPI(title="runpod-live-api") + from runpod.apps.handles import ApiHandle + from runpod.runtimes.task.runner import _materialize_source + + path = _materialize_source(source) + namespace: dict = {"__name__": "__runpod_live__", "__file__": path} + exec(compile(source, path, "exec"), namespace) # noqa: S102 + + handle = None + for value in namespace.values(): + if isinstance(value, ApiHandle) and value.spec.name == resource: + handle = value + break + if handle is None: + raise RuntimeError( + f"no @app.api resource named '{resource}' in shipped module" + ) + if handle._cls is None: + return _build_factory_app(handle) - @app.get("/ping") - async def ping(): - return {"status": "healthy"} + instance = handle._cls() + if handle._init_name: + await _maybe_await(getattr(instance, handle._init_name)()) + app = FastAPI(title=f"{resource} (live)") + _mount_routes(app, handle, instance) + return app - @app.post("/execute") - async def execute(request: dict): - from runpod.runtimes.task.runner import execute_request - return execute_request(request.get("input", request)) +class _LiveDispatcher: + """asgi front for dev sessions. - return app + the client pushes module source to /_runpod/sync; the api app is + materialized from it (rebuilt when the source hash changes) and + serves every route. /ping and /execute always work. + """ + + def __init__(self): + self._core = self._build_core() + self._inner: Any = None + self._hash: Optional[str] = None + + def _build_core(self) -> Any: + import hashlib + + from fastapi import FastAPI + + app = FastAPI(title="runpod-live-api") + + @app.get("/ping") + async def ping(): + return {"status": "healthy"} + + @app.post("/execute") + async def execute(request: dict): + from runpod.runtimes.task.runner import execute_request + + return execute_request(request.get("input", request)) + + @app.post("/_runpod/sync") + async def sync(request: dict): + source = request.get("source") or "" + resource = request.get("resource") or "" + digest = hashlib.sha256(source.encode()).hexdigest() + if digest != self._hash: + self._inner = await _materialize_live_api(source, resource) + self._hash = digest + return {"status": "synced", "hash": digest} + + return app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self._core(scope, receive, send) + path = scope.get("path", "") + if path in ("/ping", "/execute", "/_runpod/sync") or self._inner is None: + return await self._core(scope, receive, send) + return await self._inner(scope, receive, send) + + +def _build_live_app() -> Any: + return _LiveDispatcher() def build_app() -> Any: diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py index 84afaccec..28307add4 100644 --- a/tests/test_apps/test_api_server.py +++ b/tests/test_apps/test_api_server.py @@ -157,3 +157,66 @@ def test_execute_endpoint(self, monkeypatch): data = response.json() assert data["success"] is True assert data["json_result"] == 42 + + +class TestLiveDispatcher: + def test_sync_materializes_routes(self): + from fastapi.testclient import TestClient + + from runpod.runtimes.api.server import _build_live_app + + source = ( + "import runpod\n" + "from runpod.apps.markers import get, init, post\n" + "app = runpod.App('live')\n" + "@app.api(cpu='cpu3c-1-2')\n" + "class Counter:\n" + " @init\n" + " def setup(self):\n" + " self.count = 0\n" + " @post('/bump')\n" + " async def bump(self, body: dict):\n" + " self.count += body.get('by', 1)\n" + " return {'count': self.count}\n" + " @get('/value')\n" + " async def value(self):\n" + " return {'count': self.count}\n" + ) + + client = TestClient(_build_live_app()) + # before sync: core routes only + assert client.get("/ping").status_code == 200 + assert client.post("/bump", json={}).status_code == 404 + + r = client.post( + "/_runpod/sync", json={"source": source, "resource": "Counter"} + ) + assert r.status_code == 200 + + assert client.post("/bump", json={"by": 3}).json() == {"count": 3} + assert client.get("/value").json() == {"count": 3} + # core routes still work after materialization + assert client.get("/ping").status_code == 200 + + def test_resync_same_source_keeps_state(self): + from fastapi.testclient import TestClient + + from runpod.runtimes.api.server import _build_live_app + + source = ( + "import runpod\n" + "from runpod.apps.markers import post\n" + "app = runpod.App('live2')\n" + "@app.api(cpu='cpu3c-1-2')\n" + "class S:\n" + " @post('/add')\n" + " async def add(self, body: dict):\n" + " self.total = getattr(self, 'total', 0) + body['x']\n" + " return {'total': self.total}\n" + ) + client = TestClient(_build_live_app()) + client.post("/_runpod/sync", json={"source": source, "resource": "S"}) + assert client.post("/add", json={"x": 2}).json() == {"total": 2} + # identical source: no rebuild, state survives + client.post("/_runpod/sync", json={"source": source, "resource": "S"}) + assert client.post("/add", json={"x": 3}).json() == {"total": 5} diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index 8f4ce2815..2ed21b6a5 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -336,3 +336,20 @@ def test_nested_source_extraction_from_shipped_module(self): # the first hop sent assert "def sibling" in response["json_result"] assert "@app.queue()" in response["json_result"] + + def test_missing_module_on_deserialize_is_actionable(self): + import base64 + + import cloudpickle + import pytest + + from runpod.apps.errors import RemoteExecutionError + from runpod.apps.serialization import deserialize_result + + class FakePickle: + def __reduce__(self): + return (__import__, ("nonexistent_pkg_xyz",)) + + payload = base64.b64encode(cloudpickle.dumps(FakePickle())).decode() + with pytest.raises(RemoteExecutionError, match="nonexistent_pkg_xyz"): + deserialize_result(payload) diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index a78faba8c..56b3ca1dd 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -285,10 +285,18 @@ def test_device_name_passes_through(self): assert _device_names(["NVIDIA L40S"]) == ["NVIDIA L40S"] - def test_none_means_any(self): + def test_none_expands_to_all_devices(self): + # the pod api has no "any" wildcard; unconstrained gpu means + # every known device name + from runpod.apps.gpu import POOLS_TO_TYPES from runpod.apps.tasks import _device_names - assert _device_names(None) == ["any"] + names = _device_names(None) + assert "any" not in names + all_devices = { + t.value for types in POOLS_TO_TYPES.values() for t in types + } + assert set(names) == all_devices def test_mixed_pool_and_device(self): from runpod.apps.tasks import _device_names From 2d1bc642d4adb6df6c4d90c51534a352b90e2f9a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:44:14 -0700 Subject: [PATCH 049/120] fix: cancellable entrypoint with spinner-based session cleanup --- runpod/apps/context.py | 18 ++++++++++++- runpod/apps/dev.py | 25 +++++++++++++++--- runpod/rp_cli/console.py | 57 ++++++++++++++++++++++++++++++++++++++++ runpod/rp_cli/main.py | 39 +++++++++++++++++++++++---- 4 files changed, 129 insertions(+), 10 deletions(-) diff --git a/runpod/apps/context.py b/runpod/apps/context.py index 4aeb2919c..1c96fe726 100644 --- a/runpod/apps/context.py +++ b/runpod/apps/context.py @@ -68,7 +68,23 @@ def _ensure_loop(cls) -> asyncio.AbstractEventLoop: @classmethod def run(cls, coro: Coroutine[Any, Any, Any]) -> Any: loop = cls._ensure_loop() - return asyncio.run_coroutine_threadsafe(coro, loop).result() + future = asyncio.run_coroutine_threadsafe(coro, loop) + try: + # wait in short slices: an uninterruptible result() would + # block signal delivery, making ctrl-c useless while a + # remote call is in flight. TimeoutError is ambiguous (the + # wait slice and the coroutine's own timeouts share the + # type since 3.11), so future.done() decides + while True: + try: + return future.result(timeout=0.2) + except TimeoutError: + if future.done(): + raise + except BaseException: + # ctrl-c (or anything else) abandons the in-flight call + future.cancel() + raise def block(coro: Coroutine[Any, Any, Any]) -> Any: diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index de6480b9a..739edfd2d 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -354,14 +354,31 @@ async def refresh(self, apps: List[App]) -> None: self.generation, ) - async def stop(self) -> None: - """delete every endpoint this session owns.""" - for name, endpoint_id in list(self._endpoints.items()): + async def stop(self, events: Optional[object] = None) -> None: + """delete every endpoint this session owns. + + events, when given, overrides the session sink for teardown + rendering: cleanup_started(total), deleting(name), deleted(name), + delete_failed(name). + """ + sink = events if events is not None else self.events + + def _tell(event: str, *args) -> None: + handler = getattr(sink, event, None) + if handler is not None: + handler(*args) + + pending = list(self._endpoints.items()) + _tell("cleanup_started", len(pending)) + for name, endpoint_id in pending: + resource = _resource_of(name) + _tell("deleting", resource) try: await self.api.delete_endpoint(endpoint_id) - self._emit("deleted", _resource_of(name)) + _tell("deleted", resource) log.info("deleted dev endpoint %s (%s)", name, endpoint_id) except Exception as exc: + _tell("delete_failed", resource) log.warning( "failed to delete dev endpoint %s: %s", endpoint_id, exc ) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index c83e1544c..6b108e8d7 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -407,6 +407,63 @@ def request_failed(self, name: str, elapsed_s: float) -> None: ) +class CleanupEvents: + """session teardown sink: one spinner line with delete progress.""" + + def __init__(self) -> None: + self._progress: Optional[Progress] = None + self._task: Optional[TaskID] = None + self._total = 0 + self._done = 0 + + def cleanup_started(self, total: int) -> None: + self._total = total + if total == 0: + return + console.print() + self._progress = Progress( + SpinnerColumn("dots", style=ACCENT, finished_text="[ok]✓[/ok]"), + TextColumn("[white]{task.description}[/white]"), + TextColumn("[dim]{task.fields[detail]}[/dim]"), + console=console, + ) + self._progress.start() + self._task = self._progress.add_task( + "cleaning up", detail="", total=total + ) + + def deleting(self, name: str) -> None: + if self._progress is not None and self._task is not None: + self._progress.update( + self._task, detail=f"{name} · {self._done}/{self._total}" + ) + + def deleted(self, name: str) -> None: + self._done += 1 + if self._progress is not None and self._task is not None: + self._progress.update( + self._task, + completed=self._done, + detail=f"{name} · {self._done}/{self._total}", + ) + + def delete_failed(self, name: str) -> None: + self._done += 1 + if self._progress is not None and self._task is not None: + self._progress.update(self._task, completed=self._done) + console.print(f" [warn]![/warn] [dim]could not delete {name}[/dim]") + + def close(self) -> None: + if self._progress is not None and self._task is not None: + self._progress.update( + self._task, + completed=self._total, + detail=f"{self._done} deleted", + ) + self._progress.stop() + self._progress = None + + class Timer: """context timer for elapsed reporting.""" diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 1acd2f25f..3278c4c36 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -215,6 +215,33 @@ async def _wait_for_rerun(watcher: "FileWatcher") -> str: pass await asyncio.sleep(0.5) + async def _run_entrypoint_cancellable(fn) -> None: + """drive the entrypoint on a daemon thread. + + the entrypoint is user code full of blocking .remote() calls; + running it inline would pin the main loop and make ctrl-c + undeliverable (asyncio's sigint handler cancels the main task, + which needs an await point). a daemon thread keeps the loop + free, and cancellation simply abandons the in-flight call. + """ + loop = asyncio.get_running_loop() + done: asyncio.Future = loop.create_future() + + def _runner() -> None: + try: + run_entrypoint(fn) + except BaseException as exc: # noqa: BLE001 - marshalled to the loop + loop.call_soon_threadsafe( + lambda: done.set_exception(exc) if not done.done() else None + ) + else: + loop.call_soon_threadsafe( + lambda: done.set_result(None) if not done.done() else None + ) + + threading.Thread(target=_runner, daemon=True).start() + await done + async def _session() -> int: nonlocal apps, entrypoint ui.set_name_width(_all_resource_names(apps)) @@ -230,8 +257,10 @@ async def _session() -> int: ui.entrypoint_header(getattr(entrypoint, "__name__", "")) with ui.Timer() as t: try: - run_entrypoint(entrypoint) + await _run_entrypoint_cancellable(entrypoint) ui.entrypoint_success(t.so_far) + except asyncio.CancelledError: + raise except Exception as exc: # noqa: BLE001 - dev loop survives user errors ui.entrypoint_failure(t.so_far, str(exc)) if once: @@ -245,13 +274,13 @@ async def _session() -> int: ui.set_name_width(_all_resource_names(apps)) await session.refresh(apps) finally: - ui.console.print() - ui.info("cleaning up ...") - await session.stop() + events = ui.CleanupEvents() + await session.stop(events=events) + events.close() try: sys.exit(asyncio.run(_session())) - except (KeyboardInterrupt, EOFError): + except (KeyboardInterrupt, EOFError, asyncio.CancelledError): ui.console.print() except Exception as exc: # noqa: BLE001 - surface engine errors cleanly raise click.ClickException(str(exc)) from exc From c2a25aea9439e88c90a1290ee33149ba1319b1f5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:23:18 -0700 Subject: [PATCH 050/120] feat: app and environment management, undeploy, and browser login --- runpod/api/graphql.py | 19 +- runpod/apps/api.py | 113 ++++++++++ runpod/apps/auth.py | 83 +++++++ runpod/apps/manage.py | 142 ++++++++++++ runpod/rp_cli/main.py | 314 ++++++++++++++++++++++++++- runpod/serverless/utils/rp_upload.py | 9 +- tests/test_apps/test_auth.py | 60 +++++ tests/test_apps/test_manage.py | 150 +++++++++++++ 8 files changed, 880 insertions(+), 10 deletions(-) create mode 100644 runpod/apps/auth.py create mode 100644 runpod/apps/manage.py create mode 100644 tests/test_apps/test_auth.py create mode 100644 tests/test_apps/test_manage.py diff --git a/runpod/api/graphql.py b/runpod/api/graphql.py index 8074163b3..02a033b72 100644 --- a/runpod/api/graphql.py +++ b/runpod/api/graphql.py @@ -28,12 +28,14 @@ def _resolve_api_key(api_key: Optional[str]) -> str: return effective_api_key -def _build_headers(api_key: str) -> Dict[str, str]: - return { +def _build_headers(api_key: Optional[str]) -> Dict[str, str]: + headers = { "Content-Type": "application/json", "User-Agent": USER_AGENT, - "Authorization": f"Bearer {api_key}", } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers def _build_payload(query: str, variables: Optional[Dict[str, Any]]) -> Dict[str, Any]: @@ -84,6 +86,7 @@ async def run_graphql_query_async( api_key: Optional[str] = None, variables: Optional[Dict[str, Any]] = None, timeout: float = 60.0, + allow_anonymous: bool = False, ) -> Dict[str, Any]: """ Async variant of run_graphql_query, sharing the same url, headers, @@ -94,10 +97,18 @@ async def run_graphql_query_async( api_key: Optional API key to use for this query. variables: Optional GraphQL variables to send with the query. timeout: Total request timeout in seconds. + allow_anonymous: Send the request without credentials when no + key is available (pre-login flows). """ import aiohttp # pylint: disable=import-outside-toplevel - effective_api_key = _resolve_api_key(api_key) + if allow_anonymous: + try: + effective_api_key: Optional[str] = _resolve_api_key(api_key) + except error.AuthenticationError: + effective_api_key = None + else: + effective_api_key = _resolve_api_key(api_key) client_timeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(timeout=client_timeout) as session: diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 84d339bb2..609cd8faa 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -158,6 +158,119 @@ async def create_environment(self, app_id: str, name: str) -> Dict[str, Any]: ) return data["createFlashEnvironment"] + # -- app / environment management -- + + async def list_apps(self) -> List[Dict[str, Any]]: + """all flash apps with their environments and builds.""" + query = """ + query getFlashApps { + myself { + flashApps { + id + name + flashEnvironments { + id + name + state + createdAt + activeBuildId + } + flashBuilds { id createdAt } + } + } + } + """ + data = await self._execute(query) + return data["myself"].get("flashApps") or [] + + async def get_environment_by_name( + self, app_name: str, env_name: str + ) -> Optional[Dict[str, Any]]: + """an environment with its attached endpoints and volumes.""" + app = await self.get_app_by_name(app_name) + if app is None: + return None + query = """ + query getFlashEnvironmentByName($input: FlashEnvironmentByNameInput!) { + flashEnvironmentByName(input: $input) { + id + name + state + activeBuildId + createdAt + endpoints { id name } + networkVolumes { id name } + } + } + """ + try: + data = await self._execute( + query, + {"input": {"flashAppId": app["id"], "name": env_name}}, + ) + except QueryError as exc: + if "not found" in str(exc).lower(): + return None + raise + return data["flashEnvironmentByName"] + + async def delete_app(self, app_id: str) -> bool: + mutation = """ + mutation deleteFlashApp($flashAppId: String!) { + deleteFlashApp(flashAppId: $flashAppId) + } + """ + data = await self._execute(mutation, {"flashAppId": app_id}) + return "deleteFlashApp" in data + + async def delete_environment(self, environment_id: str) -> bool: + mutation = """ + mutation deleteFlashEnvironment($flashEnvironmentId: String!) { + deleteFlashEnvironment(flashEnvironmentId: $flashEnvironmentId) + } + """ + data = await self._execute( + mutation, {"flashEnvironmentId": environment_id} + ) + return "deleteFlashEnvironment" in data + + # -- browser auth (login) -- + + async def create_auth_request(self) -> Dict[str, Any]: + """open a browser-approval auth request; no credentials needed.""" + mutation = """ + mutation createFlashAuthRequest { + createFlashAuthRequest { + id + status + expiresAt + } + } + """ + response = await run_graphql_query_async( + mutation, api_key=self._api_key, allow_anonymous=True + ) + return response["data"].get("createFlashAuthRequest") or {} + + async def get_auth_request_status(self, request_id: str) -> Dict[str, Any]: + query = """ + query flashAuthRequestStatus($flashAuthRequestId: String!) { + flashAuthRequestStatus(flashAuthRequestId: $flashAuthRequestId) { + id + status + expiresAt + apiKey + } + } + """ + response = await run_graphql_query_async( + query, + api_key=self._api_key, + variables={"flashAuthRequestId": request_id}, + allow_anonymous=True, + ) + return response["data"].get("flashAuthRequestStatus") or {} + async def prepare_artifact_upload( self, app_id: str, tarball_size: int ) -> Dict[str, Any]: diff --git a/runpod/apps/auth.py b/runpod/apps/auth.py new file mode 100644 index 000000000..b84765922 --- /dev/null +++ b/runpod/apps/auth.py @@ -0,0 +1,83 @@ +"""browser-approval login flow. + +opens a console url tied to a short-lived auth request; the api key +lands when the user approves in the browser. no credentials are sent +before approval, so an expired local key can never poison the flow. +""" + +import asyncio +import datetime as dt +import os +from typing import Any, Callable, Dict, Optional + +from .api import AppsApiClient +from .errors import AppError + +CONSOLE_URL = os.environ.get("RUNPOD_CONSOLE_URL", "https://console.runpod.io") + +POLL_INTERVAL_SECONDS = 2.0 +DEFAULT_TIMEOUT_SECONDS = 600.0 + +_APPROVED_STATUSES = frozenset({"APPROVED", "CONSUMED"}) +_FAILED_STATUSES = frozenset({"DENIED", "EXPIRED"}) + + +class LoginError(AppError): + pass + + +def _parse_expires_at(value: Optional[str]) -> Optional[dt.datetime]: + if not value: + return None + try: + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + + +def auth_url(request_id: str) -> str: + return f"{CONSOLE_URL}/flash/login?request={request_id}" + + +async def browser_login( + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, + on_url: Optional[Callable[[str], None]] = None, + api: Optional[AppsApiClient] = None, +) -> str: + """run the browser-approval flow and return the granted api key. + + on_url is called with the approval url as soon as the request is + created (the caller renders/opens it). + """ + client = api or AppsApiClient() + request: Dict[str, Any] = await client.create_auth_request() + request_id = request.get("id") + if not request_id: + raise LoginError("auth request failed to initialize") + + if on_url is not None: + on_url(auth_url(request_id)) + + deadline = dt.datetime.now(dt.timezone.utc) + dt.timedelta( + seconds=timeout_seconds + ) + expires_at = _parse_expires_at(request.get("expiresAt")) + if expires_at and expires_at < deadline: + deadline = expires_at + + while True: + status_payload = await client.get_auth_request_status(request_id) + status = status_payload.get("status") + api_key = status_payload.get("apiKey") + + if api_key and status in _APPROVED_STATUSES: + return api_key + if status in _FAILED_STATUSES: + raise LoginError(f"login {status.lower()}") + if status == "CONSUMED": + raise LoginError("login request was already used") + if dt.datetime.now(dt.timezone.utc) >= deadline: + raise LoginError("login timed out") + + await asyncio.sleep(POLL_INTERVAL_SECONDS) diff --git a/runpod/apps/manage.py b/runpod/apps/manage.py new file mode 100644 index 000000000..7745fe4e8 --- /dev/null +++ b/runpod/apps/manage.py @@ -0,0 +1,142 @@ +"""app and environment lifecycle: list, inspect, undeploy, delete. + +everything resolves server-side (the flash app registry is the source +of truth); nothing is tracked locally. undeploying an environment +deletes its endpoints first so no orphaned workers keep billing. +""" + +import logging +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +from .api import AppsApiClient +from .errors import AppError + +log = logging.getLogger(__name__) + + +class AppNotFound(AppError): + def __init__(self, app_name: str): + super().__init__( + f"no app named '{app_name}' found. run `rp deploy` to create one." + ) + + +class EnvironmentNotFound(AppError): + def __init__(self, app_name: str, env_name: str): + super().__init__( + f"no environment '{env_name}' in app '{app_name}'." + ) + + +@dataclass +class UndeployResult: + """outcome of tearing down one environment.""" + + endpoints_deleted: int = 0 + environment_deleted: bool = False + app_deleted: bool = False + failures: List[str] = field(default_factory=list) + + +async def list_apps(api: Optional[AppsApiClient] = None) -> List[Dict]: + """all apps with their environments, newest first.""" + client = api or AppsApiClient() + apps = await client.list_apps() + return sorted(apps, key=lambda a: a.get("name") or "") + + +async def get_app( + app_name: str, api: Optional[AppsApiClient] = None +) -> Dict: + client = api or AppsApiClient() + app = await client.get_app_by_name(app_name) + if app is None: + raise AppNotFound(app_name) + return app + + +async def get_environment( + app_name: str, env_name: str, api: Optional[AppsApiClient] = None +) -> Dict: + client = api or AppsApiClient() + env = await client.get_environment_by_name(app_name, env_name) + if env is None: + raise EnvironmentNotFound(app_name, env_name) + return env + + +async def undeploy_environment( + app_name: str, + env_name: str, + *, + api: Optional[AppsApiClient] = None, + delete_env: bool = True, + events: Optional[object] = None, +) -> UndeployResult: + """tear down an environment: endpoints first, then the environment. + + events, when given, receives cleanup_started(total), deleting(name), + deleted(name), delete_failed(name). + """ + client = api or AppsApiClient() + env = await get_environment(app_name, env_name, api=client) + + def _tell(event: str, *args) -> None: + handler = getattr(events, event, None) + if handler is not None: + handler(*args) + + result = UndeployResult() + endpoints = env.get("endpoints") or [] + _tell("cleanup_started", len(endpoints)) + for endpoint in endpoints: + name = endpoint.get("name") or endpoint.get("id") + _tell("deleting", name) + try: + await client.delete_endpoint(endpoint["id"]) + result.endpoints_deleted += 1 + _tell("deleted", name) + log.info("deleted endpoint %s (%s)", name, endpoint["id"]) + except Exception as exc: # noqa: BLE001 - collected for the caller + result.failures.append(f"endpoint {name}: {exc}") + _tell("delete_failed", name) + + if delete_env and not result.failures: + try: + await client.delete_environment(env["id"]) + result.environment_deleted = True + log.info("deleted environment %s (%s)", env_name, env["id"]) + except Exception as exc: # noqa: BLE001 - collected for the caller + result.failures.append(f"environment {env_name}: {exc}") + + return result + + +async def delete_app( + app_name: str, + *, + api: Optional[AppsApiClient] = None, + events: Optional[object] = None, +) -> UndeployResult: + """delete an app after undeploying every environment in it.""" + client = api or AppsApiClient() + app = await get_app(app_name, api=client) + + result = UndeployResult() + for env in app.get("flashEnvironments") or []: + env_result = await undeploy_environment( + app_name, env["name"], api=client, events=events + ) + result.endpoints_deleted += env_result.endpoints_deleted + result.failures.extend(env_result.failures) + + if not result.failures: + try: + await client.delete_app(app["id"]) + result.app_deleted = True + log.info("deleted app %s (%s)", app_name, app["id"]) + except Exception as exc: # noqa: BLE001 - collected for the caller + result.failures.append(f"app {app_name}: {exc}") + + return result diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 3278c4c36..307ec69d3 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -328,16 +328,320 @@ async def _run(): @cli.command() -def login(): - """Authenticate with Runpod and store the API key.""" +@click.option( + "--no-open", is_flag=True, help="Print the url instead of opening a browser." +) +@click.option( + "--api-key", + "api_key_opt", + default=None, + help="Store this API key directly (skips the browser flow).", +) +def login(no_open, api_key_opt): + """Authenticate with Runpod and store the API key. + + Opens the Runpod console for browser approval by default; pass + --api-key to store a key directly. + """ + from runpod.apps.auth import LoginError, browser_login from runpod.cli.groups.config.functions import set_credentials + from runpod.rp_cli import console as ui + + if api_key_opt: + try: + set_credentials(api_key_opt, overwrite=True) + except ValueError as exc: + _fail(str(exc)) + ui.success("credentials saved to [dim]~/.runpod/config.toml[/dim]") + return + + def _show_url(url: str) -> None: + ui.console.print() + ui.console.print(" [white]authorize in your browser:[/white]") + ui.console.print(f" [accent.light][link={url}]{url}[/link][/accent.light]") + ui.console.print() + if not no_open: + click.launch(url) - api_key = click.prompt("Runpod API key", hide_input=True) try: + with ui.console.status("[dim]waiting for approval ...[/dim]"): + api_key = asyncio.run(browser_login(on_url=_show_url)) set_credentials(api_key, overwrite=True) - except ValueError as exc: + except (LoginError, ValueError) as exc: _fail(str(exc)) - click.echo("credentials saved to ~/.runpod/config.toml") + ui.success("logged in · credentials saved to [dim]~/.runpod/config.toml[/dim]") + + +# ---------------------------------------------------------- app / env + + +def _fmt_ts(value) -> str: + if not value: + return "-" + text = str(value) + return text[:10] if len(text) >= 10 else text + + +@cli.group() +def app(): + """Manage deployed apps.""" + + +@app.command(name="list") +def app_list(): + """List all apps and their environments.""" + from runpod.apps.manage import list_apps + from runpod.rp_cli import console as ui + + try: + apps = asyncio.run(list_apps()) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + if not apps: + ui.console.print("\n no apps deployed · run [white]rp deploy[/white]\n") + return + + width = max(len(a["name"]) for a in apps) + ui.console.print() + for entry in apps: + envs = entry.get("flashEnvironments") or [] + names = ", ".join(e["name"] for e in envs) or "-" + ui.console.print( + f" [white]{entry['name']:<{width}}[/white] [dim]{names}[/dim]" + ) + ui.console.print() + + +@app.command(name="get") +@click.argument("app_name") +def app_get(app_name): + """Show one app with its environments.""" + from runpod.apps.manage import get_app + from runpod.rp_cli import console as ui + + try: + entry = asyncio.run(get_app(app_name)) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + envs = entry.get("flashEnvironments") or [] + ui.console.print() + ui.console.print(f" [white]{entry['name']}[/white]") + if envs: + width = max(len(e["name"]) for e in envs) + for env_entry in envs: + build = (env_entry.get("activeBuildId") or "no build")[:12] + ui.console.print( + f" [white]{env_entry['name']:<{width}}[/white] [dim]{build}[/dim]" + ) + ui.console.print() + + +@app.command(name="delete") +@click.argument("app_name") +@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") +def app_delete(app_name, yes): + """Delete an app, undeploying every environment in it.""" + from runpod.apps.manage import delete_app + from runpod.rp_cli import console as ui + + if not yes: + click.confirm( + f"delete app '{app_name}' and all its environments?", abort=True + ) + + events = ui.CleanupEvents() + try: + result = asyncio.run(delete_app(app_name, events=events)) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + finally: + events.close() + + if result.failures: + for failure in result.failures: + ui.error(failure) + raise click.ClickException("undeploy incomplete; app kept") + ui.success( + f"deleted [white]{app_name}[/white] " + f"[dim]{result.endpoints_deleted} endpoints removed[/dim]" + ) + + +@cli.group() +def env(): + """Manage app environments.""" + + +def _resolve_app_name(app_name) -> str: + """--app flag or the single app discovered in cwd.""" + if app_name: + return app_name + from runpod.apps.discovery import DiscoveryError, discover_apps + + try: + apps = discover_apps(Path.cwd()) + except DiscoveryError: + apps = [] + if len(apps) == 1: + return apps[0].name + _fail("pass --app (no unique app found in the current directory)") + + +@env.command(name="list") +@click.option("--app", "-a", "app_name", default=None, help="App name.") +def env_list(app_name): + """List environments for an app.""" + from runpod.apps.manage import get_app + from runpod.rp_cli import console as ui + + name = _resolve_app_name(app_name) + try: + entry = asyncio.run(get_app(name)) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + envs = entry.get("flashEnvironments") or [] + if not envs: + ui.console.print( + f"\n no environments in [white]{name}[/white] · run [white]rp deploy[/white]\n" + ) + return + ui.console.print() + width = max(len(e["name"]) for e in envs) + for env_entry in envs: + build = (env_entry.get("activeBuildId") or "-")[:12] + ui.console.print( + f" [white]{env_entry['name']:<{width}}[/white] " + f"[dim]{build} · {_fmt_ts(env_entry.get('createdAt'))}[/dim]" + ) + ui.console.print() + + +@env.command(name="get") +@click.argument("env_name") +@click.option("--app", "-a", "app_name", default=None, help="App name.") +def env_get(env_name, app_name): + """Show an environment with its endpoints.""" + from runpod.apps.manage import get_environment + from runpod.rp_cli import console as ui + + name = _resolve_app_name(app_name) + try: + entry = asyncio.run(get_environment(name, env_name)) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + ui.console.print() + ui.console.print( + f" [white]{name}/{entry['name']}[/white] " + f"[dim]build {(entry.get('activeBuildId') or '-')[:12]}[/dim]" + ) + endpoints = entry.get("endpoints") or [] + if endpoints: + width = max(len(e["name"]) for e in endpoints) + for endpoint in endpoints: + ui.console.print( + f" [white]{endpoint['name']:<{width}}[/white] " + f"{ui.endpoint_link(endpoint['id'])}" + ) + ui.console.print() + + +@env.command(name="create") +@click.argument("env_name") +@click.option("--app", "-a", "app_name", default=None, help="App name.") +def env_create(env_name, app_name): + """Create a new environment in an app.""" + from runpod.apps.api import AppsApiClient + from runpod.apps.manage import get_app + from runpod.rp_cli import console as ui + + name = _resolve_app_name(app_name) + + async def _create(): + client = AppsApiClient() + entry = await get_app(name, api=client) + return await client.create_environment(entry["id"], env_name) + + try: + asyncio.run(_create()) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + ui.success(f"created [white]{name}/{env_name}[/white]") + + +@env.command(name="delete") +@click.argument("env_name") +@click.option("--app", "-a", "app_name", default=None, help="App name.") +@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") +def env_delete(env_name, app_name, yes): + """Undeploy and delete an environment.""" + from runpod.apps.manage import undeploy_environment + from runpod.rp_cli import console as ui + + name = _resolve_app_name(app_name) + if not yes: + click.confirm( + f"undeploy and delete '{name}/{env_name}'?", abort=True + ) + + events = ui.CleanupEvents() + try: + result = asyncio.run( + undeploy_environment(name, env_name, events=events) + ) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + finally: + events.close() + + if result.failures: + for failure in result.failures: + ui.error(failure) + raise click.ClickException("undeploy incomplete; environment kept") + ui.success( + f"deleted [white]{name}/{env_name}[/white] " + f"[dim]{result.endpoints_deleted} endpoints removed[/dim]" + ) + + +@cli.command() +@click.option("--app", "-a", "app_name", default=None, help="App name.") +@click.option("--env", "-e", "env_name", default="default", show_default=True) +@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") +def undeploy(app_name, env_name, yes): + """Tear down a deployed environment's endpoints. + + Deletes the environment's endpoints and the environment itself; + the app and its build history remain. + """ + from runpod.apps.manage import undeploy_environment + from runpod.rp_cli import console as ui + + name = _resolve_app_name(app_name) + if not yes: + click.confirm(f"undeploy '{name}/{env_name}'?", abort=True) + + events = ui.CleanupEvents() + try: + result = asyncio.run( + undeploy_environment(name, env_name, events=events) + ) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + finally: + events.close() + + if result.failures: + for failure in result.failures: + ui.error(failure) + raise click.ClickException("undeploy incomplete") + ui.success( + f"undeployed [white]{name}/{env_name}[/white] " + f"[dim]{result.endpoints_deleted} endpoints removed[/dim]" + ) # -------------------------------------------------------- legacy groups diff --git a/runpod/serverless/utils/rp_upload.py b/runpod/serverless/utils/rp_upload.py index d4e9a015b..407400fbd 100644 --- a/runpod/serverless/utils/rp_upload.py +++ b/runpod/serverless/utils/rp_upload.py @@ -19,9 +19,16 @@ from boto3.s3.transfer import TransferConfig from botocore.client import BaseClient +# configured on this logger only: basicConfig would install a root +# handler for the entire process at import time logger = logging.getLogger("runpod upload utility") FMT = "%(filename)-20s:%(lineno)-4d %(asctime)s %(message)s" -logging.basicConfig(level=logging.INFO, format=FMT, handlers=[logging.StreamHandler()]) +if not logger.handlers: + _handler = logging.StreamHandler() + _handler.setFormatter(logging.Formatter(FMT)) + logger.addHandler(_handler) + logger.setLevel(logging.INFO) + logger.propagate = False def _import_boto3_dependencies(): diff --git a/tests/test_apps/test_auth.py b/tests/test_apps/test_auth.py new file mode 100644 index 000000000..335bfc2f3 --- /dev/null +++ b/tests/test_apps/test_auth.py @@ -0,0 +1,60 @@ +"""browser login flow.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from runpod.apps.auth import LoginError, auth_url, browser_login + + +def _api(statuses): + api = AsyncMock() + api.create_auth_request.return_value = { + "id": "req-1", + "status": "PENDING", + "expiresAt": None, + } + api.get_auth_request_status.side_effect = statuses + return api + + +class TestBrowserLogin: + def test_returns_key_on_approval(self, monkeypatch): + monkeypatch.setattr("runpod.apps.auth.POLL_INTERVAL_SECONDS", 0) + api = _api( + [ + {"status": "PENDING"}, + {"status": "APPROVED", "apiKey": "rpa_new"}, + ] + ) + urls = [] + key = asyncio.run( + browser_login(api=api, on_url=urls.append) + ) + assert key == "rpa_new" + assert urls == [auth_url("req-1")] + + def test_denied_raises(self, monkeypatch): + monkeypatch.setattr("runpod.apps.auth.POLL_INTERVAL_SECONDS", 0) + api = _api([{"status": "DENIED"}]) + with pytest.raises(LoginError, match="denied"): + asyncio.run(browser_login(api=api)) + + def test_timeout_raises(self, monkeypatch): + monkeypatch.setattr("runpod.apps.auth.POLL_INTERVAL_SECONDS", 0) + api = _api([{"status": "PENDING"}] * 50) + with pytest.raises(LoginError, match="timed out"): + asyncio.run(browser_login(api=api, timeout_seconds=0)) + + def test_missing_request_id_raises(self): + api = AsyncMock() + api.create_auth_request.return_value = {} + with pytest.raises(LoginError, match="initialize"): + asyncio.run(browser_login(api=api)) + + def test_consumed_without_key_raises(self, monkeypatch): + monkeypatch.setattr("runpod.apps.auth.POLL_INTERVAL_SECONDS", 0) + api = _api([{"status": "CONSUMED"}]) + with pytest.raises(LoginError, match="already used"): + asyncio.run(browser_login(api=api)) diff --git a/tests/test_apps/test_manage.py b/tests/test_apps/test_manage.py new file mode 100644 index 000000000..637505a51 --- /dev/null +++ b/tests/test_apps/test_manage.py @@ -0,0 +1,150 @@ +"""app/environment lifecycle: list, inspect, undeploy, delete.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from runpod.apps.manage import ( + AppNotFound, + EnvironmentNotFound, + delete_app, + get_app, + get_environment, + list_apps, + undeploy_environment, +) + + +def _api(**overrides): + api = AsyncMock() + api.list_apps.return_value = [] + api.get_app_by_name.return_value = None + api.get_environment_by_name.return_value = None + api.delete_endpoint.return_value = True + api.delete_environment.return_value = True + api.delete_app.return_value = True + for key, value in overrides.items(): + getattr(api, key).return_value = value + return api + + +class TestQueries: + def test_list_sorted_by_name(self): + api = _api( + list_apps=[{"name": "zeta"}, {"name": "alpha"}] + ) + apps = asyncio.run(list_apps(api=api)) + assert [a["name"] for a in apps] == ["alpha", "zeta"] + + def test_get_app_not_found(self): + with pytest.raises(AppNotFound, match="rp deploy"): + asyncio.run(get_app("ghost", api=_api())) + + def test_get_environment_not_found(self): + with pytest.raises(EnvironmentNotFound, match="ghost"): + asyncio.run(get_environment("app", "ghost", api=_api())) + + +class TestUndeployEnvironment: + def _env(self, n_endpoints=2): + return { + "id": "env-1", + "name": "default", + "endpoints": [ + {"id": f"ep-{i}", "name": f"svc-{i}"} + for i in range(n_endpoints) + ], + } + + def test_deletes_endpoints_then_environment(self): + api = _api(get_environment_by_name=self._env()) + result = asyncio.run( + undeploy_environment("app", "default", api=api) + ) + assert result.endpoints_deleted == 2 + assert result.environment_deleted + assert not result.failures + api.delete_environment.assert_awaited_once_with("env-1") + + def test_keep_env_flag(self): + api = _api(get_environment_by_name=self._env()) + result = asyncio.run( + undeploy_environment("app", "default", api=api, delete_env=False) + ) + assert result.endpoints_deleted == 2 + assert not result.environment_deleted + api.delete_environment.assert_not_awaited() + + def test_endpoint_failure_keeps_environment(self): + api = _api(get_environment_by_name=self._env(1)) + api.delete_endpoint.side_effect = RuntimeError("in use") + result = asyncio.run( + undeploy_environment("app", "default", api=api) + ) + assert result.endpoints_deleted == 0 + assert not result.environment_deleted + assert result.failures + api.delete_environment.assert_not_awaited() + + def test_events_emitted(self): + events = [] + + class Sink: + def cleanup_started(self, total): + events.append(("started", total)) + + def deleting(self, name): + events.append(("deleting", name)) + + def deleted(self, name): + events.append(("deleted", name)) + + api = _api(get_environment_by_name=self._env(1)) + asyncio.run( + undeploy_environment("app", "default", api=api, events=Sink()) + ) + assert events == [ + ("started", 1), + ("deleting", "svc-0"), + ("deleted", "svc-0"), + ] + + +class TestDeleteApp: + def test_undeploys_all_envs_then_app(self): + app_data = { + "id": "app-1", + "name": "myapp", + "flashEnvironments": [ + {"id": "env-1", "name": "default"}, + {"id": "env-2", "name": "staging"}, + ], + } + api = _api(get_app_by_name=app_data) + api.get_environment_by_name.side_effect = [ + {"id": "env-1", "name": "default", "endpoints": [{"id": "e1", "name": "a"}]}, + {"id": "env-2", "name": "staging", "endpoints": []}, + ] + result = asyncio.run(delete_app("myapp", api=api)) + assert result.endpoints_deleted == 1 + assert result.app_deleted + api.delete_app.assert_awaited_once_with("app-1") + + def test_failure_keeps_app(self): + app_data = { + "id": "app-1", + "name": "myapp", + "flashEnvironments": [{"id": "env-1", "name": "default"}], + } + api = _api(get_app_by_name=app_data) + api.get_environment_by_name.return_value = { + "id": "env-1", + "name": "default", + "endpoints": [{"id": "e1", "name": "a"}], + } + api.delete_endpoint.side_effect = RuntimeError("nope") + result = asyncio.run(delete_app("myapp", api=api)) + assert not result.app_deleted + assert result.failures + api.delete_app.assert_not_awaited() From c028747e498f343a159f464aa5149fa59680f635 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:53:40 -0700 Subject: [PATCH 051/120] feat: task lifecycle streaming in dev, long tasks via submit-poll --- runpod/apps/api.py | 23 +++- runpod/apps/app.py | 5 +- runpod/apps/dev.py | 4 + runpod/apps/discovery.py | 7 ++ runpod/apps/monitor.py | 217 +++++++++++++++++--------------- runpod/apps/targets.py | 52 +++++++- runpod/apps/tasks.py | 45 ++++--- runpod/rp_cli/console.py | 29 ++++- runpod/rp_cli/main.py | 28 ++++- runpod/runtimes/task/runner.py | 5 +- tests/test_apps/test_monitor.py | 9 +- 11 files changed, 289 insertions(+), 135 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 609cd8faa..5fae9ec6d 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -14,6 +14,9 @@ from ..error import QueryError +_TRANSPORT_RETRIES = 4 + + class AppsApiClient: """async control-plane client scoped to apps provisioning.""" @@ -23,10 +26,22 @@ def __init__(self, api_key: Optional[str] = None): async def _execute( self, query: str, variables: Optional[Dict[str, Any]] = None ) -> Dict[str, Any]: - response = await run_graphql_query_async( - query, api_key=self._api_key, variables=variables - ) - return response["data"] + import asyncio + + last_exc: Optional[Exception] = None + for attempt in range(_TRANSPORT_RETRIES): + if attempt: + await asyncio.sleep(2 ** (attempt - 1)) + try: + response = await run_graphql_query_async( + query, api_key=self._api_key, variables=variables + ) + return response["data"] + except (aiohttp.ClientError, OSError, asyncio.TimeoutError) as exc: + # transport-level failures (ssl hiccups, resets, dns) + # are transient; graphql/auth errors propagate untouched + last_exc = exc + raise last_exc # type: ignore[misc] # -- endpoints (dev session provisioning) -- diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 244a0d9c3..6dacee7b7 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -56,6 +56,8 @@ def __init__(self, name: str, *, env: str = DEFAULT_ENV): self.name = name self.env = env self._resources: Dict[str, Union[FunctionHandle, ApiHandle]] = {} + # event sink for task lifecycle rendering, set by dev sessions + self._dev_events: Optional[object] = None # populated by an `rp dev` session: resource name -> live target self._dev_targets: Dict[str, InvocationTarget] = {} _REGISTRY.append(self) @@ -233,7 +235,8 @@ async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: if spec.kind is ResourceKind.TASK: handle = self._resources.get(spec.name) fn = getattr(handle, "_fn", None) - return PodTarget(spec, fn) + # dev sessions attach an event sink for lifecycle rendering + return PodTarget(spec, fn, events=self._dev_events) ctx = current_context() diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 739edfd2d..0ddf08ded 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -250,6 +250,9 @@ async def start(self) -> None: """provision (or adopt) a live endpoint per queue/api resource and register the targets on each app.""" self._emit("session_starting") + for app in self.apps: + # task targets read the sink off the app at resolve time + app._dev_events = self.events existing = {e["name"]: e for e in await self.api.list_my_endpoints()} for app in self.apps: @@ -385,6 +388,7 @@ def _tell(event: str, *args) -> None: self._endpoints.clear() for app in self.apps: app._dev_targets.clear() + app._dev_events = None async def __aenter__(self) -> "DevSession": await self.start() diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index d011b3754..a61a8eff7 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -124,6 +124,7 @@ def discover_apps(target: Path) -> List[App]: os.environ[DISCOVERY_ENV] = "1" try: for path in _python_files(target): + seen = set(id(a) for a in get_registered_apps()) try: _import_module(path) except DiscoveryError as exc: @@ -131,6 +132,12 @@ def discover_apps(target: Path) -> List[App]: raise failures.append(str(exc)) log.warning("%s", exc) + continue + # stamp fresh apps with their defining file so callers can + # report where each app came from + for app in get_registered_apps(): + if id(app) not in seen and not hasattr(app, "_source_file"): + app._source_file = path finally: os.environ.pop(DISCOVERY_ENV, None) if sys_path_added: diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index f11aa2440..3aae241f8 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -67,6 +67,113 @@ def emit(sink: Optional[object], event: str, *args: Any) -> None: log.debug("event sink %s failed", event, exc_info=True) +class PodLogStream: + """follows one pod's container logs and emits worker_log events. + + shared by live endpoint workers and task pods: attach() starts the + follow (retrying aggressively while the gateway warms up to a fresh + pod), stop() cancels it and falls back to a snapshot when the + stream never yielded a line (jobs shorter than the attach window). + lines dedup across reconnects and sdk runtime frames are filtered. + """ + + def __init__(self, pod_id: str, resource_name: str, events: object): + self.pod_id = pod_id + self.name = resource_name + self.events = events + self._task: Optional[asyncio.Task] = None + self._since = datetime.now(timezone.utc).isoformat() + self._lines_emitted = 0 + # (ts, line) pairs already shown; reconnect backfill overlaps + # the previous window and must not replay lines + self._seen: set = set() + + def attach(self) -> None: + if self._task is None: + self._task = asyncio.ensure_future(self._follow()) + + async def stop(self) -> None: + if self._task is not None: + self._task.cancel() + await asyncio.gather(self._task, return_exceptions=True) + self._task = None + # the stream often cannot attach within a short job's lifetime + # (fresh pods 403 on the logs endpoint until the gateway can + # resolve them); a final snapshot recovers the output + if not self._lines_emitted: + await self._snapshot() + + def _emit_line(self, key: tuple, line: str) -> None: + if key in self._seen: + return + self._seen.add(key) + shown = _filter_line(line) + if shown: + emit(self.events, "worker_log", self.name, shown) + + async def _snapshot(self) -> None: + from .logs import pod_logs + + try: + logs = await pod_logs(self.pod_id, log_type="container") + except Exception: # noqa: BLE001 - observability is best-effort + log.debug("log snapshot for %s failed", self.pod_id, exc_info=True) + return + for raw in logs.get("container") or []: + if _line_before(raw, self._since): + continue + line = _TS_PREFIX.sub("", raw.rstrip()) + if line: + self._emit_line((raw.rstrip(), line), line) + + async def _follow(self) -> None: + from .logs import stream_pod_logs + + attempt = 0 + while True: + streamed = False + try: + # backfill (tail) + since: lines printed between request + # start and the attach finally succeeding (fresh pods + # 403 until the gateway can resolve them) are recovered + # from the backfill instead of dropped + async for event in stream_pod_logs( + self.pod_id, + log_type="container", + tail=1000, + since=self._since, + ): + streamed = True + attempt = 0 + raw = (event.get("line") or "").rstrip() + line = _TS_PREFIX.sub("", raw) + if line: + self._lines_emitted += 1 + self._emit_line((event.get("ts") or raw, line), line) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - observability is best-effort + log.debug( + "log stream attach for %s failed (attempt %d)", + self.pod_id, + attempt + 1, + exc_info=True, + ) + if streamed: + # the server closed a healthy stream (1h cap or worker + # teardown); resume immediately from where we left off + self._since = datetime.now(timezone.utc).isoformat() + continue + attempt += 1 + # attach latency is the product: retry aggressively while + # the gateway warms up to the fresh pod (sub-second polls), + # then ease off if it stays unreachable + if attempt <= 20: + await asyncio.sleep(0.5) + else: + await asyncio.sleep(min(attempt - 20, 15)) + + class WorkerMonitor: """observes one in-flight request against a live endpoint. @@ -89,13 +196,8 @@ def __init__( # ai key; the user api key is rejected there self.metrics_key = metrics_key self._tasks: List[asyncio.Task] = [] - self._streamed_workers: set = set() + self._streams: Dict[str, PodLogStream] = {} self._last_counts: Optional[Dict[str, int]] = None - self._since = datetime.now(timezone.utc).isoformat() - self._lines_emitted = 0 - # (ts, line) pairs already shown; reconnect backfill overlaps - # the previous window and must not replay lines - self._seen: set = set() async def start(self) -> None: if self.metrics_key: @@ -104,12 +206,11 @@ async def start(self) -> None: def on_status(self, data: Dict[str, Any]) -> None: """inspect a job-status payload for the assigned worker.""" worker_id = data.get("workerId") - if worker_id and worker_id not in self._streamed_workers: - self._streamed_workers.add(worker_id) + if worker_id and worker_id not in self._streams: emit(self.events, "worker_ready", self.name, str(worker_id)) - self._tasks.append( - asyncio.ensure_future(self._stream_logs(str(worker_id))) - ) + stream = PodLogStream(str(worker_id), self.name, self.events) + stream.attach() + self._streams[str(worker_id)] = stream async def stop(self) -> None: for task in self._tasks: @@ -117,34 +218,8 @@ async def stop(self) -> None: if self._tasks: await asyncio.gather(*self._tasks, return_exceptions=True) self._tasks.clear() - # the stream often cannot attach within a short job's lifetime - # (fresh pods 403 on the logs endpoint until the gateway can - # resolve them); a final snapshot recovers the output - if self._streamed_workers and not self._lines_emitted: - await self._snapshot_logs() - - async def _snapshot_logs(self) -> None: - from .logs import pod_logs - - for worker_id in self._streamed_workers: - try: - logs = await pod_logs(worker_id, log_type="container") - except Exception: # noqa: BLE001 - observability is best-effort - log.debug("log snapshot for %s failed", worker_id, exc_info=True) - continue - for raw in logs.get("container") or []: - if _line_before(raw, self._since): - continue - line = _TS_PREFIX.sub("", raw.rstrip()) - if not line: - continue - key = (raw.rstrip(), line) - if key in self._seen: - continue - self._seen.add(key) - shown = _filter_line(line) - if shown: - emit(self.events, "worker_log", self.name, shown) + for stream in self._streams.values(): + await stream.stop() async def _poll_metrics(self) -> None: from .targets import _endpoint_url_base, _get_json @@ -164,7 +239,7 @@ async def _poll_metrics(self) -> None: def _report_counts(self, data: Dict[str, Any]) -> None: # once a worker picked the job up, pool churn is noise - if self._streamed_workers: + if self._streams: return workers = data.get("workers") if not isinstance(workers, dict): @@ -184,68 +259,6 @@ def _report_counts(self, data: Dict[str, Any]) -> None: return emit(self.events, "worker_status", self.name, counts) - async def _stream_logs(self, worker_id: str) -> None: - """follow the worker's container logs, retrying the attach. - - freshly provisioned workers 403 on the logs endpoint until their - pod row is resolvable by the gateway, and the stream itself can - drop mid-request; the attach retries with backoff for as long - as the request is in flight (stop() cancels the task). - """ - from .logs import stream_pod_logs - - attempt = 0 - while True: - streamed = False - try: - # backfill (tail) + since: lines printed between request - # start and the attach finally succeeding (fresh workers - # 403 until the gateway can resolve them) are recovered - # from the backfill instead of dropped - async for event in stream_pod_logs( - worker_id, - log_type="container", - tail=1000, - since=self._since, - ): - streamed = True - attempt = 0 - raw = (event.get("line") or "").rstrip() - line = _TS_PREFIX.sub("", raw) - if line: - key = (event.get("ts") or raw, line) - if key in self._seen: - continue - self._seen.add(key) - self._lines_emitted += 1 - shown = _filter_line(line) - if shown: - emit( - self.events, "worker_log", self.name, shown - ) - except asyncio.CancelledError: - raise - except Exception: # noqa: BLE001 - observability is best-effort - log.debug( - "log stream attach for %s failed (attempt %d)", - worker_id, - attempt + 1, - exc_info=True, - ) - if streamed: - # the server closed a healthy stream (1h cap or worker - # teardown); resume immediately from where we left off - self._since = datetime.now(timezone.utc).isoformat() - continue - attempt += 1 - # attach latency is the product: retry aggressively while - # the gateway warms up to the fresh pod (sub-second polls), - # then ease off if it stays unreachable - if attempt <= 20: - await asyncio.sleep(0.5) - else: - await asyncio.sleep(min(attempt - 20, 15)) - def _line_before(raw: str, since_iso: str) -> bool: """true when the log line's leading timestamp predates since.""" diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 39a836cb5..efbc328cc 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -562,9 +562,15 @@ class PodTarget(InvocationTarget): # tasks default to a long window; the pod's terminateAfter is the backstop TASK_TIMEOUT_SECONDS = 3600.0 - def __init__(self, spec: ResourceSpec, fn: Callable): + def __init__( + self, + spec: ResourceSpec, + fn: Callable, + events: Optional[object] = None, + ): self.spec = spec self.fn = fn + self.events = events def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict @@ -588,16 +594,56 @@ def build_payload( async def invoke( self, payload: Dict[str, Any], *, timeout: float = TASK_TIMEOUT_SECONDS ) -> Any: + import time + + from .monitor import emit from .tasks import TaskExecution, unwrap_task_response + name = self.spec.name + hardware = ",".join(self.spec.cpu or self.spec.gpu or ["any"]) + emit(self.events, "dispatch", name) + emit(self.events, "task_status", name, f"provisioning pod · {hardware}") + start = time.monotonic() + + stream = None execution = TaskExecution(self.spec) - await execution.start() try: + await execution.start() + if execution.pod_id: + emit( + self.events, + "task_status", + name, + f"pod {execution.pod_id[:12]} · waiting for runtime", + ) + if self.events is not None: + from .monitor import PodLogStream + + stream = PodLogStream(execution.pod_id, name, self.events) + stream.attach() await execution.wait_ready() + emit(self.events, "worker_ready", name, execution.pod_id or "") response = await execution.execute(payload, timeout) + except Exception: + emit( + self.events, + "request_failed", + name, + time.monotonic() - start, + ) + raise finally: + if stream is not None: + await stream.stop() await execution.terminate() - return unwrap_task_response(response) + result = unwrap_task_response(response) + emit( + self.events, + "request_completed", + name, + time.monotonic() - start, + ) + return result async def submit(self, payload: Dict[str, Any]) -> Any: from .tasks import TaskExecution, TaskJob diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 662dc54e6..01f6fa30a 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -179,38 +179,47 @@ async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: ) async def execute(self, request: Dict[str, Any], timeout: float) -> Dict[str, Any]: - """run to completion via /execute. + """run to completion: submit, then poll for the result. + + the pod proxy caps how long a single request can stay open + (long jobs would 524), so completion always goes through the + background slot + short /result polls. + """ + await self.submit(request) + deadline = time.monotonic() + timeout + while True: + if time.monotonic() >= deadline: + raise TimeoutError( + f"task on pod {self.pod_id} did not finish in {timeout}s" + ) + response = await self.poll_result() + if response is not None: + return response + await asyncio.sleep(RESULT_POLL_INTERVAL) + + async def submit(self, request: Dict[str, Any]) -> None: + """start in the background via /submit. the proxy answers 404 for pods an edge node has not learned about yet; since this pod demonstrably exists (we created it and /ping succeeded), 404s here are propagation races and are retried briefly rather than surfaced. """ - url = f"{_proxy_url(self.pod_id)}/execute" - client_timeout = aiohttp.ClientTimeout(total=timeout) + url = f"{_proxy_url(self.pod_id)}/submit" attempts = 6 - async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with aiohttp.ClientSession() as session: for attempt in range(attempts): async with session.post( - url, json=request, headers=self._headers + url, + json=request, + headers=self._headers, + timeout=aiohttp.ClientTimeout(total=30), ) as resp: if resp.status == 404 and attempt < attempts - 1: await asyncio.sleep(2 * (attempt + 1)) continue resp.raise_for_status() - return await resp.json() - - async def submit(self, request: Dict[str, Any]) -> None: - """start in the background via /submit.""" - url = f"{_proxy_url(self.pod_id)}/submit" - async with aiohttp.ClientSession() as session: - async with session.post( - url, - json=request, - headers=self._headers, - timeout=aiohttp.ClientTimeout(total=30), - ) as resp: - resp.raise_for_status() + return async def poll_result(self) -> Optional[Dict[str, Any]]: """fetch /result; returns the response dict once DONE, else None.""" diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 6b108e8d7..9b8272b99 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -143,11 +143,33 @@ def dev_banner(app_names: Iterable[str], module: str) -> None: ) -def deploy_banner(app_name: str, env: str, resources: Iterable[Tuple[str, str]]) -> None: +def deploy_plan(apps: Iterable[Tuple[str, str, int]]) -> None: + """upfront summary for multi-app deploys: (name, source, n_resources).""" + apps = list(apps) + console.print( + f"[bold white]deploy[/bold white] [dim]{len(apps)} apps[/dim]" + ) + width = max(len(name) for name, _, _ in apps) + for name, source, count in apps: + detail = f"{source} · " if source else "" + console.print( + f" [white]{name:<{width}}[/white] " + f"[dim]{detail}{count} resource{'s' if count != 1 else ''}[/dim]" + ) + + +def deploy_banner( + app_name: str, + env: str, + resources: Iterable[Tuple[str, str]], + source: str = "", +) -> None: """resources: (name, kind) pairs.""" names = ", ".join(name for name, _ in resources) or "(no resources)" + origin = f" · {source}" if source else "" console.print( - f"[bold white]deploy[/bold white] [dim]{app_name} → {env} · {names}[/dim]" + f"[bold white]deploy[/bold white] [white]{app_name}[/white] " + f"[dim]→ {env}{origin} · {names}[/dim]" ) @@ -383,6 +405,9 @@ def worker_status(self, name: str, counts: Dict[str, int]) -> None: f" [dim]○ {name}() waiting · {format_worker_counts(counts)}[/dim]" ) + def task_status(self, name: str, detail: str) -> None: + console.print(f" [dim]○ {name}() {detail}[/dim]") + def worker_ready(self, name: str, worker_id: str) -> None: console.print( f" [accent]●[/accent] [white]{name}()[/white] [dim]running on " diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 307ec69d3..7dc3acd03 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -24,6 +24,17 @@ def _fail(message: str) -> None: raise click.ClickException(message) +def _app_source(found, project_root: Path) -> str: + """the file an app was discovered in, relative to the project.""" + source = getattr(found, "_source_file", None) + if source is None: + return "" + try: + return str(Path(source).resolve().relative_to(project_root)) + except ValueError: + return str(source) + + # ---------------------------------------------------------------- deploy @@ -71,12 +82,27 @@ def deploy(target, env, python_version, exclude): ' app = App("my-app")' ) - for found in apps: + if len(apps) > 1: + ui.deploy_plan( + [ + ( + found.name, + _app_source(found, project_root), + len(found.resources), + ) + for found in apps + ] + ) + + for index, found in enumerate(apps): + if index or len(apps) > 1: + ui.console.print() ui.set_name_width(list(found.resources)) ui.deploy_banner( found.name, env or found.env, [(h.spec.name, h.spec.kind.value) for h in found.resources.values()], + source=_app_source(found, project_root), ) events = ui.DeployEvents() with ui.Timer() as t: diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index 1d53d1faa..b56792e23 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -320,7 +320,9 @@ def run(): self._send(404, {"error": "not found"}) def log_message(self, format, *args): # noqa: A002 - stdlib signature - sys.stderr.write(f"[task-runner] {format % args}\n") + # request logging is noise in container logs (dev sessions + # stream them as the function's output) + pass def main() -> None: @@ -328,7 +330,6 @@ def main() -> None: sys.stderr.write("[task-runner] RUNPOD_TASK_TOKEN not set, exiting\n") sys.exit(1) server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) - sys.stderr.write(f"[task-runner] listening on :{PORT}\n") server.serve_forever() diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index 7900e75dc..787366f65 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -57,11 +57,16 @@ def test_worker_id_triggers_ready_and_stream(self): monitor = WorkerMonitor("ep1", "calc", sink) async def run(): - with patch.object( - monitor, "_stream_logs", return_value=asyncio.sleep(0) + with patch( + "runpod.apps.monitor.PodLogStream.attach" + ) as attach, patch( + "runpod.apps.monitor.PodLogStream.stop", + return_value=asyncio.sleep(0), ): monitor.on_status({"workerId": "w123", "status": "IN_PROGRESS"}) monitor.on_status({"workerId": "w123", "status": "IN_PROGRESS"}) + # one stream per worker, not per status payload + assert attach.call_count == 1 await monitor.stop() asyncio.run(run()) From a52577f8314757cf84ac0ff778801f05336586c5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:24:14 -0700 Subject: [PATCH 052/120] feat: task pod self-termination watchdog for abandoned clients --- runpod/runtimes/task/runner.py | 87 +++++++++++++++++++++++- tests/test_apps/test_task_runner_http.py | 33 +++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index b56792e23..4e70388dd 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -34,9 +34,88 @@ PORT = int(os.environ.get("RUNPOD_TASK_PORT", "8080")) TOKEN = os.environ.get("RUNPOD_TASK_TOKEN", "") +# watchdog: self-terminate when the client is clearly gone. RUNNING +# jobs are never killed (terminateAfter is the runaway backstop); +# NONE means the client died before submitting, DONE means the result +# sat uncollected. normal flows poll every ~2s, so these never fire +# for a live client. +IDLE_TIMEOUT = float(os.environ.get("RUNPOD_TASK_IDLE_TIMEOUT", "600")) +WATCHDOG_INTERVAL = 15.0 + # single background job slot for /submit + /result _job_lock = threading.Lock() _job_state = {"status": "NONE", "response": None} +_last_contact = {"ts": None} # set at server start + + +def _touch_contact(): + import time + + _last_contact["ts"] = time.time() + + +def _should_self_terminate(status, last_contact, now, idle_timeout): + """the watchdog decision: kill only provably-abandoned pods.""" + if status == "RUNNING": + return False + if last_contact is None: + return False + return (now - last_contact) > idle_timeout + + +def _terminate_self(): + """terminate this pod via the injected pod-scoped api key. + + every pod carries a RUNPOD_API_KEY scoped to itself; podTerminate + with it removes the pod entirely. exiting the process is the + fallback (stops the workload; terminateAfter finishes the job). + """ + import json as _json + import urllib.request + + pod_id = os.environ.get("RUNPOD_POD_ID") + api_key = os.environ.get("RUNPOD_API_KEY") + if pod_id and api_key: + try: + api_base = os.environ.get( + "RUNPOD_API_BASE_URL", "https://api.runpod.io" + ) + payload = _json.dumps( + { + "query": ( + "mutation podTerminate($input: PodTerminateInput!) " + "{ podTerminate(input: $input) }" + ), + "variables": {"input": {"podId": pod_id}}, + } + ).encode() + request = urllib.request.Request( + f"{api_base}/graphql", + data=payload, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + ) + urllib.request.urlopen(request, timeout=30) # noqa: S310 + sys.stderr.write("[task-runner] self-terminated (abandoned)\n") + except Exception: # noqa: BLE001 - fall through to process exit + pass + os._exit(0) + + +def _watchdog(): + import time + + while True: + time.sleep(WATCHDOG_INTERVAL) + if _should_self_terminate( + _job_state["status"], + _last_contact["ts"], + time.time(), + IDLE_TIMEOUT, + ): + _terminate_self() def _load_cloudpickle(install: bool = False): @@ -267,7 +346,11 @@ def _send(self, code, payload): def _authed(self): header = self.headers.get("Authorization", "") - return TOKEN and header == f"Bearer {TOKEN}" + authed = TOKEN and header == f"Bearer {TOKEN}" + if authed: + # any authenticated contact proves the client is alive + _touch_contact() + return authed def _read_request(self): length = int(self.headers.get("Content-Length", "0")) @@ -329,6 +412,8 @@ def main() -> None: if not TOKEN: sys.stderr.write("[task-runner] RUNPOD_TASK_TOKEN not set, exiting\n") sys.exit(1) + _touch_contact() + threading.Thread(target=_watchdog, daemon=True).start() server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) server.serve_forever() diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index 30ec46c6f..fea641ad3 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -210,3 +210,36 @@ def test_prints_reach_real_stdout_and_response(self, capfd): assert "live line" in response["stdout"] # and written through to the container's stdout for log streams assert "live line" in capfd.readouterr().out + + +class TestWatchdog: + def test_running_job_never_killed(self): + from runpod.runtimes.task.runner import _should_self_terminate + + assert not _should_self_terminate("RUNNING", 0, 10_000, 600) + + def test_abandoned_before_submit(self): + from runpod.runtimes.task.runner import _should_self_terminate + + assert _should_self_terminate("NONE", 0, 601, 600) + + def test_uncollected_result(self): + from runpod.runtimes.task.runner import _should_self_terminate + + assert _should_self_terminate("DONE", 0, 601, 600) + + def test_live_client_keeps_pod(self): + from runpod.runtimes.task.runner import _should_self_terminate + + # polls every ~2s: last contact is always recent + assert not _should_self_terminate("DONE", 599, 600, 600) + + def test_no_contact_recorded_yet(self): + from runpod.runtimes.task.runner import _should_self_terminate + + assert not _should_self_terminate("NONE", None, 10_000, 600) + + def test_authed_requests_touch_contact(self, server, monkeypatch): + monkeypatch.setattr(task_runner, "_last_contact", {"ts": None}) + _request(f"{server}/result") + assert task_runner._last_contact["ts"] is not None From 54919bf71ab3cfae77d6d693b449dc850e0d95b6 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:27:22 -0700 Subject: [PATCH 053/120] fix: keep task event sink attached across dev reloads --- runpod/apps/dev.py | 4 ++++ tests/test_apps/test_dev.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 0ddf08ded..6967ed9cd 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -301,6 +301,10 @@ async def refresh(self, apps: List[App]) -> None: provisioned, removed ones deleted.""" self.generation += 1 self.apps = apps + for app in apps: + # re-scanned apps are fresh instances; re-attach the sink + # so task lifecycle events keep rendering after reloads + app._dev_events = self.events desired: Dict[str, tuple] = {} for app in apps: diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 59d2d422b..45f25c3bd 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -440,3 +440,31 @@ def q(): payload = _endpoint_input(app, q.spec) assert payload["gpuIds"] == "ADA_24" + + +class TestRefreshKeepsTaskEvents: + async def test_dev_events_reattached_after_refresh(self): + sink = object() + app = App("evt-app") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.return_value = {"id": "ep-1"} + + session = DevSession([app], api=api, events=sink) + await session.start() + assert app._dev_events is sink + + # file change: discovery builds a brand-new app instance + app2 = App("evt-app") + + @app2.queue(name="q", cpu="cpu3c-1-2") + def q2(): + pass + + await session.refresh([app2]) + assert app2._dev_events is sink From 4dcf3295457f8bfddad08c7d404d31ff7d7a4ff2 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:40:49 -0700 Subject: [PATCH 054/120] fix: bind entrypoint exception before marshalling to the loop --- runpod/rp_cli/main.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 7dc3acd03..ed5886144 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -253,17 +253,23 @@ async def _run_entrypoint_cancellable(fn) -> None: loop = asyncio.get_running_loop() done: asyncio.Future = loop.create_future() + def _finish(exc: "BaseException | None") -> None: + if done.done(): + return + if exc is None: + done.set_result(None) + else: + done.set_exception(exc) + def _runner() -> None: try: run_entrypoint(fn) except BaseException as exc: # noqa: BLE001 - marshalled to the loop - loop.call_soon_threadsafe( - lambda: done.set_exception(exc) if not done.done() else None - ) + # bind the exception now: `as exc` is unbound once the + # except block exits, before the loop callback runs + loop.call_soon_threadsafe(_finish, exc) else: - loop.call_soon_threadsafe( - lambda: done.set_result(None) if not done.done() else None - ) + loop.call_soon_threadsafe(_finish, None) threading.Thread(target=_runner, daemon=True).start() await done From 51f3317cfff054d5037be0c11cd0ac4bda8a8cee Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:48:19 -0700 Subject: [PATCH 055/120] feat: resolve gpu shorthand strings to device names and pools --- runpod/apps/gpu.py | 61 ++++++++++++++++++++++++++++++++++-- runpod/apps/spec.py | 7 ++++- tests/test_apps/test_app.py | 62 +++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 3 deletions(-) diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py index ddddec307..5c7c12cb7 100644 --- a/runpod/apps/gpu.py +++ b/runpod/apps/gpu.py @@ -156,12 +156,69 @@ class CpuInstanceType(str, Enum): GpuLike = Union[GpuGroup, GpuType, str] +def resolve_gpu_string(value: str) -> List[str]: + """resolve a user-supplied gpu string to api-facing values. + + accepted forms, in match order: + - "any" + - pool ids ("ADA_24", case-insensitive) + - exact device names ("NVIDIA B200") + - enum-style names ("NVIDIA_B200") + - shorthand device fragments ("B200", "4090", "h100"): every + device whose name contains the fragment matches, so "H100" + selects all H100 variants + + unknown strings raise with the full list of valid options. + """ + text = value.strip() + if text.lower() == "any": + return [GpuGroup.ANY.value] + + upper = text.upper() + for group in GpuGroup.all(): + if group.value.upper() == upper: + return [group.value] + + for gpu_type in GpuType.all(): + if gpu_type.value.upper() == upper or gpu_type.name == upper: + return [gpu_type.value] + + fragment = upper.replace("_", " ") + matches = [ + gpu_type.value + for gpu_type in GpuType.all() + if fragment in gpu_type.value.upper() + ] + if matches: + return matches + + from .errors import InvalidResourceError + + pools = ", ".join(g.value for g in GpuGroup.all()) + devices = ", ".join(t.value for t in GpuType.all()) + raise InvalidResourceError( + f"unknown gpu '{value}'. use a pool id ({pools}), a device " + f"name ({devices}), or a fragment like '4090' or 'H100'" + ) + + def gpu_ids_value(gpu: Optional[List[str]]) -> str: """the gpuIds string for an endpoint payload. "any gpu" (no selection, or the ANY sentinel) means every pool id: - the api has no wildcard and rejects "any". + the api has no wildcard and rejects "any". device names map back + to their pool (endpoints select by pool, not device). """ if not gpu or any(str(g).lower() == "any" for g in gpu): return ",".join(g.value for g in GpuGroup.all()) - return ",".join(str(g) for g in gpu) + pools: List[str] = [] + for entry in gpu: + value = str(entry) + try: + pool = pool_for_gpu_type(GpuType(value)) + value = pool.value if pool is not None else value + except ValueError: + pass + if value not in pools: + pools.append(value) + return ",".join(pools) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 947b9275f..1055eb051 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -65,12 +65,17 @@ def normalize_gpu( return None if not isinstance(gpu, list): gpu = [gpu] + from .gpu import resolve_gpu_string + out: List[str] = [] for g in gpu: if isinstance(g, (GpuGroup, GpuType)): out.append(g.value) elif isinstance(g, str): - out.append(g) + # strings resolve strictly: pool ids and device names pass + # through, shorthands ("4090", "B200") expand, and typos + # fail at decoration time instead of at the api + out.extend(resolve_gpu_string(g)) else: raise InvalidResourceError( f"gpu must be a GpuGroup, GpuType, or string, got {type(g).__name__}" diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py index 236cd72b7..803b30738 100644 --- a/tests/test_apps/test_app.py +++ b/tests/test_apps/test_app.py @@ -260,3 +260,65 @@ def q(): "networkVolume": "my-volume", "env": {"KEY": "val"}, } + + +class TestGpuStringResolution: + def test_pool_id_passthrough(self): + from runpod.apps.spec import normalize_gpu + + assert normalize_gpu("ADA_24") == ["ADA_24"] + assert normalize_gpu("ada_24") == ["ADA_24"] + + def test_device_name_passthrough(self): + from runpod.apps.spec import normalize_gpu + + assert normalize_gpu("NVIDIA B200") == ["NVIDIA B200"] + + def test_enum_style_name(self): + from runpod.apps.spec import normalize_gpu + + assert normalize_gpu("NVIDIA_B200") == ["NVIDIA B200"] + + def test_shorthand_fragments(self): + from runpod.apps.spec import normalize_gpu + + assert normalize_gpu("B200") == ["NVIDIA B200"] + assert normalize_gpu("4090") == ["NVIDIA GeForce RTX 4090"] + assert normalize_gpu("5090") == ["NVIDIA GeForce RTX 5090"] + + def test_fragment_matches_all_variants(self): + from runpod.apps.spec import normalize_gpu + + matches = normalize_gpu("A100") + assert set(matches) == { + "NVIDIA A100 80GB PCIe", + "NVIDIA A100-SXM4-80GB", + } + + def test_any_sentinel(self): + from runpod.apps.spec import normalize_gpu + + assert normalize_gpu("any") == ["any"] + + def test_unknown_raises_at_decoration(self): + import pytest + + from runpod.apps.errors import InvalidResourceError + from runpod.apps.spec import normalize_gpu + + with pytest.raises(InvalidResourceError, match="unknown gpu 'B300'"): + normalize_gpu("B300") + + def test_gpu_ids_maps_devices_to_pools(self): + from runpod.apps.gpu import gpu_ids_value + + # endpoints select by pool; device names map back + assert gpu_ids_value(["NVIDIA B200"]) == "BLACKWELL_180" + assert gpu_ids_value(["NVIDIA GeForce RTX 4090"]) == "ADA_24" + # pool ids pass through untouched + assert gpu_ids_value(["ADA_24"]) == "ADA_24" + # duplicates collapse (two devices in the same pool) + assert ( + gpu_ids_value(["NVIDIA A100 80GB PCIe", "NVIDIA A100-SXM4-80GB"]) + == "AMPERE_80" + ) From 85f17dbbdc269644b875ccb418ed1f5f2ab07a63 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:54:35 -0700 Subject: [PATCH 056/120] feat: network volumes with app-wide placement solve --- runpod/__init__.py | 1 + runpod/apps/__init__.py | 2 + runpod/apps/api.py | 95 +++++++++++++ runpod/apps/app.py | 23 ++-- runpod/apps/deploy.py | 29 ++++ runpod/apps/dev.py | 19 +++ runpod/apps/placement.py | 197 +++++++++++++++++++++++++++ runpod/apps/spec.py | 6 +- runpod/apps/tasks.py | 29 +++- runpod/apps/volume.py | 216 ++++++++++++++++++++++++++++++ runpod/rp_cli/console.py | 5 + tests/test_apps/test_placement.py | 165 +++++++++++++++++++++++ tests/test_apps/test_tasks.py | 4 +- tests/test_apps/test_volume.py | 180 +++++++++++++++++++++++++ 14 files changed, 954 insertions(+), 17 deletions(-) create mode 100644 runpod/apps/placement.py create mode 100644 runpod/apps/volume.py create mode 100644 tests/test_apps/test_placement.py create mode 100644 tests/test_apps/test_volume.py diff --git a/runpod/__init__.py b/runpod/__init__.py index fc74ba060..fd8985af6 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -35,6 +35,7 @@ DataCenter, EndpointNotFound, Queue, + Volume, is_local, local_entrypoint, schedule, diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index f0284adac..83cd43079 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -26,6 +26,7 @@ from .schedule import schedule from .spec import ResourceKind, ResourceSpec, RouteSpec from .stubs import Api, Queue +from .volume import Volume __all__ = [ "Api", @@ -37,6 +38,7 @@ "EndpointNotFound", "FunctionHandle", "Queue", + "Volume", "RemoteExecutionError", "ResourceKind", "ResourceSpec", diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 5fae9ec6d..a11ae6ca7 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -173,6 +173,101 @@ async def create_environment(self, app_id: str, name: str) -> Dict[str, Any]: ) return data["createFlashEnvironment"] + # -- stock (placement) -- + + async def gpu_stock_status( + self, gpu_id: str, data_center_id: str, gpu_count: int = 1 + ) -> Optional[str]: + """stock signal for a gpu device in one datacenter.""" + query = """ + query GpuStock($gpuTypesInput: GpuTypeFilter, $lowestPriceInput: GpuLowestPriceInput) { + gpuTypes(input: $gpuTypesInput) { + lowestPrice(input: $lowestPriceInput) { stockStatus } + } + } + """ + data = await self._execute( + query, + { + "gpuTypesInput": {"ids": [gpu_id]}, + "lowestPriceInput": { + "dataCenterId": data_center_id, + "gpuCount": gpu_count, + "secureCloud": True, + "includeAiApi": True, + }, + }, + ) + gpu_types = data.get("gpuTypes") or [] + first = gpu_types[0] if gpu_types else {} + price = first.get("lowestPrice") if isinstance(first, dict) else None + return (price or {}).get("stockStatus") + + async def cpu_stock_status( + self, instance_id: str, data_center_id: str + ) -> Optional[str]: + """stock signal for a cpu flavor in one datacenter.""" + flavor = instance_id.split("-", 1)[0] + query = """ + query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { + cpuFlavors(input: $cpuFlavorInput) { + specifics(input: $specificsInput) { stockStatus } + } + } + """ + data = await self._execute( + query, + { + "cpuFlavorInput": {"id": flavor}, + "specificsInput": { + "dataCenterId": data_center_id, + "instanceId": instance_id, + }, + }, + ) + flavors = data.get("cpuFlavors") or [] + first = flavors[0] if flavors else {} + specifics = first.get("specifics") if isinstance(first, dict) else None + return (specifics or {}).get("stockStatus") + + # -- network volumes -- + + async def list_network_volumes(self) -> List[Dict[str, Any]]: + query = """ + query myVolumes { + myself { + networkVolumes { id name size dataCenterId } + } + } + """ + data = await self._execute(query) + return data["myself"].get("networkVolumes") or [] + + async def create_network_volume( + self, name: str, size: int, data_center_id: str + ) -> Dict[str, Any]: + mutation = """ + mutation createNetworkVolume($input: CreateNetworkVolumeInput!) { + createNetworkVolume(input: $input) { + id + name + size + dataCenterId + } + } + """ + data = await self._execute( + mutation, + { + "input": { + "name": name, + "size": size, + "dataCenterId": data_center_id, + } + }, + ) + return data["createNetworkVolume"] + # -- app / environment management -- async def list_apps(self) -> List[Dict[str, Any]]: diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 6dacee7b7..50961d579 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -291,16 +291,19 @@ def _datacenter_list( return [str(d) for d in datacenter] -def _volume_ref(volume: Any) -> Optional[str]: - """normalize a volume argument to a name-or-id string reference.""" +def _volume_ref(volume: Any) -> Optional[Any]: + """validate a volume argument: a Volume, or a name/id string. + + Volume objects pass through whole so creation config (size, + datacenter) survives to provision time. + """ if volume is None: return None - if isinstance(volume, str): + from .volume import Volume + + if isinstance(volume, (str, Volume)): return volume - name = getattr(volume, "name", None) or getattr(volume, "id", None) - if name is None: - raise InvalidResourceError( - f"volume must be a string name/id or an object with .name/.id, " - f"got {type(volume).__name__}" - ) - return str(name) + raise InvalidResourceError( + f"volume must be a runpod.Volume or a name/id string, " + f"got {type(volume).__name__}" + ) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 8c5648cc1..b323c7d8f 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -359,11 +359,15 @@ async def reconcile_endpoints( if h.spec.kind in (ResourceKind.QUEUE, ResourceKind.API) } + from .volume import VolumeResolver + + resolver = VolumeResolver(client) endpoints: Dict[str, str] = {} for name, handle in sorted(provisionable.items()): payload = _deployed_endpoint_input( app, handle.spec, environment["id"], build_id, python_version ) + await attach_endpoint_volumes(payload, handle.spec, resolver, app) if name in existing: payload["id"] = existing[name] result = await client.save_endpoint(payload) @@ -383,6 +387,31 @@ async def reconcile_endpoints( return endpoints +async def attach_endpoint_volumes( + payload: Dict[str, Any], spec, resolver, app +) -> None: + """resolve a resource's volumes onto an endpoint payload. + + endpoints may span regions: one volume per datacenter, locations + derived from the resolved volumes so lists cannot disagree. + """ + from .volume import specs_sharing_volume, volume_list + + volumes = volume_list(spec.volume) + if not volumes: + return + resolved = [] + for volume in volumes: + sharing = specs_sharing_volume([app], volume.name) or [spec] + resolved.append(await resolver.resolve(volume, sharing)) + payload["networkVolumeIds"] = [ + {"networkVolumeId": r["id"]} for r in resolved + ] + payload["locations"] = ",".join( + dict.fromkeys(r["dataCenterId"] for r in resolved) + ) + + def _phase(events, name: str, detail: str = "") -> None: handler = getattr(events, "phase", None) if handler is not None: diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 6967ed9cd..ac1bb3a3b 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -226,12 +226,29 @@ def __init__( self._endpoints: Dict[str, str] = {} # endpoint name -> comparable payload, for refresh diffing self._payloads: Dict[str, Dict] = {} + # volumes resolve once per session (placement is stable) + self._volume_resolver = None def _emit(self, event: str, *args) -> None: handler = getattr(self.events, event, None) if handler is not None: handler(*args) + async def _attach_volumes(self, payload: Dict, spec, app) -> None: + """resolve the resource's volumes onto a dev endpoint payload.""" + from .deploy import attach_endpoint_volumes + from .volume import VolumeResolver + + if not spec.volume: + return + if self._volume_resolver is None: + self._volume_resolver = VolumeResolver( + self.api, events=self.events + ) + await attach_endpoint_volumes( + payload, spec, self._volume_resolver, app + ) + @property def _endpoint_ids(self) -> List[str]: return list(self._endpoints.values()) @@ -260,6 +277,7 @@ async def start(self) -> None: spec = handle.spec name = dev_endpoint_name(app.name, spec.name) payload = _endpoint_input(app, spec, self.generation) + await self._attach_volumes(payload, spec, app) hardware = ",".join(spec.cpu or spec.gpu or ["any"]) found = existing.get(name) @@ -327,6 +345,7 @@ async def refresh(self, apps: List[App]) -> None: # update survivors (config + generation bump) and create additions for name, (app, handle) in desired.items(): payload = _endpoint_input(app, handle.spec, self.generation) + await self._attach_volumes(payload, handle.spec, app) existing_id = self._endpoints.get(name) comparable = _comparable(payload) previous = self._payloads.get(name) diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py new file mode 100644 index 000000000..db5327904 --- /dev/null +++ b/runpod/apps/placement.py @@ -0,0 +1,197 @@ +"""datacenter placement: solving where volumes and resources can live. + +a network volume pins everything attached to it to one datacenter, so +placement is a constraint solve over the whole app: each resource's +schedulable datacenters (hardware stock ∩ user pins ∩ storage support) +intersected per volume, ranked maximin by stock so the chosen DC is +the one where the most-constrained resource has the best availability. +""" + +import asyncio +import logging +from typing import Dict, Iterable, List, Optional, Set, Tuple + +from .datacenter import DataCenter +from .errors import AppError +from .gpu import GpuGroup + +log = logging.getLogger(__name__) + +# stock signal ranking; unknown/none scores zero +_STOCK_SCORE = {"HIGH": 3, "MEDIUM": 2, "LOW": 1} + + +class PlacementError(AppError): + pass + + +def _score(status: Optional[str]) -> int: + if not isinstance(status, str): + return 0 + return _STOCK_SCORE.get(status.strip().upper(), 0) + + +def _hardware_keys(spec) -> List[Tuple[str, str]]: + """(kind, id) stock lookup keys for a resource's hardware. + + gpu entries may be pool ids or device names; pools expand to their + device names since the stock api takes devices. + """ + if spec.is_cpu: + return [("cpu", c) for c in spec.cpu or []] + gpu = spec.gpu + if not gpu or any(str(g).lower() == "any" for g in gpu): + return [("gpu", "*")] + keys: List[Tuple[str, str]] = [] + for entry in gpu: + try: + for device in GpuGroup(str(entry)).device_names(): + keys.append(("gpu", device)) + except ValueError: + keys.append(("gpu", str(entry))) + return keys + + +class StockMap: + """per-(hardware, datacenter) stock signals, fetched lazily in bulk.""" + + def __init__(self, api=None): + self._api = api + self._gpu: Dict[Tuple[str, str], int] = {} + self._cpu: Dict[Tuple[str, str], int] = {} + self._fetched_gpu: Set[str] = set() + self._fetched_cpu: Set[str] = set() + + async def _client(self): + if self._api is None: + from .api import AppsApiClient + + self._api = AppsApiClient() + return self._api + + async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: + """populate stock for the given hardware keys across all DCs.""" + gpu_ids = { + k[1] + for k in keys + if k[0] == "gpu" and k[1] != "*" and k[1] not in self._fetched_gpu + } + cpu_ids = { + k[1] for k in keys if k[0] == "cpu" and k[1] not in self._fetched_cpu + } + client = await self._client() + jobs = [] + for gpu_id in gpu_ids: + self._fetched_gpu.add(gpu_id) + for dc in DataCenter.all(): + jobs.append(self._fetch_gpu(client, gpu_id, dc.value)) + for cpu_id in cpu_ids: + self._fetched_cpu.add(cpu_id) + for dc in DataCenter.all(): + jobs.append(self._fetch_cpu(client, cpu_id, dc.value)) + if jobs: + await asyncio.gather(*jobs) + + async def _fetch_gpu(self, client, gpu_id: str, dc: str) -> None: + try: + status = await client.gpu_stock_status(gpu_id, dc) + except Exception: # noqa: BLE001 - stock is advisory + log.debug("gpu stock query failed for %s@%s", gpu_id, dc, exc_info=True) + status = None + self._gpu[(gpu_id, dc)] = _score(status) + + async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: + try: + status = await client.cpu_stock_status(instance_id, dc) + except Exception: # noqa: BLE001 - stock is advisory + log.debug("cpu stock query failed for %s@%s", instance_id, dc, exc_info=True) + status = None + self._cpu[(instance_id, dc)] = _score(status) + + def score(self, key: Tuple[str, str], dc: str) -> int: + kind, hw = key + if kind == "gpu": + if hw == "*": + # any gpu: best signal among fetched devices, else assume ok + scores = [ + s for (g, d), s in self._gpu.items() if d == dc + ] + return max(scores, default=1) + return self._gpu.get((hw, dc), 0) + return self._cpu.get((hw, dc), 0) + + +def candidates(spec, stock: StockMap) -> Set[str]: + """datacenters where a resource is schedulable. + + hardware needs stock in the DC (any of the resource's acceptable + devices/flavors), intersected with an explicit datacenter pin. + """ + allowed = {dc.value for dc in DataCenter.all()} + if spec.datacenter: + allowed &= {str(d) for d in spec.datacenter} + + keys = _hardware_keys(spec) + if not keys: + return allowed + viable = set() + for dc in allowed: + if any(stock.score(key, dc) > 0 for key in keys): + viable.add(dc) + return viable + + +def _resource_best_score(spec, stock: StockMap, dc: str) -> int: + keys = _hardware_keys(spec) + if not keys: + return 1 + return max(stock.score(key, dc) for key in keys) + + +def solve_placement( + specs: List, + stock: StockMap, + *, + volume_name: str, + existing_dc: Optional[str] = None, +) -> str: + """pick the datacenter for one volume given every resource using it. + + an existing volume's DC is a hard constraint (verified schedulable); + a new volume lands in the intersection of every resource's candidate + set, ranked maximin: the DC where the most-constrained resource has + the best stock. + """ + per_resource = {spec.name: candidates(spec, stock) for spec in specs} + + if existing_dc is not None: + blocked = [ + name for name, dcs in per_resource.items() if existing_dc not in dcs + ] + if blocked: + raise PlacementError( + f"volume '{volume_name}' lives in {existing_dc}, but " + f"{', '.join(blocked)} cannot schedule there " + f"(no hardware stock or conflicting datacenter pin)" + ) + return existing_dc + + shared = set.intersection(*per_resource.values()) if per_resource else set() + if not shared: + lines = [ + f" {name:<12} schedulable in: {', '.join(sorted(dcs)) or '(nowhere)'}" + for name, dcs in per_resource.items() + ] + raise PlacementError( + f"cannot place volume '{volume_name}': no datacenter can host " + f"every resource using it\n" + "\n".join(lines) + "\n" + f"use separate volumes or compatible hardware" + ) + + # maximin: rank each DC by the worst resource's stock there, + # tiebreak by the aggregate + def rank(dc: str) -> Tuple[int, int]: + scores = [_resource_best_score(spec, stock, dc) for spec in specs] + return (min(scores), sum(scores)) + + return max(sorted(shared), key=rank) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 1055eb051..de8441d95 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -111,7 +111,7 @@ class ResourceSpec: idle_timeout: int = 60 dependencies: Optional[List[str]] = None system_dependencies: Optional[List[str]] = None - volume: Optional[str] = None + volume: Optional[Any] = None env: Optional[Dict[str, str]] = None datacenter: Optional[List[str]] = None image: Optional[str] = None @@ -150,7 +150,9 @@ def to_manifest(self) -> Dict[str, Any]: if self.system_dependencies: data["systemDependencies"] = self.system_dependencies if self.volume: - data["networkVolume"] = self.volume + data["networkVolume"] = getattr( + self.volume, "name", None + ) or str(self.volume) if self.env: data["env"] = self.env if self.datacenter: diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 01f6fa30a..82024b08a 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -128,8 +128,6 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] pool = CPU3_DATACENTERS if spec.is_cpu else DataCenter.all() pod["dataCenterIds"] = [dc.value for dc in pool] - if spec.volume: - pod["networkVolumeId"] = spec.volume if spec.is_cpu: pod["instanceIds"] = spec.cpu else: @@ -152,13 +150,36 @@ def _headers(self) -> Dict[str, str]: return {"Authorization": f"Bearer {self.token}"} async def start(self) -> None: + pod = _pod_input(self.spec, self.token, self.spec.name) + if self.spec.volume: + pod = await self._attach_volume(pod) result = await self.api.deploy_task_pod( - _pod_input(self.spec, self.token, self.spec.name), - is_cpu=self.spec.is_cpu, + pod, is_cpu=self.spec.is_cpu ) self.pod_id = result["id"] log.info("task pod %s deployed for %s", self.pod_id, self.spec.name) + async def _attach_volume(self, pod: Dict[str, Any]) -> Dict[str, Any]: + """resolve the task's volume and pin the pod to its datacenter.""" + from .volume import VolumeError, VolumeResolver, volume_list + + volumes = volume_list(self.spec.volume) + if len(volumes) > 1: + raise VolumeError( + f"task '{self.spec.name}': pods mount exactly one volume " + f"({len(volumes)} given)" + ) + resolver = VolumeResolver(self.api) + resolved = await resolver.resolve(volumes[0], [self.spec]) + from .volume import POD_MOUNT_PATH + + pod["networkVolumeId"] = resolved["id"] + # the host bind-mounts to this target; empty means a broken + # mount config and the container never starts + pod["volumeMountPath"] = str(POD_MOUNT_PATH) + pod["dataCenterIds"] = [resolved["dataCenterId"]] + return pod + async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: """poll the runner's /ping through the pod proxy until it answers.""" url = f"{_proxy_url(self.pod_id)}/ping" diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py new file mode 100644 index 000000000..4927b6904 --- /dev/null +++ b/runpod/apps/volume.py @@ -0,0 +1,216 @@ +"""network volumes: durable storage shared across resources. + +a Volume is a lazy reference by name, resolved (and created when +missing) at provision time. workers see the volume at /runpod-volume. + + models = runpod.Volume("models") # create if missing, 50GB + models = runpod.Volume("models", size=100) + + @app.task(gpu="4090", volume=models) + def train(): + torch.save(sd, models.path / "model.pt") + +placement: a volume lives in exactly one datacenter, so everything +attached to it must schedule there. resolution runs the placement +solve over every resource sharing the volume (see runpod.apps.placement). +tasks/pods take one volume; endpoints may take several (one per +datacenter, locations derived from the volumes). +""" + +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .errors import AppError + +log = logging.getLogger(__name__) + +DEFAULT_SIZE_GB = 50 +# platform mount conventions: pods bind network volumes at /workspace, +# serverless endpoint workers at /runpod-volume +POD_MOUNT_PATH = Path("/workspace") +ENDPOINT_MOUNT_PATH = Path("/runpod-volume") + + +class VolumeError(AppError): + pass + + +class Volume: + """a named network volume reference, created on first use.""" + + def __init__( + self, + name: str, + *, + size: int = DEFAULT_SIZE_GB, + datacenter: Optional[str] = None, + create: bool = True, + ): + if not name or not isinstance(name, str): + raise VolumeError("volume name must be a non-empty string") + self.name = name + self.size = size + self.datacenter = datacenter + self.create = create + + @property + def path(self) -> Path: + """where the volume appears in the current worker. + + context-sensitive because the platform mounts differ: task + pods use /workspace, endpoint workers /runpod-volume. resolved + at access time inside the worker, so the same function body + works from either. + """ + import os + + if os.environ.get("RUNPOD_ENDPOINT_ID"): + return ENDPOINT_MOUNT_PATH + return POD_MOUNT_PATH + + def __repr__(self) -> str: + return f"" + + +def _as_volume(ref: Any) -> Volume: + if isinstance(ref, Volume): + return ref + if isinstance(ref, str): + return Volume(ref) + raise VolumeError( + f"volume must be a runpod.Volume or name/id string, " + f"got {type(ref).__name__}" + ) + + +def volume_list(spec_volume: Any) -> List[Volume]: + """normalize a spec's volume field to a list of Volume refs.""" + if spec_volume is None: + return [] + if isinstance(spec_volume, (list, tuple)): + return [_as_volume(v) for v in spec_volume] + return [_as_volume(spec_volume)] + + +class VolumeResolver: + """resolves every volume in an app once per provision run. + + resolution: find by id or name; when missing, run the placement + solve over all resources sharing the volume and create it in the + chosen datacenter. results cache by name so each volume resolves + exactly once regardless of how many resources reference it. + """ + + def __init__(self, api=None, events: Optional[object] = None): + self._api = api + self.events = events + self._resolved: Dict[str, Dict[str, Any]] = {} + self._stock = None + + async def _client(self): + if self._api is None: + from .api import AppsApiClient + + self._api = AppsApiClient() + return self._api + + async def resolve( + self, volume: Volume, specs: List + ) -> Dict[str, Any]: + """resolve one volume to {'id', 'dataCenterId'}. + + specs are all resource specs that attach this volume; they + drive placement for creation and validate an existing DC. + """ + cached = self._resolved.get(volume.name) + if cached is not None: + return cached + + from .monitor import emit + from .placement import PlacementError, StockMap, solve_placement + + client = await self._client() + existing = await client.list_network_volumes() + + record = None + for entry in existing: + if entry["id"] == volume.name: + record = entry + break + if record is None: + matches = [e for e in existing if e.get("name") == volume.name] + if len(matches) > 1: + raise VolumeError( + f"multiple volumes named '{volume.name}' " + f"({', '.join(m['id'] for m in matches)}); reference by id" + ) + record = matches[0] if matches else None + + if self._stock is None: + self._stock = StockMap(client) + from .placement import _hardware_keys + + keys = [k for spec in specs for k in _hardware_keys(spec)] + await self._stock.fetch(keys) + + if record is not None: + # existing volume: its DC is a hard constraint + dc = solve_placement( + specs, + self._stock, + volume_name=volume.name, + existing_dc=record["dataCenterId"], + ) + resolved = {"id": record["id"], "dataCenterId": dc} + self._resolved[volume.name] = resolved + return resolved + + if not volume.create: + raise VolumeError( + f"volume '{volume.name}' not found and create=False" + ) + + if volume.datacenter: + dc = solve_placement( + specs, + self._stock, + volume_name=volume.name, + existing_dc=volume.datacenter, + ) + else: + dc = solve_placement( + specs, self._stock, volume_name=volume.name + ) + + created = await client.create_network_volume( + name=volume.name, size=volume.size, data_center_id=dc + ) + emit( + self.events, + "volume_created", + volume.name, + volume.size, + dc, + ) + log.info( + "created volume %s (%s, %dGB, %s)", + volume.name, + created["id"], + volume.size, + dc, + ) + resolved = {"id": created["id"], "dataCenterId": dc} + self._resolved[volume.name] = resolved + return resolved + + +def specs_sharing_volume(apps: List, name: str) -> List: + """every resource spec (across apps) attaching the named volume.""" + out = [] + for app in apps: + for handle in app.resources.values(): + for ref in volume_list(handle.spec.volume): + if ref.name == name: + out.append(handle.spec) + return out diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 9b8272b99..7f37180b4 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -408,6 +408,11 @@ def worker_status(self, name: str, counts: Dict[str, int]) -> None: def task_status(self, name: str, detail: str) -> None: console.print(f" [dim]○ {name}() {detail}[/dim]") + def volume_created(self, name: str, size: int, dc: str) -> None: + console.print( + f" [ok]+[/ok] [white]volume {name}[/white] [dim]{size}GB · {dc}[/dim]" + ) + def worker_ready(self, name: str, worker_id: str) -> None: console.print( f" [accent]●[/accent] [white]{name}()[/white] [dim]running on " diff --git a/tests/test_apps/test_placement.py b/tests/test_apps/test_placement.py new file mode 100644 index 000000000..db4d64ff8 --- /dev/null +++ b/tests/test_apps/test_placement.py @@ -0,0 +1,165 @@ +"""placement solve: candidates, intersections, maximin ranking.""" + +import pytest + +from runpod.apps.placement import ( + PlacementError, + StockMap, + candidates, + solve_placement, +) +from runpod.apps.spec import ResourceKind, ResourceSpec + + +def _spec(name="r", gpu=None, cpu=None, datacenter=None): + return ResourceSpec( + kind=ResourceKind.TASK, + name=name, + gpu=gpu, + cpu=cpu, + datacenter=datacenter, + ) + + +def _stock(gpu=None, cpu=None): + stock = StockMap(api=object()) + stock._gpu = gpu or {} + stock._cpu = cpu or {} + return stock + + +class TestCandidates: + def test_gpu_stock_filters_dcs(self): + stock = _stock( + gpu={ + ("NVIDIA GeForce RTX 4090", "EU-RO-1"): 3, + ("NVIDIA GeForce RTX 4090", "US-KS-2"): 0, + } + ) + dcs = candidates(_spec(gpu=["NVIDIA GeForce RTX 4090"]), stock) + assert "EU-RO-1" in dcs + assert "US-KS-2" not in dcs + + def test_pool_id_expands_to_devices(self): + stock = _stock(gpu={("NVIDIA GeForce RTX 4090", "EU-RO-1"): 2}) + dcs = candidates(_spec(gpu=["ADA_24"]), stock) + assert "EU-RO-1" in dcs + + def test_datacenter_pin_intersects(self): + stock = _stock( + gpu={ + ("NVIDIA GeForce RTX 4090", "EU-RO-1"): 3, + ("NVIDIA GeForce RTX 4090", "US-KS-2"): 3, + } + ) + dcs = candidates( + _spec(gpu=["NVIDIA GeForce RTX 4090"], datacenter=["US-KS-2"]), + stock, + ) + assert dcs == {"US-KS-2"} + + def test_cpu_stock(self): + stock = _stock(cpu={("cpu5c-2-4", "EU-RO-1"): 2}) + dcs = candidates(_spec(cpu=["cpu5c-2-4"]), stock) + assert dcs == {"EU-RO-1"} + + def test_any_gpu_allows_everywhere(self): + stock = _stock(gpu={("NVIDIA GeForce RTX 4090", "EU-RO-1"): 1}) + dcs = candidates(_spec(gpu=None), stock) + assert "EU-RO-1" in dcs + + +class TestSolvePlacement: + def test_intersection_picks_shared_dc(self): + stock = _stock( + gpu={ + ("NVIDIA GeForce RTX 4090", "EU-RO-1"): 3, + ("NVIDIA GeForce RTX 4090", "US-KS-2"): 3, + ("NVIDIA H200", "EU-RO-1"): 2, + } + ) + dc = solve_placement( + [ + _spec("train", gpu=["NVIDIA H200"]), + _spec("infer", gpu=["NVIDIA GeForCE RTX 4090".replace("CE", "ce")]), + ], + stock, + volume_name="models", + ) + assert dc == "EU-RO-1" + + def test_disjoint_hardware_errors_with_details(self): + stock = _stock( + gpu={ + ("NVIDIA H200", "EU-RO-1"): 3, + ("NVIDIA B200", "US-KS-2"): 3, + } + ) + with pytest.raises(PlacementError, match="no datacenter can host"): + solve_placement( + [ + _spec("train", gpu=["NVIDIA H200"]), + _spec("eval", gpu=["NVIDIA B200"]), + ], + stock, + volume_name="models", + ) + + def test_existing_dc_is_hard_constraint(self): + stock = _stock(gpu={("NVIDIA H200", "EU-RO-1"): 3}) + dc = solve_placement( + [_spec("train", gpu=["NVIDIA H200"])], + stock, + volume_name="models", + existing_dc="EU-RO-1", + ) + assert dc == "EU-RO-1" + + def test_existing_dc_unschedulable_errors(self): + stock = _stock(gpu={("NVIDIA H200", "EU-RO-1"): 3}) + with pytest.raises(PlacementError, match="lives in US-KS-2"): + solve_placement( + [_spec("train", gpu=["NVIDIA H200"])], + stock, + volume_name="models", + existing_dc="US-KS-2", + ) + + def test_maximin_prefers_worst_case_stock(self): + # both DCs host both resources; EU is (3, 1), US is (2, 2): + # maximin picks US because its worst resource is better off + stock = _stock( + gpu={ + ("NVIDIA H200", "EU-RO-1"): 3, + ("NVIDIA H200", "US-KS-2"): 2, + ("NVIDIA B200", "EU-RO-1"): 1, + ("NVIDIA B200", "US-KS-2"): 2, + } + ) + dc = solve_placement( + [ + _spec("a", gpu=["NVIDIA H200"]), + _spec("b", gpu=["NVIDIA B200"]), + ], + stock, + volume_name="v", + ) + assert dc == "US-KS-2" + + def test_mixed_cpu_gpu_sharing(self): + stock = _stock( + gpu={ + ("NVIDIA GeForce RTX 4090", "EU-RO-1"): 3, + ("NVIDIA GeForce RTX 4090", "US-KS-2"): 3, + }, + cpu={("cpu5c-2-4", "EU-RO-1"): 2}, + ) + dc = solve_placement( + [ + _spec("gpures", gpu=["NVIDIA GeForce RTX 4090"]), + _spec("cpures", cpu=["cpu5c-2-4"]), + ], + stock, + volume_name="shared", + ) + assert dc == "EU-RO-1" diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 56b3ca1dd..cf449d7c0 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -163,7 +163,9 @@ def test_custom_image_bootstraps_runner(self): "t", ) assert pod["imageName"] == "my/image:1" - assert pod["networkVolumeId"] == "vol-1" + # volumes resolve at TaskExecution.start (placement solve), + # not in the static pod input + assert "networkVolumeId" not in pod # custom images get the env-injection fallback env = {e["key"]: e["value"] for e in pod["env"]} assert "RUNPOD_TASK_RUNNER_B64" in env diff --git a/tests/test_apps/test_volume.py b/tests/test_apps/test_volume.py new file mode 100644 index 000000000..bda7cc445 --- /dev/null +++ b/tests/test_apps/test_volume.py @@ -0,0 +1,180 @@ +"""volume references and provision-time resolution.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from runpod.apps.spec import ResourceKind, ResourceSpec +from runpod.apps.volume import ( + Volume, + VolumeError, + VolumeResolver, + volume_list, +) + + +def _spec(name="r", gpu=None, cpu=None): + return ResourceSpec( + kind=ResourceKind.TASK, name=name, gpu=gpu, cpu=cpu + ) + + +def _api(volumes=None, created=None): + api = AsyncMock() + api.list_network_volumes.return_value = volumes or [] + api.create_network_volume.return_value = created or { + "id": "nv-new", + "name": "models", + "size": 50, + "dataCenterId": "EU-RO-1", + } + # stock queries: everything available everywhere + api.gpu_stock_status.return_value = "High" + api.cpu_stock_status.return_value = "High" + return api + + +class TestVolumeRef: + def test_path_follows_context(self, monkeypatch): + from pathlib import Path + + # task pods mount at /workspace + monkeypatch.delenv("RUNPOD_ENDPOINT_ID", raising=False) + assert Volume("models").path == Path("/workspace") + # endpoint workers mount at /runpod-volume + monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") + assert Volume("models").path == Path("/runpod-volume") + + def test_empty_name_raises(self): + with pytest.raises(VolumeError): + Volume("") + + def test_volume_list_normalizes(self): + vols = volume_list([Volume("a"), "b"]) + assert [v.name for v in vols] == ["a", "b"] + assert volume_list(None) == [] + assert [v.name for v in volume_list("single")] == ["single"] + + +class TestVolumeResolver: + def test_existing_by_name(self): + api = _api( + volumes=[ + {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + ] + ) + resolver = VolumeResolver(api) + resolved = asyncio.run( + resolver.resolve(Volume("models"), [_spec(gpu=None)]) + ) + assert resolved == {"id": "nv-1", "dataCenterId": "EU-RO-1"} + api.create_network_volume.assert_not_awaited() + + def test_existing_by_id(self): + api = _api( + volumes=[ + {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + ] + ) + resolver = VolumeResolver(api) + resolved = asyncio.run( + resolver.resolve(Volume("nv-1"), [_spec(gpu=None)]) + ) + assert resolved["id"] == "nv-1" + + def test_missing_creates_with_placement(self): + api = _api() + resolver = VolumeResolver(api) + resolved = asyncio.run( + resolver.resolve(Volume("models"), [_spec(gpu=None)]) + ) + assert resolved["id"] == "nv-new" + api.create_network_volume.assert_awaited_once() + + def test_missing_no_create_raises(self): + api = _api() + resolver = VolumeResolver(api) + with pytest.raises(VolumeError, match="create=False"): + asyncio.run( + resolver.resolve( + Volume("models", create=False), [_spec(gpu=None)] + ) + ) + + def test_duplicate_names_raise(self): + api = _api( + volumes=[ + {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"}, + {"id": "nv-2", "name": "models", "size": 50, "dataCenterId": "US-KS-2"}, + ] + ) + resolver = VolumeResolver(api) + with pytest.raises(VolumeError, match="reference by id"): + asyncio.run( + resolver.resolve(Volume("models"), [_spec(gpu=None)]) + ) + + def test_resolution_cached_per_name(self): + api = _api( + volumes=[ + {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + ] + ) + resolver = VolumeResolver(api) + + async def run(): + await resolver.resolve(Volume("models"), [_spec(gpu=None)]) + await resolver.resolve(Volume("models"), [_spec(gpu=None)]) + + asyncio.run(run()) + assert api.list_network_volumes.await_count == 1 + + def test_created_event_emitted(self): + events = [] + + class Sink: + def volume_created(self, name, size, dc): + events.append((name, size, dc)) + + api = _api() + resolver = VolumeResolver(api, events=Sink()) + asyncio.run(resolver.resolve(Volume("models"), [_spec(gpu=None)])) + assert len(events) == 1 + name, size, dc = events[0] + assert (name, size) == ("models", 50) + assert dc # placement picks a concrete datacenter + + +class TestTaskVolume: + def test_single_volume_only(self): + from runpod.apps.tasks import TaskExecution + + spec = ResourceSpec( + kind=ResourceKind.TASK, + name="t", + cpu=["cpu3c-1-2"], + volume=[Volume("a"), Volume("b")], + ) + execution = TaskExecution(spec, api=_api()) + with pytest.raises(VolumeError, match="exactly one volume"): + asyncio.run(execution._attach_volume({})) + + def test_pod_pins_to_volume_dc(self): + from runpod.apps.tasks import TaskExecution + + api = _api( + volumes=[ + {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + ] + ) + spec = ResourceSpec( + kind=ResourceKind.TASK, + name="t", + cpu=["cpu3c-1-2"], + volume=Volume("models"), + ) + execution = TaskExecution(spec, api=api) + pod = asyncio.run(execution._attach_volume({})) + assert pod["networkVolumeId"] == "nv-1" + assert pod["dataCenterIds"] == ["EU-RO-1"] From 5b46d61070cdcb2940d761aba04f7dae337a1dd3 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:42:26 -0700 Subject: [PATCH 057/120] feat: platform secrets with env rendering and rp secret cli --- runpod/__init__.py | 1 + runpod/apps/__init__.py | 2 + runpod/apps/api.py | 42 +++++++++++++++ runpod/apps/app.py | 6 +-- runpod/apps/deploy.py | 15 +++++- runpod/apps/dev.py | 20 ++++++- runpod/apps/secret.py | 95 ++++++++++++++++++++++++++++++++++ runpod/apps/spec.py | 6 ++- runpod/apps/tasks.py | 4 +- runpod/rp_cli/main.py | 78 ++++++++++++++++++++++++++++ tests/test_apps/test_secret.py | 89 +++++++++++++++++++++++++++++++ 11 files changed, 350 insertions(+), 8 deletions(-) create mode 100644 runpod/apps/secret.py create mode 100644 tests/test_apps/test_secret.py diff --git a/runpod/__init__.py b/runpod/__init__.py index fd8985af6..871d79b47 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -35,6 +35,7 @@ DataCenter, EndpointNotFound, Queue, + Secret, Volume, is_local, local_entrypoint, diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index 83cd43079..4caa47ab7 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -26,6 +26,7 @@ from .schedule import schedule from .spec import ResourceKind, ResourceSpec, RouteSpec from .stubs import Api, Queue +from .secret import Secret from .volume import Volume __all__ = [ @@ -38,6 +39,7 @@ "EndpointNotFound", "FunctionHandle", "Queue", + "Secret", "Volume", "RemoteExecutionError", "ResourceKind", diff --git a/runpod/apps/api.py b/runpod/apps/api.py index a11ae6ca7..9f58c0736 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -268,6 +268,48 @@ async def create_network_volume( ) return data["createNetworkVolume"] + # -- secrets -- + + async def list_secrets(self) -> List[Dict[str, Any]]: + query = """ + query mySecrets { + myself { + secrets { id name description createdAt } + } + } + """ + data = await self._execute(query) + return data["myself"].get("secrets") or [] + + async def create_secret( + self, name: str, value: str, description: str = "" + ) -> Dict[str, Any]: + mutation = """ + mutation secretCreate($input: SecretCreateInput!) { + secretCreate(input: $input) { id name } + } + """ + data = await self._execute( + mutation, + { + "input": { + "name": name, + "value": value, + "description": description, + } + }, + ) + return data["secretCreate"] + + async def delete_secret(self, secret_id: str) -> bool: + mutation = """ + mutation secretDelete($id: ID!) { + secretDelete(id: $id) + } + """ + data = await self._execute(mutation, {"id": secret_id}) + return "secretDelete" in data + # -- app / environment management -- async def list_apps(self) -> List[Dict[str, Any]]: diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 50961d579..1732807f4 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -89,7 +89,7 @@ def queue( dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, volume: Optional[Any] = None, - env: Optional[Dict[str, str]] = None, + env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, ) -> Callable[[Callable], FunctionHandle]: @@ -132,7 +132,7 @@ def task( dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, volume: Optional[Any] = None, - env: Optional[Dict[str, str]] = None, + env: Optional[Dict[str, Any]] = None, image: Optional[str] = None, datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Callable], FunctionHandle]: @@ -175,7 +175,7 @@ def api( dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, volume: Optional[Any] = None, - env: Optional[Dict[str, str]] = None, + env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, ) -> Callable[[Any], ApiHandle]: diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index b323c7d8f..f81758538 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -246,11 +246,13 @@ def _deployed_endpoint_input( from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS, DataCenter from .images import image_for_spec + from .secret import render_env + template_env = { "FLASH_RESOURCE_NAME": spec.name, # version-triggering: a new build recreates all workers "RUNPOD_BUILD_ID": build_id, - **(spec.env or {}), + **render_env(spec.env), } # cross-resource calls from inside a worker go through the sentinel, @@ -432,6 +434,17 @@ async def deploy_app( client = api or AppsApiClient() env_name = env_name or app.env + # fail fast on unresolvable secret references (workers would boot + # with the literal template string otherwise) + from .secret import secret_names, validate_secrets + + referenced = [ + name + for handle in app.resources.values() + for name in secret_names(handle.spec.env) + ] + await validate_secrets(referenced, api=client) + _phase(events, "vendor", f"python {python_version}") build: BuildResult = build_environment( app, diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index ac1bb3a3b..6e34c09a6 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -98,6 +98,12 @@ def _bootstrap_docker_args() -> str: return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") +def _render_env(env): + from .secret import render_env + + return render_env(env) + + def _client_api_key() -> str: from .targets import _api_key @@ -157,7 +163,10 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: else [] ), {"key": "RUNPOD_DEV_APP", "value": app.name}, - *({"key": k, "value": v} for k, v in (spec.env or {}).items()), + *( + {"key": k, "value": v} + for k, v in _render_env(spec.env).items() + ), ], }, } @@ -267,6 +276,15 @@ async def start(self) -> None: """provision (or adopt) a live endpoint per queue/api resource and register the targets on each app.""" self._emit("session_starting") + from .secret import secret_names, validate_secrets + + referenced = [ + name + for app in self.apps + for handle in app.resources.values() + for name in secret_names(handle.spec.env) + ] + await validate_secrets(referenced, api=self.api) for app in self.apps: # task targets read the sink off the app at resolve time app._dev_events = self.events diff --git a/runpod/apps/secret.py b/runpod/apps/secret.py new file mode 100644 index 000000000..0c6399955 --- /dev/null +++ b/runpod/apps/secret.py @@ -0,0 +1,95 @@ +"""platform secrets: encrypted values injected into worker env vars. + +a Secret marks an env value for reference-syntax rendering; the +platform decrypts it when the worker boots. values never travel +through the sdk. + + @app.queue(env={"HF_TOKEN": runpod.Secret("hf-token")}) + def download(): ... + +the worker sees a plain HF_TOKEN env var with the decrypted value. +create and manage secrets with `rp secret add/list/rm`. +""" + +import re +from typing import Any, Dict, List, Optional + +from .errors import AppError + +# the platform's env reference syntax (decrypted server-side at boot) +_REFERENCE_TEMPLATE = "{{{{ RUNPOD_SECRET_{name} }}}}" +_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") + + +class SecretError(AppError): + pass + + +class Secret: + """a reference to a platform secret by name.""" + + def __init__(self, name: str): + if not name or not isinstance(name, str): + raise SecretError("secret name must be a non-empty string") + if not _NAME_RE.match(name): + raise SecretError( + f"invalid secret name '{name}': letters, digits, " + f"underscores, dots, and dashes only" + ) + self.name = name + + @property + def reference(self) -> str: + """the env-var value the platform substitutes at boot.""" + return _REFERENCE_TEMPLATE.format(name=self.name) + + def __repr__(self) -> str: + return f"" + + +def render_env(env: Optional[Dict[str, Any]]) -> Dict[str, str]: + """render an env mapping, expanding Secret values to references. + + plain values stringify; Secret values become the platform's + reference syntax. + """ + if not env: + return {} + rendered: Dict[str, str] = {} + for key, value in env.items(): + if isinstance(value, Secret): + rendered[key] = value.reference + else: + rendered[key] = str(value) + return rendered + + +def secret_names(env: Optional[Dict[str, Any]]) -> List[str]: + """names of every Secret referenced in an env mapping.""" + if not env: + return [] + return [v.name for v in env.values() if isinstance(v, Secret)] + + +async def validate_secrets( + names: List[str], api=None +) -> None: + """fail fast when referenced secrets do not exist. + + workers with unresolvable references boot with the literal + template string in the env var, which is a confusing runtime + failure; checking up front turns it into a clear provision error. + """ + if not names: + return + if api is None: + from .api import AppsApiClient + + api = AppsApiClient() + existing = {s["name"] for s in await api.list_secrets()} + missing = sorted(set(names) - existing) + if missing: + raise SecretError( + f"secret{'s' if len(missing) > 1 else ''} not found: " + f"{', '.join(missing)}. create with `rp secret add `" + ) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index de8441d95..01bf68b1a 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -112,7 +112,7 @@ class ResourceSpec: dependencies: Optional[List[str]] = None system_dependencies: Optional[List[str]] = None volume: Optional[Any] = None - env: Optional[Dict[str, str]] = None + env: Optional[Dict[str, Any]] = None datacenter: Optional[List[str]] = None image: Optional[str] = None schedule: Optional[str] = None @@ -154,7 +154,9 @@ def to_manifest(self) -> Dict[str, Any]: self.volume, "name", None ) or str(self.volume) if self.env: - data["env"] = self.env + from .secret import render_env + + data["env"] = render_env(self.env) if self.datacenter: data["locations"] = ",".join(self.datacenter) if self.image: diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 82024b08a..3e836b90c 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -89,10 +89,12 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME ).isoformat() + from .secret import render_env + env = { "RUNPOD_TASK_TOKEN": token, "RUNPOD_TASK_PORT": str(TASK_PORT), - **(spec.env or {}), + **render_env(spec.env), } from .images import image_for_spec, local_python_version diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index ed5886144..617ed9c76 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -501,6 +501,84 @@ def app_delete(app_name, yes): ) +@cli.group() +def secret(): + """Manage platform secrets (encrypted env values).""" + + +@secret.command(name="add") +@click.argument("name") +@click.option("--value", default=None, help="Secret value (prompted if omitted).") +@click.option("--description", default="", help="Optional description.") +def secret_add(name, value, description): + """Create a secret. Reference it with runpod.Secret(NAME).""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + if value is None: + value = click.prompt("value", hide_input=True) + + try: + asyncio.run(AppsApiClient().create_secret(name, value, description)) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + ui.success( + f"secret [white]{name}[/white] created · " + f"[dim]use runpod.Secret(\"{name}\")[/dim]" + ) + + +@secret.command(name="list") +def secret_list(): + """List secret names (values are never shown).""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + try: + secrets = asyncio.run(AppsApiClient().list_secrets()) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + if not secrets: + ui.console.print("\n no secrets · run [white]rp secret add [/white]\n") + return + ui.console.print() + width = max(len(s["name"]) for s in secrets) + for entry in sorted(secrets, key=lambda s: s["name"]): + description = entry.get("description") or "" + ui.console.print( + f" [white]{entry['name']:<{width}}[/white] [dim]{description}[/dim]" + ) + ui.console.print() + + +@secret.command(name="rm") +@click.argument("name") +@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") +def secret_rm(name, yes): + """Delete a secret by name.""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + async def _rm(): + client = AppsApiClient() + secrets = await client.list_secrets() + match = next((s for s in secrets if s["name"] == name), None) + if match is None: + _fail(f"no secret named '{name}'") + await client.delete_secret(match["id"]) + + if not yes: + click.confirm(f"delete secret '{name}'?", abort=True) + try: + asyncio.run(_rm()) + except click.ClickException: + raise + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + ui.success(f"deleted secret [white]{name}[/white]") + + @cli.group() def env(): """Manage app environments.""" diff --git a/tests/test_apps/test_secret.py b/tests/test_apps/test_secret.py new file mode 100644 index 000000000..fe40d8ea9 --- /dev/null +++ b/tests/test_apps/test_secret.py @@ -0,0 +1,89 @@ +"""secret references, env rendering, and provision-time validation.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from runpod.apps.secret import ( + Secret, + SecretError, + render_env, + secret_names, + validate_secrets, +) + + +class TestSecretRef: + def test_reference_syntax(self): + assert Secret("hf-token").reference == "{{ RUNPOD_SECRET_hf-token }}" + + def test_invalid_names_raise(self): + with pytest.raises(SecretError): + Secret("") + with pytest.raises(SecretError): + Secret("has spaces") + with pytest.raises(SecretError): + Secret("no}}braces") + + def test_valid_names(self): + for name in ("hf-token", "MY_KEY", "a.b-c_d", "x1"): + assert Secret(name).name == name + + +class TestRenderEnv: + def test_mixed_env(self): + rendered = render_env( + {"HF_TOKEN": Secret("hf-token"), "MODE": "prod", "N": 3} + ) + assert rendered == { + "HF_TOKEN": "{{ RUNPOD_SECRET_hf-token }}", + "MODE": "prod", + "N": "3", + } + + def test_empty(self): + assert render_env(None) == {} + assert render_env({}) == {} + + def test_secret_names_extraction(self): + names = secret_names( + {"A": Secret("one"), "B": "plain", "C": Secret("two")} + ) + assert sorted(names) == ["one", "two"] + assert secret_names(None) == [] + + +class TestValidateSecrets: + def test_existing_pass(self): + api = AsyncMock() + api.list_secrets.return_value = [{"name": "hf-token"}] + asyncio.run(validate_secrets(["hf-token"], api=api)) + + def test_missing_raises_with_hint(self): + api = AsyncMock() + api.list_secrets.return_value = [{"name": "other"}] + with pytest.raises(SecretError, match="rp secret add"): + asyncio.run(validate_secrets(["hf-token"], api=api)) + + def test_no_references_no_api_call(self): + api = AsyncMock() + asyncio.run(validate_secrets([], api=api)) + api.list_secrets.assert_not_awaited() + + +class TestSpecManifest: + def test_manifest_renders_secret_references(self): + from runpod.apps.spec import ResourceKind, ResourceSpec + + spec = ResourceSpec( + kind=ResourceKind.QUEUE, + name="q", + cpu=["cpu3c-1-2"], + env={"TOKEN": Secret("tok"), "MODE": "x"}, + ) + manifest = spec.to_manifest() + assert manifest["env"] == { + "TOKEN": "{{ RUNPOD_SECRET_tok }}", + "MODE": "x", + } From 54301077a78960c3f52112d1c270f05cdf04c558 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:18:48 -0700 Subject: [PATCH 058/120] feat: wrangler-style prompting for rp secret add --- runpod/rp_cli/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 617ed9c76..de96371ed 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -507,14 +507,20 @@ def secret(): @secret.command(name="add") -@click.argument("name") +@click.argument("name", required=False) @click.option("--value", default=None, help="Secret value (prompted if omitted).") @click.option("--description", default="", help="Optional description.") def secret_add(name, value, description): - """Create a secret. Reference it with runpod.Secret(NAME).""" + """Create a secret. Reference it with runpod.Secret(NAME). + + Prompts for anything not provided: bare `rp secret add` asks for + name and value, `rp secret add NAME` asks for just the value. + """ from runpod.apps.api import AppsApiClient from runpod.rp_cli import console as ui + if name is None: + name = click.prompt("name") if value is None: value = click.prompt("value", hide_input=True) From a418531cfea698d9d3d8dabe3d2ddecd29990190 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:24:21 -0700 Subject: [PATCH 059/120] feat: registry credentials with name resolution and rp registry cli --- runpod/apps/api.py | 42 ++++++++++++++++ runpod/apps/app.py | 6 +++ runpod/apps/deploy.py | 6 +++ runpod/apps/dev.py | 9 +++- runpod/apps/registry.py | 50 +++++++++++++++++++ runpod/apps/spec.py | 3 ++ runpod/apps/tasks.py | 6 +++ runpod/rp_cli/main.py | 82 ++++++++++++++++++++++++++++++++ tests/test_apps/test_registry.py | 69 +++++++++++++++++++++++++++ 9 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 runpod/apps/registry.py create mode 100644 tests/test_apps/test_registry.py diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 9f58c0736..763f6148b 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -268,6 +268,48 @@ async def create_network_volume( ) return data["createNetworkVolume"] + # -- registry credentials -- + + async def list_registry_auths(self) -> List[Dict[str, Any]]: + query = """ + query myRegistryCreds { + myself { + containerRegistryCreds { id name } + } + } + """ + data = await self._execute(query) + return data["myself"].get("containerRegistryCreds") or [] + + async def create_registry_auth( + self, name: str, username: str, password: str + ) -> Dict[str, Any]: + mutation = """ + mutation SaveRegistryAuth($input: SaveRegistryAuthInput!) { + saveRegistryAuth(input: $input) { id name } + } + """ + data = await self._execute( + mutation, + { + "input": { + "name": name, + "username": username, + "password": password, + } + }, + ) + return data["saveRegistryAuth"] + + async def delete_registry_auth(self, auth_id: str) -> bool: + mutation = """ + mutation DeleteRegistryAuth($registryAuthId: String!) { + deleteRegistryAuth(registryAuthId: $registryAuthId) + } + """ + data = await self._execute(mutation, {"registryAuthId": auth_id}) + return "deleteRegistryAuth" in data + # -- secrets -- async def list_secrets(self) -> List[Dict[str, Any]]: diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 1732807f4..d5dd52d09 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -92,6 +92,7 @@ def queue( env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, + registry_auth: Optional[str] = None, ) -> Callable[[Callable], FunctionHandle]: """declare a queue-based serverless endpoint from a function. @@ -115,6 +116,7 @@ def decorator(fn: Callable) -> FunctionHandle: env=env, datacenter=_datacenter_list(datacenter), image=image, + registry_auth=registry_auth, ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -134,6 +136,7 @@ def task( volume: Optional[Any] = None, env: Optional[Dict[str, Any]] = None, image: Optional[str] = None, + registry_auth: Optional[str] = None, datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Callable], FunctionHandle]: """declare ephemeral pod compute from a function. @@ -155,6 +158,7 @@ def decorator(fn: Callable) -> FunctionHandle: volume=_volume_ref(volume), env=env, image=image, + registry_auth=registry_auth, datacenter=_datacenter_list(datacenter), ) handle = FunctionHandle(self, fn, spec) @@ -178,6 +182,7 @@ def api( env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, + registry_auth: Optional[str] = None, ) -> Callable[[Any], ApiHandle]: """declare a load-balanced serverless endpoint. @@ -216,6 +221,7 @@ def decorator(target: Any) -> ApiHandle: env=env, datacenter=_datacenter_list(datacenter), image=image, + registry_auth=registry_auth, ) handle = ApiHandle(self, target, spec) self._register(spec.name, handle) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index f81758538..8d797b887 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -361,6 +361,7 @@ async def reconcile_endpoints( if h.spec.kind in (ResourceKind.QUEUE, ResourceKind.API) } + from .registry import resolve_registry_auth from .volume import VolumeResolver resolver = VolumeResolver(client) @@ -370,6 +371,11 @@ async def reconcile_endpoints( app, handle.spec, environment["id"], build_id, python_version ) await attach_endpoint_volumes(payload, handle.spec, resolver, app) + auth_id = await resolve_registry_auth( + handle.spec.registry_auth, api=client + ) + if auth_id: + payload["template"]["containerRegistryAuthId"] = auth_id if name in existing: payload["id"] = existing[name] result = await client.save_endpoint(payload) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 6e34c09a6..6d82c37fd 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -244,10 +244,17 @@ def _emit(self, event: str, *args) -> None: handler(*args) async def _attach_volumes(self, payload: Dict, spec, app) -> None: - """resolve the resource's volumes onto a dev endpoint payload.""" + """resolve the resource's volumes and registry auth onto a + dev endpoint payload.""" from .deploy import attach_endpoint_volumes + from .registry import resolve_registry_auth from .volume import VolumeResolver + if spec.registry_auth: + auth_id = await resolve_registry_auth( + spec.registry_auth, api=self.api + ) + payload["template"]["containerRegistryAuthId"] = auth_id if not spec.volume: return if self._volume_resolver is None: diff --git a/runpod/apps/registry.py b/runpod/apps/registry.py new file mode 100644 index 000000000..336fa283c --- /dev/null +++ b/runpod/apps/registry.py @@ -0,0 +1,50 @@ +"""container registry credentials for private images. + +credentials are pure provision-time config referenced by name: + + @app.queue(image="ghcr.io/me/private:latest", registry_auth="my-ghcr") + def infer(): ... + +create and manage them with `rp registry add/list/rm`. resolution +(name -> containerRegistryAuthId) happens when the resource provisions. +""" + +import logging +from typing import Optional + +from .errors import AppError + +log = logging.getLogger(__name__) + + +class RegistryAuthError(AppError): + pass + + +async def resolve_registry_auth( + name: Optional[str], api=None +) -> Optional[str]: + """resolve a credential name (or id) to a containerRegistryAuthId.""" + if not name: + return None + if api is None: + from .api import AppsApiClient + + api = AppsApiClient() + creds = await api.list_registry_auths() + for cred in creds: + if cred["id"] == name: + return cred["id"] + matches = [c for c in creds if c.get("name") == name] + if len(matches) > 1: + raise RegistryAuthError( + f"multiple registry credentials named '{name}' " + f"({', '.join(m['id'] for m in matches)}); reference by id" + ) + if matches: + return matches[0]["id"] + available = ", ".join(sorted(c["name"] for c in creds)) or "(none)" + raise RegistryAuthError( + f"registry credential '{name}' not found. available: {available}. " + f"create one with `rp registry add {name}`" + ) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 01bf68b1a..621202c1c 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -115,6 +115,7 @@ class ResourceSpec: env: Optional[Dict[str, Any]] = None datacenter: Optional[List[str]] = None image: Optional[str] = None + registry_auth: Optional[str] = None schedule: Optional[str] = None routes: List[RouteSpec] = field(default_factory=list) asgi_factory: Optional[str] = None @@ -161,6 +162,8 @@ def to_manifest(self) -> Dict[str, Any]: data["locations"] = ",".join(self.datacenter) if self.image: data["imageName"] = self.image + if self.registry_auth: + data["registryAuth"] = self.registry_auth if self.schedule: data["schedule"] = self.schedule if self.routes: diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 3e836b90c..1f4e47221 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -153,6 +153,12 @@ def _headers(self) -> Dict[str, str]: async def start(self) -> None: pod = _pod_input(self.spec, self.token, self.spec.name) + if self.spec.registry_auth: + from .registry import resolve_registry_auth + + pod["containerRegistryAuthId"] = await resolve_registry_auth( + self.spec.registry_auth, api=self.api + ) if self.spec.volume: pod = await self._attach_volume(pod) result = await self.api.deploy_task_pod( diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index de96371ed..3c5e89ec9 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -501,6 +501,88 @@ def app_delete(app_name, yes): ) +@cli.group() +def registry(): + """Manage container registry credentials (private images).""" + + +@registry.command(name="add") +@click.argument("name", required=False) +@click.option("--username", default=None, help="Registry username (prompted if omitted).") +@click.option("--password", default=None, help="Registry password/token (prompted if omitted).") +def registry_add(name, username, password): + """Create a registry credential. Reference it with registry_auth=NAME.""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + if name is None: + name = click.prompt("name") + if username is None: + username = click.prompt("username") + if password is None: + password = click.prompt("password", hide_input=True) + + try: + asyncio.run( + AppsApiClient().create_registry_auth(name, username, password) + ) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + ui.success( + f"registry credential [white]{name}[/white] created · " + f'[dim]use registry_auth="{name}"[/dim]' + ) + + +@registry.command(name="list") +def registry_list(): + """List registry credential names.""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + try: + creds = asyncio.run(AppsApiClient().list_registry_auths()) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + + if not creds: + ui.console.print( + "\n no registry credentials · run [white]rp registry add [/white]\n" + ) + return + ui.console.print() + for entry in sorted(creds, key=lambda c: c["name"]): + ui.console.print(f" [white]{entry['name']}[/white] [dim]{entry['id']}[/dim]") + ui.console.print() + + +@registry.command(name="rm") +@click.argument("name") +@click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") +def registry_rm(name, yes): + """Delete a registry credential by name.""" + from runpod.apps.api import AppsApiClient + from runpod.rp_cli import console as ui + + async def _rm(): + client = AppsApiClient() + creds = await client.list_registry_auths() + match = next((c for c in creds if c["name"] == name), None) + if match is None: + _fail(f"no registry credential named '{name}'") + await client.delete_registry_auth(match["id"]) + + if not yes: + click.confirm(f"delete registry credential '{name}'?", abort=True) + try: + asyncio.run(_rm()) + except click.ClickException: + raise + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + ui.success(f"deleted registry credential [white]{name}[/white]") + + @cli.group() def secret(): """Manage platform secrets (encrypted env values).""" diff --git a/tests/test_apps/test_registry.py b/tests/test_apps/test_registry.py new file mode 100644 index 000000000..f8ecfde61 --- /dev/null +++ b/tests/test_apps/test_registry.py @@ -0,0 +1,69 @@ +"""registry credential resolution.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from runpod.apps.registry import RegistryAuthError, resolve_registry_auth + + +def _api(creds=None): + api = AsyncMock() + api.list_registry_auths.return_value = creds or [] + return api + + +class TestResolveRegistryAuth: + def test_none_passthrough(self): + assert ( + asyncio.run(resolve_registry_auth(None, api=_api())) is None + ) + + def test_resolves_by_name(self): + api = _api([{"id": "cra-1", "name": "my-ghcr"}]) + assert ( + asyncio.run(resolve_registry_auth("my-ghcr", api=api)) == "cra-1" + ) + + def test_resolves_by_id(self): + api = _api([{"id": "cra-1", "name": "my-ghcr"}]) + assert ( + asyncio.run(resolve_registry_auth("cra-1", api=api)) == "cra-1" + ) + + def test_missing_lists_available(self): + api = _api([{"id": "cra-1", "name": "other"}]) + with pytest.raises( + RegistryAuthError, match="available: other" + ): + asyncio.run(resolve_registry_auth("ghost", api=api)) + + def test_duplicates_require_id(self): + api = _api( + [ + {"id": "cra-1", "name": "dup"}, + {"id": "cra-2", "name": "dup"}, + ] + ) + with pytest.raises(RegistryAuthError, match="reference by id"): + asyncio.run(resolve_registry_auth("dup", api=api)) + + +class TestSpecPlumbing: + def test_decorator_accepts_registry_auth(self): + from runpod.apps.app import App + + app = App("regtest") + + @app.queue( + name="q", + cpu=["cpu3c-1-2"], + image="ghcr.io/me/x:1", + registry_auth="my-ghcr", + ) + def q(): + pass + + assert q.spec.registry_auth == "my-ghcr" + assert q.spec.to_manifest()["registryAuth"] == "my-ghcr" From be5d7e95389390ac92e2b1aad72cc3098c1be817 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:44:39 -0700 Subject: [PATCH 060/120] feat: platform-cached model references --- runpod/__init__.py | 1 + runpod/apps/__init__.py | 2 + runpod/apps/app.py | 6 +++ runpod/apps/deploy.py | 6 +++ runpod/apps/dev.py | 4 ++ runpod/apps/model.py | 91 +++++++++++++++++++++++++++++++++++ runpod/apps/spec.py | 5 ++ tests/test_apps/test_model.py | 68 ++++++++++++++++++++++++++ 8 files changed, 183 insertions(+) create mode 100644 runpod/apps/model.py create mode 100644 tests/test_apps/test_model.py diff --git a/runpod/__init__.py b/runpod/__init__.py index 871d79b47..296f0c659 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -34,6 +34,7 @@ App, DataCenter, EndpointNotFound, + Model, Queue, Secret, Volume, diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index 4caa47ab7..c179e482b 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -26,6 +26,7 @@ from .schedule import schedule from .spec import ResourceKind, ResourceSpec, RouteSpec from .stubs import Api, Queue +from .model import Model from .secret import Secret from .volume import Volume @@ -38,6 +39,7 @@ "DataCenter", "EndpointNotFound", "FunctionHandle", + "Model", "Queue", "Secret", "Volume", diff --git a/runpod/apps/app.py b/runpod/apps/app.py index d5dd52d09..7913cc0db 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -93,6 +93,7 @@ def queue( datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, registry_auth: Optional[str] = None, + model: Optional[Any] = None, ) -> Callable[[Callable], FunctionHandle]: """declare a queue-based serverless endpoint from a function. @@ -117,6 +118,7 @@ def decorator(fn: Callable) -> FunctionHandle: datacenter=_datacenter_list(datacenter), image=image, registry_auth=registry_auth, + model=model, ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -137,6 +139,7 @@ def task( env: Optional[Dict[str, Any]] = None, image: Optional[str] = None, registry_auth: Optional[str] = None, + model: Optional[Any] = None, datacenter: Optional[Union[str, List[str]]] = None, ) -> Callable[[Callable], FunctionHandle]: """declare ephemeral pod compute from a function. @@ -159,6 +162,7 @@ def decorator(fn: Callable) -> FunctionHandle: env=env, image=image, registry_auth=registry_auth, + model=model, datacenter=_datacenter_list(datacenter), ) handle = FunctionHandle(self, fn, spec) @@ -183,6 +187,7 @@ def api( datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, registry_auth: Optional[str] = None, + model: Optional[Any] = None, ) -> Callable[[Any], ApiHandle]: """declare a load-balanced serverless endpoint. @@ -222,6 +227,7 @@ def decorator(target: Any) -> ApiHandle: datacenter=_datacenter_list(datacenter), image=image, registry_auth=registry_auth, + model=model, ) handle = ApiHandle(self, target, spec) self._register(spec.name, handle) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 8d797b887..5660c4eeb 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -376,6 +376,12 @@ async def reconcile_endpoints( ) if auth_id: payload["template"]["containerRegistryAuthId"] = auth_id + if handle.spec.model: + from .model import model_reference + + payload["modelReferences"] = [ + model_reference(handle.spec.model) + ] if name in existing: payload["id"] = existing[name] result = await client.save_endpoint(payload) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 6d82c37fd..1a65610f8 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -255,6 +255,10 @@ async def _attach_volumes(self, payload: Dict, spec, app) -> None: spec.registry_auth, api=self.api ) payload["template"]["containerRegistryAuthId"] = auth_id + if spec.model: + from .model import model_reference + + payload["modelReferences"] = [model_reference(spec.model)] if not spec.volume: return if self._volume_resolver is None: diff --git a/runpod/apps/model.py b/runpod/apps/model.py new file mode 100644 index 000000000..d751b8d73 --- /dev/null +++ b/runpod/apps/model.py @@ -0,0 +1,91 @@ +"""platform-cached models: weights staged on hosts before workers start. + +a Model references huggingface weights by repo id. attached to a +resource, the platform stages the weights on the host and defers +worker start until they are ready, so cold starts never download +model files. + + llama = runpod.Model("meta-llama/Llama-3.1-8B-Instruct") + + @app.queue(gpu="H100", model=llama, env={"HF_TOKEN": runpod.Secret("hf")}) + def chat(prompt: str): + llm = vllm.LLM(model=str(llama.path)) + +inside the worker the weights appear in two layouts: + - runpod store: /runpod/model-store/huggingface/{org}/{name}/{revision} + (Model.path, revision from the MODEL_REVISION env var) + - hf cache: /runpod-volume/huggingface-cache/hub/models--{org}--{name} + (transformers/vllm find it via the standard cache convention) + +gated models need an HF_TOKEN env var on the same resource (a Secret +reference works; the platform decrypts it for validation). +""" + +import os +import re +from pathlib import Path +from typing import Optional + +from .errors import AppError + +MODEL_STORE_ROOT = Path("/runpod/model-store/huggingface") +HF_CACHE_ROOT = Path("/runpod-volume/huggingface-cache") + +_REPO_RE = re.compile(r"^[\w.-]+/[\w.-]+$") + + +class ModelError(AppError): + pass + + +class Model: + """a huggingface model reference, staged by the platform.""" + + def __init__(self, reference: str): + if not reference or not isinstance(reference, str): + raise ModelError("model reference must be a non-empty string") + repo, _, revision = reference.partition(":") + if not _REPO_RE.match(repo): + raise ModelError( + f"invalid model reference '{reference}': expected " + f"'owner/name' or 'owner/name:revision'" + ) + self.reference = reference + self.owner, self.name = repo.split("/") + self.revision = revision or None + + @property + def path(self) -> Path: + """the staged weights directory inside the worker. + + the platform resolves the exact revision at deploy time and + exposes it via MODEL_REVISION; outside a worker (or before the + mount exists) this still forms the correct path shape. + """ + revision = os.environ.get("MODEL_REVISION") or self.revision or "" + base = MODEL_STORE_ROOT / self.owner / self.name + return base / revision if revision else base + + @property + def hf_cache_path(self) -> Path: + """the huggingface-convention cache directory for this model.""" + return ( + HF_CACHE_ROOT / "hub" / f"models--{self.owner}--{self.name}" + ) + + def __repr__(self) -> str: + return f"" + + +def model_reference(model) -> Optional[str]: + """normalize a spec's model field to the api-facing reference.""" + if model is None: + return None + if isinstance(model, Model): + return model.reference + if isinstance(model, str): + return Model(model).reference + raise ModelError( + f"model must be a runpod.Model or 'owner/name' string, " + f"got {type(model).__name__}" + ) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 621202c1c..881875b6c 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -116,6 +116,7 @@ class ResourceSpec: datacenter: Optional[List[str]] = None image: Optional[str] = None registry_auth: Optional[str] = None + model: Optional[Any] = None schedule: Optional[str] = None routes: List[RouteSpec] = field(default_factory=list) asgi_factory: Optional[str] = None @@ -164,6 +165,10 @@ def to_manifest(self) -> Dict[str, Any]: data["imageName"] = self.image if self.registry_auth: data["registryAuth"] = self.registry_auth + if self.model: + from .model import model_reference + + data["model"] = model_reference(self.model) if self.schedule: data["schedule"] = self.schedule if self.routes: diff --git a/tests/test_apps/test_model.py b/tests/test_apps/test_model.py new file mode 100644 index 000000000..d21496a6f --- /dev/null +++ b/tests/test_apps/test_model.py @@ -0,0 +1,68 @@ +"""model references: parsing, paths, payload plumbing.""" + +from pathlib import Path + +import pytest + +from runpod.apps.model import Model, ModelError, model_reference + + +class TestModelRef: + def test_repo_parse(self): + model = Model("meta-llama/Llama-3.1-8B-Instruct") + assert model.owner == "meta-llama" + assert model.name == "Llama-3.1-8B-Instruct" + assert model.revision is None + + def test_revision_parse(self): + model = Model("org/name:abc123") + assert model.revision == "abc123" + assert model.reference == "org/name:abc123" + + def test_invalid_references(self): + for bad in ("", "no-slash", "a/b/c", "spaces in/name"): + with pytest.raises(ModelError): + Model(bad) + + def test_store_path_with_env_revision(self, monkeypatch): + monkeypatch.setenv("MODEL_REVISION", "deadbeef") + model = Model("org/name") + assert model.path == Path( + "/runpod/model-store/huggingface/org/name/deadbeef" + ) + + def test_store_path_without_revision(self, monkeypatch): + monkeypatch.delenv("MODEL_REVISION", raising=False) + model = Model("org/name") + assert model.path == Path("/runpod/model-store/huggingface/org/name") + + def test_hf_cache_path(self): + model = Model("org/name") + assert model.hf_cache_path == Path( + "/runpod-volume/huggingface-cache/hub/models--org--name" + ) + + def test_model_reference_normalization(self): + assert model_reference(None) is None + assert model_reference("org/name") == "org/name" + assert model_reference(Model("org/name:rev")) == "org/name:rev" + with pytest.raises(ModelError): + model_reference(42) + + +class TestSpecPlumbing: + def test_decorator_accepts_model(self): + from runpod.apps.app import App + + app = App("modeltest") + llama = Model("meta-llama/Llama-3.1-8B-Instruct") + + @app.queue(name="chat", gpu="4090", model=llama) + def chat(prompt: str): + pass + + assert chat.spec.model is llama + assert ( + chat.spec.to_manifest()["model"] + == "meta-llama/Llama-3.1-8B-Instruct" + ) From 245e2d6a0939194e901c46062489e268b53ccbc5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 20:56:54 -0700 Subject: [PATCH 061/120] docs: apps-first readme and example suite doubling as e2e tests --- README.md | 418 ++++++++++++++++++++------ examples/apps/01_hello_world.py | 22 ++ examples/apps/02_invocation_styles.py | 37 +++ examples/apps/03_gpu.py | 33 ++ examples/apps/04_dependencies.py | 34 +++ examples/apps/05_api_service.py | 41 +++ examples/apps/06_tasks.py | 31 ++ examples/apps/07_pipelines.py | 37 +++ examples/apps/08_volumes.py | 39 +++ examples/apps/09_secrets.py | 33 ++ examples/apps/10_cached_models.py | 33 ++ examples/apps/11_train_eval.py | 69 +++++ examples/apps/12_custom_image.py | 24 ++ examples/apps/README.md | 51 ++++ examples/apps/run_all.py | 77 +++++ runpod/apps/api.py | 14 +- runpod/apps/placement.py | 47 ++- tests/test_apps/test_placement.py | 5 +- 18 files changed, 933 insertions(+), 112 deletions(-) create mode 100644 examples/apps/01_hello_world.py create mode 100644 examples/apps/02_invocation_styles.py create mode 100644 examples/apps/03_gpu.py create mode 100644 examples/apps/04_dependencies.py create mode 100644 examples/apps/05_api_service.py create mode 100644 examples/apps/06_tasks.py create mode 100644 examples/apps/07_pipelines.py create mode 100644 examples/apps/08_volumes.py create mode 100644 examples/apps/09_secrets.py create mode 100644 examples/apps/10_cached_models.py create mode 100644 examples/apps/11_train_eval.py create mode 100644 examples/apps/12_custom_image.py create mode 100644 examples/apps/README.md create mode 100644 examples/apps/run_all.py diff --git a/README.md b/README.md index b34014b19..1d1a3f575 100644 --- a/README.md +++ b/README.md @@ -13,65 +13,331 @@ -Welcome to the official Python library for Runpod API & SDK. +The official Python SDK for [Runpod](https://runpod.io): define GPU functions in Python, run them in the cloud with one decorator, and deploy them as production endpoints. + +```python +import runpod +from runpod import App + +app = App("my-app") + +@app.queue(gpu="4090") +def embed(text: str): + import torch + ... + return vector + +@runpod.local_entrypoint +def main(): + print(embed.remote("hello world")) # runs on a cloud 4090 +``` + +```bash +pip install runpod +rp login # browser auth +rp dev main.py # live dev session: edit, re-run, logs stream back +rp deploy # production endpoints +``` ## Table of Contents -- [Table of Contents](#table-of-contents) -- [💻 | Installation](#--installation) -- [⚡ | Serverless Worker (SDK)](#--serverless-worker-sdk) - - [Quick Start](#quick-start) - - [Local Test Worker](#local-test-worker) -- [📚 | API Language Library (GraphQL Wrapper)](#--api-language-library-graphql-wrapper) - - [Endpoints](#endpoints) - - [GPU Cloud (Pods)](#gpu-cloud-pods) -- [📁 | Directory](#--directory) -- [🤝 | Community and Contributing](#--community-and-contributing) +- [Installation](#--installation) +- [Apps: Python Functions in the Cloud](#--apps-python-functions-in-the-cloud) + - [Resources: queue, api, task](#resources-queue-api-task) + - [Calling Functions](#calling-functions) + - [Hardware Selection](#hardware-selection) + - [Dependencies and Images](#dependencies-and-images) + - [Volumes](#volumes) + - [Secrets](#secrets) + - [Cached Models](#cached-models) + - [Private Registries](#private-registries) + - [The Dev Loop](#the-dev-loop) + - [Deploying](#deploying) + - [CLI Reference](#cli-reference) +- [Serverless Worker (SDK)](#--serverless-worker-sdk) +- [API Language Library](#--api-language-library-graphql-wrapper) +- [Directory](#--directory) +- [Community and Contributing](#--community-and-contributing) ## 💻 | Installation -### Install from PyPI (Stable Release) - ```bash -# Install with pip +# stable pip install runpod -# Install with uv (faster alternative) +# or with uv uv add runpod + +# latest development version +pip install git+https://github.com/runpod/runpod-python.git ``` -### Install from GitHub (Latest Changes) +*Python 3.10 or higher is required.* -To get the latest changes that haven't been released to PyPI yet: +Installing the package also installs the `rp` CLI (also available as `runpod`). Authenticate once: ```bash -# Install latest development version from main branch with pip -pip install git+https://github.com/runpod/runpod-python.git +rp login +``` + +## 🚀 | Apps: Python Functions in the Cloud -# Install with uv -uv add git+https://github.com/runpod/runpod-python.git +An **App** is a named collection of functions that run on Runpod. Decorate plain Python functions; the SDK handles packaging, provisioning, scaling, and routing. -# Install a specific branch -pip install git+https://github.com/runpod/runpod-python.git@branch-name +### Resources: queue, api, task -# Install a specific tag/release -pip install git+https://github.com/runpod/runpod-python.git@v1.0.0 +Three decorators cover the three execution models: -# Install in editable mode for development -git clone https://github.com/runpod/runpod-python.git -cd runpod-python -pip install -e . +```python +import runpod +from runpod import App + +app = App("inference") + +# queue: autoscaling job queue (scale-to-zero serverless workers) +@app.queue(gpu="4090") +def transcribe(audio_url: str): + ... + return text + +# api: load-balanced http service from a class +from runpod import init, get, post + +@app.api(cpu="cpu5c-2-4") +class Counter: + @init + def setup(self): # runs once per worker, before traffic + self.count = 0 + + @post("/bump") + async def bump(self, body: dict): + self.count += body.get("by", 1) + return {"count": self.count} + + @get("/value") + async def value(self): + return {"count": self.count} + +# task: one ephemeral pod per call, runs to completion, then terminates +@app.task(gpu="H100", gpu_count=2) +def train(steps: int = 1000): + ... + return {"loss": final_loss} ``` -*Python 3.10 or higher is required to use the latest version of this package.* +Pick by shape of work: -## ⚡ | Serverless Worker (SDK) +| | `@app.queue` | `@app.api` | `@app.task` | +|---|---|---|---| +| execution | job queue | http routes | dedicated pod per call | +| scaling | 0..N workers | 0..N workers | one pod, then gone | +| best for | inference jobs | rest services, streaming | training runs, batch jobs | +| duration | seconds to minutes | request-scoped | minutes to hours | + +`@app.api` also accepts a zero-argument function returning an ASGI app (FastAPI, Starlette) if you'd rather bring your own framework. + +### Calling Functions + +Every decorated function keeps its normal Python identity and gains remote invocation: + +```python +transcribe.remote(url) # run remotely, block for the result +await transcribe.remote.aio(url) # async variant +transcribe.spawn(url) # fire and forget -> Job +transcribe.local(url) # run in this process (no cloud) + +job = transcribe.spawn(url) +result = job.result() # collect later +``` + +Functions can call each other — from your laptop, or from inside another worker: + +```python +@app.queue(cpu="cpu3c-1-2") +def orchestrate(x: int): + return {"embedding": embed.remote(x), "meta": lookup.remote(x)} +``` + +`@runpod.local_entrypoint` marks the function `rp dev` runs as your client-side script. + +### Hardware Selection + +GPUs resolve from pool ids, exact device names, or shorthand fragments: + +```python +@app.queue(gpu="4090") # NVIDIA GeForce RTX 4090 +@app.queue(gpu="H100") # all H100 variants +@app.queue(gpu="B200", gpu_count=8) # multi-gpu +@app.queue(gpu=GpuGroup.ADA_24) # pool id enum +@app.queue(gpu=["4090", "5090"]) # any of several +@app.queue(cpu="cpu5c-2-4") # cpu instance flavor +``` + +Unknown names fail at import time with the full list of valid options — not at provision time with a cryptic API error. + +### Dependencies and Images + +```python +@app.queue( + cpu="cpu3c-1-2", + dependencies=["pyfiglet", "numpy>=2"], # pip packages + system_dependencies=["ffmpeg"], # apt packages +) +def render(text: str): ... +``` + +At deploy time the SDK resolves your dependency closure into the build artifact, so cold starts never run pip. Prefer a fully custom image: + +```python +@app.queue(image="ghcr.io/me/mine:latest", registry_auth="my-ghcr") +def custom(): ... +``` + +Custom images must include `python3`; the worker runtime bootstraps itself. + +### Volumes + +Network volumes persist across calls and are shared between resources: + +```python +from runpod import Volume + +models = Volume("models", size=100) # created on first use + +@app.task(gpu="H100", volume=models) +def train(): + torch.save(state, models.path / "run-1/model.pt") + +@app.queue(gpu="4090", volume=models) +def infer(prompt: str): + sd = torch.load(models.path / "run-1/model.pt") +``` + +A volume lives in one datacenter, and everything attached to it must schedule there. The SDK solves placement over the whole app: it intersects each resource's hardware availability per datacenter, picks the one where the most-constrained resource has the best stock, and errors with a per-resource breakdown when no datacenter can host everyone. + +`volume.path` resolves to the platform mount inside the worker (`/workspace` on task pods, `/runpod-volume` on endpoint workers). Endpoints may take a list of volumes (one per datacenter) to serve from multiple regions. + +### Secrets + +Platform-encrypted values, decrypted only inside the worker: + +```bash +rp secret add hf-token # prompts for the value +``` + +```python +from runpod import Secret -This python package can also be used to create a serverless worker that can be deployed to Runpod as a custom endpoint API. +@app.queue(gpu="4090", env={"HF_TOKEN": Secret("hf-token")}) +def download(): ... # worker sees the real value in $HF_TOKEN +``` + +The value never passes through the SDK. Deploys fail fast when a referenced secret does not exist. -### Quick Start +### Cached Models -Create a python script in your project that contains your model definition and the Runpod worker start code. Run this python code as your default container start command: +Reference HuggingFace weights and the platform stages them on the host **before** your worker starts — cold starts skip the download entirely: + +```python +from runpod import Model + +llama = Model("meta-llama/Llama-3.1-8B-Instruct") + +@app.queue(gpu="H100", model=llama, env={"HF_TOKEN": Secret("hf-token")}) +def chat(prompt: str): + llm = vllm.LLM(model=str(llama.path)) # weights already on disk +``` + +Weights also appear in the standard HuggingFace cache layout, so `transformers` and `vllm` find them with zero configuration. Gated repos authenticate with an `HF_TOKEN` env var (a `Secret` works). + +### Private Registries + +```bash +rp registry add my-ghcr # prompts for username/password +``` + +```python +@app.queue(image="ghcr.io/me/private:latest", registry_auth="my-ghcr") +def infer(): ... +``` + +### The Dev Loop + +```bash +rp dev main.py +``` + +provisions temporary live endpoints for your app, runs your `@runpod.local_entrypoint`, and streams everything back: + +``` +dev inference · main.py + transcribe queue 4090 se3u174ensbubc ↗ + + ▸ main() local entrypoint + ○ transcribe() + ● transcribe() running on worker 1x74v8dg5e9i + transcribe │ loading model... + transcribe │ transcribing 42s of audio + ✓ transcribe() in 12.4s + + ✓ done in 12.6s · enter re-run · edit reload · ^C quit +``` + +- **edit a file** → session reloads, shows a diff of what changed, next call runs the new code +- **enter** → re-run the entrypoint +- **worker stdout streams live** into your terminal, attributed per function +- **ctrl-c** → everything is cleaned up; dev sessions leave nothing behind + +Code ships with each request during dev, so there is no build step between edits. + +### Deploying + +```bash +rp deploy # every app found in the current directory +rp deploy main.py # one file +rp deploy -e staging # a named environment +``` + +Deploys package your project and its resolved dependency environment into a build artifact, upload it once, and reconcile one endpoint per queue/api resource (tasks provision per call). Workers cold-start from the artifact — no dependency resolution at runtime. + +``` +deploy inference → default · transcribe +✓ resolving dependencies 89 packages +✓ packaging artifact +✓ uploading build ━━━━━━━━━━ 53.3 MB +✓ reconciling endpoints +✓ inference/default is live 44.9s + + transcribe gumuj47vi1xu4y ↗ +``` + +Manage what's deployed: + +```bash +rp app list # all apps and environments +rp env get default -a inference # endpoints in an environment +rp undeploy -a inference # tear down an environment's endpoints +rp app delete inference # remove the app entirely +``` + +### CLI Reference + +| command | | +|---|---| +| `rp login` | browser auth (or `--api-key` to paste a key) | +| `rp dev ` | live dev session | +| `rp deploy [target]` | build and deploy apps | +| `rp undeploy` | tear down an environment | +| `rp app list/get/delete` | app management | +| `rp env list/get/create/delete` | environment management | +| `rp secret add/list/rm` | platform secrets | +| `rp registry add/list/rm` | registry credentials | +| `rp logs [-f]` | pod logs (snapshot or follow) | +| `rp pod / ssh / exec / config` | pod management | + +## ⚡ | Serverless Worker (SDK) + +This package is also what you use to build the worker side of a custom serverless endpoint: a handler function that processes jobs. ```python # my_worker.py @@ -79,7 +345,6 @@ Create a python script in your project that contains your model definition and t import runpod def is_even(job): - job_input = job["input"] the_number = job_input["number"] @@ -94,13 +359,13 @@ def is_even(job): runpod.serverless.start({"handler": is_even}) ``` -Make sure that this file is ran when your container starts. This can be accomplished by calling it in the docker command when you set up a template at [console.runpod.io/serverless/user/templates](https://console.runpod.io/serverless/user/templates) or by setting it as the default command in your Dockerfile. +Make sure that this file is run when your container starts. This can be accomplished by calling it in the docker command when you set up a template at [console.runpod.io/serverless/user/templates](https://console.runpod.io/serverless/user/templates) or by setting it as the default command in your Dockerfile. -See our [blog post](https://www.runpod.io/blog/build-basic-serverless-api) for creating a basic Serverless API, or view the [details docs](https://docs.runpod.io/serverless-ai/custom-apis) for more information. +See our [blog post](https://www.runpod.io/blog/build-basic-serverless-api) for creating a basic Serverless API, or view the [detailed docs](https://docs.runpod.io/serverless-ai/custom-apis) for more information. ### Local Test Worker -You can also test your worker locally before deploying it to Runpod. This is useful for debugging and testing. +You can test your worker locally before deploying it to Runpod: ```bash python my_worker.py --rp_serve_api @@ -108,7 +373,7 @@ python my_worker.py --rp_serve_api ### Worker Fitness Checks -Fitness checks allow you to validate your worker environment at startup before processing jobs. If any check fails, the worker exits immediately, allowing your orchestrator to restart it. +Fitness checks validate your worker environment at startup before processing jobs. If any check fails, the worker exits immediately, allowing your orchestrator to restart it. ```python # my_worker.py @@ -116,7 +381,6 @@ Fitness checks allow you to validate your worker environment at startup before p import runpod import torch -# Register fitness checks using the decorator @runpod.serverless.register_fitness_check def check_gpu_available(): """Verify GPU is available.""" @@ -137,21 +401,14 @@ def handler(job): # Your handler code here return {"output": "success"} -# Fitness checks run before handler initialization (production only) runpod.serverless.start({"handler": handler}) ``` -**Key Features:** -- Supports both synchronous and asynchronous check functions -- Checks run only once at worker startup (production mode) -- Runs before handler initialization and job processing begins -- Any check failure exits with code 1 (worker marked unhealthy) - -See [Worker Fitness Checks](https://github.com/runpod/runpod-python/blob/main/docs/serverless/worker_fitness_checks.md) documentation for more examples and best practices. +Built-in system fitness checks (GPU health, disk, memory, CUDA, network) run automatically on serverless workers. ## 📚 | API Language Library (GraphQL Wrapper) -When interacting with the Runpod API you can use this library to make requests to the API. +When interacting with the Runpod API you can use this library to make requests. ```python import runpod @@ -161,11 +418,11 @@ runpod.api_key = "your_runpod_api_key_found_under_settings" ### Endpoints -You can interact with Runpod endpoints via a `run` or `run_sync` method. - -#### Basic Usage +Interact with existing endpoints by id: ```python +import runpod + endpoint = runpod.Endpoint("ENDPOINT_ID") run_request = endpoint.run( @@ -175,11 +432,13 @@ run_request = endpoint.run( # Check the status of the endpoint run request print(run_request.status()) -# Get the output of the endpoint run request, blocking until the endpoint run is complete. +# Get the output of the endpoint run request, blocking until complete print(run_request.output()) ``` ```python +import runpod + endpoint = runpod.Endpoint("ENDPOINT_ID") run_request = endpoint.run_sync( @@ -187,53 +446,20 @@ run_request = endpoint.run_sync( ) # Returns the job results if completed within 90 seconds, otherwise, returns the job status. -print(run_request ) -``` - -#### API Key Management - -The SDK supports multiple ways to set API keys: - -**1. Global API Key** (Default) -```python -import runpod - -# Set global API key -runpod.api_key = "your_runpod_api_key" - -# All endpoints will use this key by default -endpoint = runpod.Endpoint("ENDPOINT_ID") -result = endpoint.run_sync({"input": "data"}) -``` - -**2. Endpoint-Specific API Key** -```python -# Create endpoint with its own API key -endpoint = runpod.Endpoint("ENDPOINT_ID", api_key="specific_api_key") - -# This endpoint will always use the provided API key -result = endpoint.run_sync({"input": "data"}) +print(run_request) ``` -#### API Key Precedence +#### Per-Endpoint API Keys -The SDK uses this precedence order (highest to lowest): -1. Endpoint instance API key (if provided to `Endpoint()`) -2. Global API key (set via `runpod.api_key`) +Each `Endpoint` instance can use its own API key, useful for multi-tenant scenarios: ```python import runpod -# Example showing precedence -runpod.api_key = "GLOBAL_KEY" - -# This endpoint uses GLOBAL_KEY -endpoint1 = runpod.Endpoint("ENDPOINT_ID") - -# This endpoint uses ENDPOINT_KEY (overrides global) -endpoint2 = runpod.Endpoint("ENDPOINT_ID", api_key="ENDPOINT_KEY") +# Endpoint using a specific API key +endpoint1 = runpod.Endpoint("ENDPOINT_ID_1", api_key="CUSTOMER_A_KEY") +endpoint2 = runpod.Endpoint("ENDPOINT_ID_2", api_key="CUSTOMER_B_KEY") -# All requests from endpoint2 will use ENDPOINT_KEY result = endpoint2.run_sync({"input": "data"}) ``` @@ -250,7 +476,6 @@ def process_request(api_key, endpoint_id, input_data): endpoint = runpod.Endpoint(endpoint_id, api_key=api_key) return endpoint.run_sync(input_data) -# Safe concurrent usage with different API keys threads = [] for customer in customers: t = threading.Thread( @@ -297,9 +522,12 @@ runpod.terminate_pod(pod.id) ├── docs # Documentation ├── examples # Examples ├── runpod # Package source code -│ ├── api_wrapper # Language library - API (GraphQL) -│ ├── cli # Command Line Interface Functions +│ ├── api # Language library - API (GraphQL) +│ ├── apps # Apps SDK (App, queue/api/task, volumes, secrets) +│ ├── cli # Legacy CLI functions │ ├── endpoint # Language library - Endpoints +│ ├── rp_cli # The rp / runpod CLI +│ ├── runtimes # Worker runtime images (queue, api, task) │ └── serverless # SDK - Serverless Worker └── tests # Package tests ``` diff --git a/examples/apps/01_hello_world.py b/examples/apps/01_hello_world.py new file mode 100644 index 000000000..36d96506c --- /dev/null +++ b/examples/apps/01_hello_world.py @@ -0,0 +1,22 @@ +"""the smallest possible app: one queue function, one remote call. + + rp dev examples/apps/01_hello_world.py --once +""" + +import runpod +from runpod import App + +app = App("ex-hello") + + +@app.queue(cpu="cpu3c-1-2") +def hello(name: str): + print(f"saying hello to {name}") + return f"hello {name}" + + +@runpod.local_entrypoint +def main(): + result = hello.remote("world") + print("result:", result) + assert result == "hello world" diff --git a/examples/apps/02_invocation_styles.py b/examples/apps/02_invocation_styles.py new file mode 100644 index 000000000..803daf048 --- /dev/null +++ b/examples/apps/02_invocation_styles.py @@ -0,0 +1,37 @@ +"""every way to call a function: remote, async, spawn, local. + + rp dev examples/apps/02_invocation_styles.py --once +""" + +import asyncio + +import runpod +from runpod import App + +app = App("ex-invoke") + + +@app.queue(cpu="cpu3c-1-2") +def square(x: int): + return x * x + + +@runpod.local_entrypoint +def main(): + # sync remote call, blocks for the result + assert square.remote(4) == 16 + + # async variant, for use inside event loops + async def go(): + return await square.remote.aio(5) + + assert asyncio.run(go()) == 25 + + # fire and forget -> collect later + job = square.spawn(6) + assert job.result() == 36 + + # run in this process, no cloud involved + assert square.local(7) == 49 + + print("all four invocation styles ok") diff --git a/examples/apps/03_gpu.py b/examples/apps/03_gpu.py new file mode 100644 index 000000000..044dae88a --- /dev/null +++ b/examples/apps/03_gpu.py @@ -0,0 +1,33 @@ +"""gpu selection: shorthand names resolve to real devices. + + rp dev examples/apps/03_gpu.py --once +""" + +import runpod +from runpod import App + +app = App("ex-gpu") + + +@app.queue(gpu="4090", env={"MODE": "example"}) +def cuda_info(prompt: str): + import os + + import torch + + print(f"prompt: {prompt}") + return { + "cuda": torch.cuda.is_available(), + "device": torch.cuda.get_device_name(0), + "torch": str(torch.__version__), + "mode": os.environ.get("MODE"), + } + + +@runpod.local_entrypoint +def main(): + info = cuda_info.remote("hello gpu") + print("info:", info) + assert info["cuda"] is True + assert "4090" in info["device"] + assert info["mode"] == "example" diff --git a/examples/apps/04_dependencies.py b/examples/apps/04_dependencies.py new file mode 100644 index 000000000..9cc7fba16 --- /dev/null +++ b/examples/apps/04_dependencies.py @@ -0,0 +1,34 @@ +"""pip and apt dependencies, available on the worker without a custom image. + + rp dev examples/apps/04_dependencies.py --once +""" + +import runpod +from runpod import App + +app = App("ex-deps") + + +@app.queue(cpu="cpu3c-1-2", dependencies=["pyfiglet"]) +def banner(word: str): + import pyfiglet + + art = pyfiglet.figlet_format(word) + print(art) + return len(art) + + +@app.queue(cpu="cpu3c-1-2", system_dependencies=["jq"]) +def jq_version(): + import subprocess + + out = subprocess.run(["jq", "--version"], capture_output=True, text=True) + return out.stdout.strip() + + +@runpod.local_entrypoint +def main(): + assert banner.remote("hi") > 0 + version = jq_version.remote() + print("jq:", version) + assert version.startswith("jq-") diff --git a/examples/apps/05_api_service.py b/examples/apps/05_api_service.py new file mode 100644 index 000000000..47d1c8a05 --- /dev/null +++ b/examples/apps/05_api_service.py @@ -0,0 +1,41 @@ +"""a load-balanced http service from a class with route markers. + + rp dev examples/apps/05_api_service.py --once +""" + +import runpod +from runpod import App, get, init, post + +app = App("ex-api") + + +@app.api(cpu="cpu3c-1-2") +class Counter: + @init + def setup(self): + # runs once per worker before it takes traffic + self.count = 0 + print("counter initialized") + + @post("/bump") + async def bump(self, body: dict): + self.count += body.get("by", 1) + return {"count": self.count} + + @get("/value") + async def value(self): + return {"count": self.count} + + +@runpod.local_entrypoint +def main(): + first = Counter.post("/bump", {"by": 3}) + print("bump:", first) + assert first["count"] == 3 + + second = Counter.post("/bump", {"by": 2}) + assert second["count"] == 5 + + value = Counter.get("/value") + print("value:", value) + assert value["count"] == 5 diff --git a/examples/apps/06_tasks.py b/examples/apps/06_tasks.py new file mode 100644 index 000000000..afff8fe81 --- /dev/null +++ b/examples/apps/06_tasks.py @@ -0,0 +1,31 @@ +"""tasks: one ephemeral pod per call, terminated when done. + + rp dev examples/apps/06_tasks.py --once +""" + +import runpod +from runpod import App + +app = App("ex-tasks") + + +@app.task(cpu="cpu3c-1-2") +def multiply(x: int, y: int): + print(f"multiplying {x} * {y} on a dedicated pod") + return x * y + + +@app.task(gpu="4090") +def gpu_task(): + import torch + + print("checking cuda on a task pod") + return {"cuda": torch.cuda.is_available(), "torch": str(torch.__version__)} + + +@runpod.local_entrypoint +def main(): + assert multiply.remote(6, 7) == 42 + info = gpu_task.remote() + print("gpu task:", info) + assert info["cuda"] is True diff --git a/examples/apps/07_pipelines.py b/examples/apps/07_pipelines.py new file mode 100644 index 000000000..c859b4533 --- /dev/null +++ b/examples/apps/07_pipelines.py @@ -0,0 +1,37 @@ +"""functions calling functions: workers invoke sibling resources remotely. + + rp dev examples/apps/07_pipelines.py --once +""" + +import runpod +from runpod import App + +app = App("ex-pipeline") + + +@app.queue(cpu="cpu3c-1-2") +def tokenize(text: str): + tokens = text.split() + print(f"tokenized into {len(tokens)} tokens") + return tokens + + +@app.queue(cpu="cpu3c-1-2") +def count(tokens: list): + return len(tokens) + + +@app.queue(cpu="cpu3c-1-2") +def pipeline(text: str): + # nested .remote() calls run on the sibling resources + tokens = tokenize.remote(text) + total = count.remote(tokens) + return {"tokens": tokens, "total": total} + + +@runpod.local_entrypoint +def main(): + result = pipeline.remote("the quick brown fox") + print("pipeline:", result) + assert result["total"] == 4 + assert result["tokens"][0] == "the" diff --git a/examples/apps/08_volumes.py b/examples/apps/08_volumes.py new file mode 100644 index 000000000..e9a7bc7e2 --- /dev/null +++ b/examples/apps/08_volumes.py @@ -0,0 +1,39 @@ +"""shared network volume: one task writes, another reads. + +the volume is created on first use; placement co-locates both tasks +in the volume's datacenter automatically. + + rp dev examples/apps/08_volumes.py --once +""" + +import runpod +from runpod import App, Volume + +app = App("ex-volumes") + +scratch = Volume("ex-scratch", size=10) + + +@app.task(cpu="cpu3c-1-2", volume=scratch) +def write(content: str): + target = scratch.path / "message.txt" + target.write_text(content) + print(f"wrote {len(content)} bytes to {target}") + return str(target) + + +@app.task(cpu="cpu3c-1-2", volume=scratch) +def read(): + target = scratch.path / "message.txt" + content = target.read_text() + print(f"read {len(content)} bytes from {target}") + return content + + +@runpod.local_entrypoint +def main(): + message = "hello from the other pod" + write.remote(message) + result = read.remote() + print("read back:", result) + assert result == message diff --git a/examples/apps/09_secrets.py b/examples/apps/09_secrets.py new file mode 100644 index 000000000..fec3ff73c --- /dev/null +++ b/examples/apps/09_secrets.py @@ -0,0 +1,33 @@ +"""platform secrets injected as env vars, decrypted only on the worker. + +requires the secret to exist first: + + rp secret add ex-demo-secret --value "s3cret" + rp dev examples/apps/09_secrets.py --once +""" + +import runpod +from runpod import App, Secret + +app = App("ex-secrets") + + +@app.queue(cpu="cpu3c-1-2", env={"DEMO_TOKEN": Secret("ex-demo-secret"), "MODE": "example"}) +def peek(): + import os + + token = os.environ.get("DEMO_TOKEN", "") + # never print secrets; report shape only + return { + "mode": os.environ.get("MODE"), + "token_present": bool(token), + "token_length": len(token), + } + + +@runpod.local_entrypoint +def main(): + result = peek.remote() + print("peek:", result) + assert result["token_present"] is True + assert result["mode"] == "example" diff --git a/examples/apps/10_cached_models.py b/examples/apps/10_cached_models.py new file mode 100644 index 000000000..923118920 --- /dev/null +++ b/examples/apps/10_cached_models.py @@ -0,0 +1,33 @@ +"""platform-cached model weights: staged on the host before the worker starts. + + rp dev examples/apps/10_cached_models.py --once +""" + +import runpod +from runpod import App, Model + +app = App("ex-models") + +tiny = Model("sshleifer/tiny-gpt2") + + +@app.queue(gpu="4090", model=tiny, dependencies=["transformers"]) +def generate(prompt: str): + # weights are already on disk; no download happens here + from transformers import AutoModelForCausalLM, AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(str(tiny.path)) + model = AutoModelForCausalLM.from_pretrained(str(tiny.path)) + inputs = tokenizer(prompt, return_tensors="pt") + out = model.generate(**inputs, max_new_tokens=8) + text = tokenizer.decode(out[0]) + print(f"generated: {text!r}") + return {"path": str(tiny.path), "text": text} + + +@runpod.local_entrypoint +def main(): + result = generate.remote("hello") + print("result:", result) + assert result["path"].startswith("/runpod/model-store/huggingface/") + assert result["text"] diff --git a/examples/apps/11_train_eval.py b/examples/apps/11_train_eval.py new file mode 100644 index 000000000..b7e1eaf3b --- /dev/null +++ b/examples/apps/11_train_eval.py @@ -0,0 +1,69 @@ +"""a real workflow: train on one gpu pod, save to a volume, eval on another. + + rp dev examples/apps/11_train_eval.py --once +""" + +import runpod +from runpod import App, Volume + +app = App("ex-train-eval") + +models = Volume("ex-models", size=10) + + +@app.task(gpu="4090", volume=models) +def train(steps: int = 200): + import json + + import torch + import torch.nn as nn + + device = "cuda" if torch.cuda.is_available() else "cpu" + torch.manual_seed(0) + + # tiny regression problem: y = 3x + 1 with noise + x = torch.randn(2048, 1, device=device) + y = 3 * x + 1 + torch.randn_like(x) * 0.05 + + model = nn.Linear(1, 1).to(device) + opt = torch.optim.SGD(model.parameters(), lr=0.05) + for step in range(1, steps + 1): + loss = ((model(x) - y) ** 2).mean() + opt.zero_grad() + loss.backward() + opt.step() + if step % 50 == 0: + print(f"step {step}/{steps} loss={loss.item():.5f}") + + run_dir = models.path / "linear-run" + run_dir.mkdir(parents=True, exist_ok=True) + torch.save(model.state_dict(), run_dir / "model.pt") + (run_dir / "meta.json").write_text(json.dumps({"steps": steps})) + print(f"saved to {run_dir}") + return {"checkpoint": "linear-run", "loss": round(loss.item(), 5)} + + +@app.task(gpu="4090", volume=models) +def evaluate(checkpoint: str): + import torch + import torch.nn as nn + + device = "cuda" if torch.cuda.is_available() else "cpu" + model = nn.Linear(1, 1).to(device) + model.load_state_dict( + torch.load(models.path / checkpoint / "model.pt", map_location=device) + ) + weight = model.weight.item() + bias = model.bias.item() + print(f"learned y = {weight:.3f}x + {bias:.3f}") + return {"weight": round(weight, 2), "bias": round(bias, 2)} + + +@runpod.local_entrypoint +def main(): + out = train.remote() + print("train:", out) + fit = evaluate.remote(out["checkpoint"]) + print("eval:", fit) + assert abs(fit["weight"] - 3.0) < 0.3 + assert abs(fit["bias"] - 1.0) < 0.3 diff --git a/examples/apps/12_custom_image.py b/examples/apps/12_custom_image.py new file mode 100644 index 000000000..4697764f1 --- /dev/null +++ b/examples/apps/12_custom_image.py @@ -0,0 +1,24 @@ +"""a custom container image: the sdk bootstraps its runtime onto any +image with python3. + + rp dev examples/apps/12_custom_image.py --once +""" + +import runpod +from runpod import App + +app = App("ex-image") + + +@app.queue(cpu="cpu3c-1-2", image="python:3.12-slim") +def python_version(): + import sys + + return f"{sys.version_info.major}.{sys.version_info.minor}" + + +@runpod.local_entrypoint +def main(): + version = python_version.remote() + print("worker python:", version) + assert version == "3.12" diff --git a/examples/apps/README.md b/examples/apps/README.md new file mode 100644 index 000000000..f5b445938 --- /dev/null +++ b/examples/apps/README.md @@ -0,0 +1,51 @@ +# apps examples + +each file is a self-contained app demonstrating one feature, runnable +directly and asserting on its own results — the set doubles as an +end-to-end test suite against production. + +| example | shows | +|---|---| +| `01_hello_world` | one queue function, one `.remote()` call | +| `02_invocation_styles` | `.remote()`, `.remote.aio()`, `.spawn()`, `.local()` | +| `03_gpu` | gpu shorthand selection, env vars, torch/cuda | +| `04_dependencies` | pip `dependencies` and apt `system_dependencies` | +| `05_api_service` | `@app.api` class with `@init`/`@get`/`@post`, stateful workers | +| `06_tasks` | ephemeral pods (cpu and gpu) via `@app.task` | +| `07_pipelines` | workers calling sibling resources with nested `.remote()` | +| `08_volumes` | shared network volume between tasks, auto placement | +| `09_secrets` | platform secrets as env vars (needs `rp secret add ex-demo-secret`) | +| `10_cached_models` | platform-staged huggingface weights via `Model` | +| `11_train_eval` | train on one gpu pod, save to a volume, eval on another | +| `12_custom_image` | custom container image with runtime bootstrap | + +## running + +one example: + +```bash +rp dev examples/apps/01_hello_world.py --once +``` + +the whole suite: + +```bash +python examples/apps/run_all.py # sequential +python examples/apps/run_all.py --jobs 4 # parallel +python examples/apps/run_all.py 01 07 # subset by prefix +``` + +`09_secrets` needs a secret to exist first: + +```bash +rp secret add ex-demo-secret --value anything +``` + +`12_custom_image` bootstraps the worker runtime onto a bare image by +installing the runpod package at cold start. until the version with +`runpod.runtimes` is on pypi, point the bootstrap at a tarball of the +branch (plain https, not git+ — slim images have no git): + +```bash +export RUNPOD_PACKAGE_SPEC=https://github.com/runpod/runpod-python/archive/refs/heads/feat/apps-sdk.tar.gz +``` diff --git a/examples/apps/run_all.py b/examples/apps/run_all.py new file mode 100644 index 000000000..e55741931 --- /dev/null +++ b/examples/apps/run_all.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""run every example against prod as an end-to-end suite. + +each example is executed with `rp dev --once`, which provisions +real endpoints/pods, runs the example's local_entrypoint (every example +asserts on its results), and cleans up. requires `rp login` and, for +09_secrets, the demo secret: + + rp secret add ex-demo-secret --value anything + +usage: + python examples/apps/run_all.py # everything + python examples/apps/run_all.py 01 05 08 # by prefix + python examples/apps/run_all.py --jobs 4 # parallel +""" + +import argparse +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +HERE = Path(__file__).parent +TIMEOUT_SECONDS = 2400 + + +def find_examples(prefixes): + files = sorted( + p + for p in HERE.glob("[0-9][0-9]_*.py") + if not prefixes or any(p.name.startswith(pre) for pre in prefixes) + ) + if not files: + sys.exit(f"no examples match {prefixes}") + return files + + +def run_one(path: Path): + start = time.monotonic() + proc = subprocess.run( + ["rp", "dev", str(path), "--once"], + capture_output=True, + text=True, + timeout=TIMEOUT_SECONDS, + ) + elapsed = time.monotonic() - start + ok = proc.returncode == 0 + return path.name, ok, elapsed, proc.stdout + proc.stderr + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("prefixes", nargs="*", help="example prefixes, e.g. 01 08") + parser.add_argument("--jobs", type=int, default=1, help="parallel sessions") + args = parser.parse_args() + + files = find_examples(args.prefixes) + print(f"running {len(files)} examples (jobs={args.jobs})\n") + + results = [] + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + for name, ok, elapsed, output in pool.map(run_one, files): + mark = "PASS" if ok else "FAIL" + print(f" {mark} {name} ({elapsed:.0f}s)") + results.append((name, ok, output)) + + failures = [(n, out) for n, ok, out in results if not ok] + print(f"\n{len(results) - len(failures)}/{len(results)} passed") + for name, output in failures: + print(f"\n--- {name} ---") + print("\n".join(output.splitlines()[-30:])) + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 763f6148b..d796557ab 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -176,9 +176,17 @@ async def create_environment(self, app_id: str, name: str) -> Dict[str, Any]: # -- stock (placement) -- async def gpu_stock_status( - self, gpu_id: str, data_center_id: str, gpu_count: int = 1 + self, + gpu_id: str, + data_center_id: str, + gpu_count: int = 1, + pods: bool = False, ) -> Optional[str]: - """stock signal for a gpu device in one datacenter.""" + """stock signal for a gpu device in one datacenter. + + the serverless plane (includeAiApi) and the pod plane have + different availability; pods=True queries pod stock (tasks). + """ query = """ query GpuStock($gpuTypesInput: GpuTypeFilter, $lowestPriceInput: GpuLowestPriceInput) { gpuTypes(input: $gpuTypesInput) { @@ -194,7 +202,7 @@ async def gpu_stock_status( "dataCenterId": data_center_id, "gpuCount": gpu_count, "secureCloud": True, - "includeAiApi": True, + "includeAiApi": not pods, }, }, ) diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py index db5327904..cbcdb0854 100644 --- a/runpod/apps/placement.py +++ b/runpod/apps/placement.py @@ -35,20 +35,25 @@ def _hardware_keys(spec) -> List[Tuple[str, str]]: """(kind, id) stock lookup keys for a resource's hardware. gpu entries may be pool ids or device names; pools expand to their - device names since the stock api takes devices. + device names since the stock api takes devices. tasks run on pods, + whose gpu availability differs from the serverless plane, so their + keys carry a distinct kind and query pod stock. """ + from .spec import ResourceKind + if spec.is_cpu: return [("cpu", c) for c in spec.cpu or []] + gpu_kind = "gpu-pod" if spec.kind is ResourceKind.TASK else "gpu" gpu = spec.gpu if not gpu or any(str(g).lower() == "any" for g in gpu): - return [("gpu", "*")] + return [(gpu_kind, "*")] keys: List[Tuple[str, str]] = [] for entry in gpu: try: for device in GpuGroup(str(entry)).device_names(): - keys.append(("gpu", device)) + keys.append((gpu_kind, device)) except ValueError: - keys.append(("gpu", str(entry))) + keys.append((gpu_kind, str(entry))) return keys @@ -58,8 +63,10 @@ class StockMap: def __init__(self, api=None): self._api = api self._gpu: Dict[Tuple[str, str], int] = {} + self._gpu_pod: Dict[Tuple[str, str], int] = {} self._cpu: Dict[Tuple[str, str], int] = {} self._fetched_gpu: Set[str] = set() + self._fetched_gpu_pod: Set[str] = set() self._fetched_cpu: Set[str] = set() async def _client(self): @@ -71,11 +78,19 @@ async def _client(self): async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: """populate stock for the given hardware keys across all DCs.""" + keys = list(keys) gpu_ids = { k[1] for k in keys if k[0] == "gpu" and k[1] != "*" and k[1] not in self._fetched_gpu } + gpu_pod_ids = { + k[1] + for k in keys + if k[0] == "gpu-pod" + and k[1] != "*" + and k[1] not in self._fetched_gpu_pod + } cpu_ids = { k[1] for k in keys if k[0] == "cpu" and k[1] not in self._fetched_cpu } @@ -84,7 +99,11 @@ async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: for gpu_id in gpu_ids: self._fetched_gpu.add(gpu_id) for dc in DataCenter.all(): - jobs.append(self._fetch_gpu(client, gpu_id, dc.value)) + jobs.append(self._fetch_gpu(client, gpu_id, dc.value, False)) + for gpu_id in gpu_pod_ids: + self._fetched_gpu_pod.add(gpu_id) + for dc in DataCenter.all(): + jobs.append(self._fetch_gpu(client, gpu_id, dc.value, True)) for cpu_id in cpu_ids: self._fetched_cpu.add(cpu_id) for dc in DataCenter.all(): @@ -92,13 +111,16 @@ async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: if jobs: await asyncio.gather(*jobs) - async def _fetch_gpu(self, client, gpu_id: str, dc: str) -> None: + async def _fetch_gpu( + self, client, gpu_id: str, dc: str, pods: bool + ) -> None: try: - status = await client.gpu_stock_status(gpu_id, dc) + status = await client.gpu_stock_status(gpu_id, dc, pods=pods) except Exception: # noqa: BLE001 - stock is advisory log.debug("gpu stock query failed for %s@%s", gpu_id, dc, exc_info=True) status = None - self._gpu[(gpu_id, dc)] = _score(status) + target = self._gpu_pod if pods else self._gpu + target[(gpu_id, dc)] = _score(status) async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: try: @@ -110,14 +132,13 @@ async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: def score(self, key: Tuple[str, str], dc: str) -> int: kind, hw = key - if kind == "gpu": + if kind in ("gpu", "gpu-pod"): + table = self._gpu_pod if kind == "gpu-pod" else self._gpu if hw == "*": # any gpu: best signal among fetched devices, else assume ok - scores = [ - s for (g, d), s in self._gpu.items() if d == dc - ] + scores = [s for (g, d), s in table.items() if d == dc] return max(scores, default=1) - return self._gpu.get((hw, dc), 0) + return table.get((hw, dc), 0) return self._cpu.get((hw, dc), 0) diff --git a/tests/test_apps/test_placement.py b/tests/test_apps/test_placement.py index db4d64ff8..9c55c2101 100644 --- a/tests/test_apps/test_placement.py +++ b/tests/test_apps/test_placement.py @@ -23,7 +23,10 @@ def _spec(name="r", gpu=None, cpu=None, datacenter=None): def _stock(gpu=None, cpu=None): stock = StockMap(api=object()) - stock._gpu = gpu or {} + # task specs query the pod plane; mirror into both tables so the + # fixtures stay hardware-shaped rather than plane-shaped + stock._gpu = dict(gpu or {}) + stock._gpu_pod = dict(gpu or {}) stock._cpu = cpu or {} return stock From aefc85bea4aa8ab9cc24ad3b3f803f1d8e673f36 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:31:54 -0700 Subject: [PATCH 062/120] style: refine dev cli output --- runpod/apps/monitor.py | 4 +- runpod/apps/targets.py | 4 +- runpod/rp_cli/console.py | 104 +++++++++++++----- runpod/rp_cli/main.py | 16 +-- .../apps => tests/e2e/examples}/run_all.py | 6 +- tests/test_apps/test_monitor.py | 2 +- 6 files changed, 95 insertions(+), 41 deletions(-) rename {examples/apps => tests/e2e/examples}/run_all.py (91%) diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index 3aae241f8..b1b067315 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -277,10 +277,10 @@ def _line_before(raw: str, since_iso: str) -> bool: def format_worker_counts(counts: Dict[str, int]) -> str: - """human summary like '1 initializing · 2 ready'.""" + """human summary like '1 initializing, 2 ready'.""" parts = [ f"{count} {state}" for state, count in counts.items() if count ] - return " · ".join(parts) if parts else "no workers" + return ", ".join(parts) if parts else "no workers" diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index efbc328cc..66d3c2206 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -602,7 +602,7 @@ async def invoke( name = self.spec.name hardware = ",".join(self.spec.cpu or self.spec.gpu or ["any"]) emit(self.events, "dispatch", name) - emit(self.events, "task_status", name, f"provisioning pod · {hardware}") + emit(self.events, "task_status", name, f"provisioning pod on {hardware}") start = time.monotonic() stream = None @@ -614,7 +614,7 @@ async def invoke( self.events, "task_status", name, - f"pod {execution.pod_id[:12]} · waiting for runtime", + f"pod {execution.pod_id[:12]} waiting for runtime", ) if self.events is not None: from .monitor import PodLogStream diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 7f37180b4..81ecd7388 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -17,6 +17,7 @@ TaskID, TextColumn, ) +from rich.spinner import SPINNERS from rich.text import Text from rich.theme import Theme @@ -29,6 +30,49 @@ WARN = "#fbbf24" DIM = "grey58" +RUNPOD_SAND_SPINNER = "runpod_sand" +RUNPOD_SAND_SPINNER_FRAMES = [ + "⠁", + "⠂", + "⠄", + "⡀", + "⡈", + "⡐", + "⡠", + "⣀", + "⣁", + "⣂", + "⣄", + "⣌", + "⣔", + "⣤", + "⣥", + "⣦", + "⣮", + "⣶", + "⣷", + "⣿", + "⡿", + "⠿", + "⢟", + "⠟", + "⡛", + "⠛", + "⠫", + "⢋", + "⠋", + "⠍", + "⡉", + "⠉", + "⠑", + "⠡", + "⢁", +] +SPINNERS[RUNPOD_SAND_SPINNER] = { + "interval": 1000 / 16, + "frames": RUNPOD_SAND_SPINNER_FRAMES, +} + KIND_STYLES: Dict[str, str] = { "queue": "bold cyan", "api": "bold magenta", @@ -49,6 +93,15 @@ console = Console(highlight=False, theme=theme) + +def _spinner_column(finished_text: str = "[ok]✓[/ok]") -> SpinnerColumn: + return SpinnerColumn( + RUNPOD_SAND_SPINNER, + style=ACCENT, + finished_text=finished_text, + ) + + CONSOLE_URL = "https://console.runpod.io/serverless/user/endpoint" @@ -79,7 +132,7 @@ def _padded(name: str) -> str: def _pipe(name: str) -> str: - return f"[accent.light]{_padded(name)}[/accent.light] [dim]│[/dim]" + return f"[accent.light]{_padded(name)}[/accent.light][dim]:[/dim]" def kind_badge(kind: str) -> str: @@ -139,7 +192,7 @@ def request_failed(name: str, elapsed_s: float, err: Optional[str] = None) -> No def dev_banner(app_names: Iterable[str], module: str) -> None: console.print( - f"[bold white]dev[/bold white] [dim]{', '.join(app_names)} · {module}[/dim]" + f"[bold white]dev[/bold white] [dim]{', '.join(app_names)} in {module}[/dim]" ) @@ -151,7 +204,7 @@ def deploy_plan(apps: Iterable[Tuple[str, str, int]]) -> None: ) width = max(len(name) for name, _, _ in apps) for name, source, count in apps: - detail = f"{source} · " if source else "" + detail = f"{source}, " if source else "" console.print( f" [white]{name:<{width}}[/white] " f"[dim]{detail}{count} resource{'s' if count != 1 else ''}[/dim]" @@ -166,10 +219,10 @@ def deploy_banner( ) -> None: """resources: (name, kind) pairs.""" names = ", ".join(name for name, _ in resources) or "(no resources)" - origin = f" · {source}" if source else "" + origin = f" from {source}" if source else "" console.print( f"[bold white]deploy[/bold white] [white]{app_name}[/white] " - f"[dim]→ {env}{origin} · {names}[/dim]" + f"[dim]→ {env}{origin}, {names}[/dim]" ) @@ -233,7 +286,7 @@ class DeployEvents: def __init__(self) -> None: self._progress = Progress( - SpinnerColumn("dots", style=ACCENT, finished_text=f"[ok]✓[/ok]"), + _spinner_column(), TextColumn("[white]{task.description}[/white]"), TextColumn("{task.fields[detail]}"), console=console, @@ -284,7 +337,7 @@ def vendor_progress(self, count: int, package: str) -> None: self._package_count = count self._progress.update( self._current, - detail=f"[accent.light]{package}[/accent.light] [dim]· {count}[/dim]", + detail=f"[accent.light]{package}[/accent.light] [dim]({count})[/dim]", ) def upload_progress(self, sent: int, total: int) -> None: @@ -331,7 +384,7 @@ def __init__(self) -> None: def session_starting(self) -> None: self._progress = Progress( - SpinnerColumn("dots", style=ACCENT, finished_text="[ok]✓[/ok]"), + _spinner_column(), TextColumn("[white]{task.description}[/white]"), TextColumn("[dim]{task.fields[detail]}[/dim]"), console=console, @@ -357,7 +410,7 @@ def _row(self, name: str, detail: str) -> None: self._progress.update(self._tasks[name], detail=detail) def provisioning(self, name: str, kind: str, hardware: str) -> None: - self._row(name, f"provisioning {kind} · {hardware}") + self._row(name, f"provisioning {kind} on {hardware}") def adopted(self, name: str, endpoint_id: str) -> None: self._row(name, "adopting") @@ -376,7 +429,7 @@ def deleted(self, name: str) -> None: def resource_added(self, name: str, kind: str, hardware: str) -> None: console.print( f" [ok]+[/ok] [white]{name}[/white] " - f"[dim]{kind} · {hardware}[/dim]" + f"[dim]{kind} on {hardware}[/dim]" ) def resource_changed(self, name: str, fields: List[str]) -> None: @@ -391,32 +444,33 @@ def resource_removed(self, name: str) -> None: # -- request lifecycle (LiveTarget events) -- # next.js/wrangler-style event lines: leading status glyph, the # function call in normal weight, everything else dim, timing at - # the end. function stdout renders as `name │ line` so concurrent + # the end. function stdout renders as `name: line` so concurrent # calls interleave without losing attribution. def dispatch(self, name: str, label: str = "") -> None: detail = f" [dim]{label}[/dim]" if label else "" - console.print(f" [accent]○[/accent] [white]{name}()[/white]{detail}") + console.print(f" [accent]→[/accent] [white]{name}()[/white]{detail}") def worker_status(self, name: str, counts: Dict[str, int]) -> None: from runpod.apps.monitor import format_worker_counts console.print( - f" [dim]○ {name}() waiting · {format_worker_counts(counts)}[/dim]" + f" [dim]↳ {name}() waiting: {format_worker_counts(counts)}[/dim]" ) def task_status(self, name: str, detail: str) -> None: - console.print(f" [dim]○ {name}() {detail}[/dim]") + console.print(f" [dim]↳ {name}() {detail}[/dim]") def volume_created(self, name: str, size: int, dc: str) -> None: console.print( - f" [ok]+[/ok] [white]volume {name}[/white] [dim]{size}GB · {dc}[/dim]" + f" [ok]+[/ok] [white]volume {name}[/white] [dim]{size}GB in {dc}[/dim]" ) def worker_ready(self, name: str, worker_id: str) -> None: + worker = worker_id[:12] if worker_id else "pod" console.print( - f" [accent]●[/accent] [white]{name}()[/white] [dim]running on " - f"worker {worker_id[:12]}[/dim]" + f" [accent]↳[/accent] [white]{name}()[/white] [dim]worker " + f"{worker} ready[/dim]" ) def worker_log(self, name: str, line: str) -> None: @@ -452,7 +506,7 @@ def cleanup_started(self, total: int) -> None: return console.print() self._progress = Progress( - SpinnerColumn("dots", style=ACCENT, finished_text="[ok]✓[/ok]"), + _spinner_column(), TextColumn("[white]{task.description}[/white]"), TextColumn("[dim]{task.fields[detail]}[/dim]"), console=console, @@ -465,7 +519,7 @@ def cleanup_started(self, total: int) -> None: def deleting(self, name: str) -> None: if self._progress is not None and self._task is not None: self._progress.update( - self._task, detail=f"{name} · {self._done}/{self._total}" + self._task, detail=f"{name} ({self._done}/{self._total})" ) def deleted(self, name: str) -> None: @@ -474,7 +528,7 @@ def deleted(self, name: str) -> None: self._progress.update( self._task, completed=self._done, - detail=f"{name} · {self._done}/{self._total}", + detail=f"{name} ({self._done}/{self._total})", ) def delete_failed(self, name: str) -> None: @@ -521,7 +575,7 @@ def entrypoint_header(fn_name: str = "") -> None: console.print() if fn_name: console.print( - f" [accent]▸[/accent] [white]{fn_name}()[/white] " + f" [accent]◆[/accent] [white]{fn_name}()[/white] " f"[dim]local entrypoint[/dim]" ) @@ -529,8 +583,8 @@ def entrypoint_header(fn_name: str = "") -> None: def entrypoint_success(elapsed: float) -> None: console.print() console.print( - f" [ok]✓[/ok] [white]done[/white] [dim]in {_fmt_elapsed(elapsed)} " - f"· enter re-run · edit reload · ^C quit[/dim]" + f" [ok]✓[/ok] [white]done[/white] [dim]in {_fmt_elapsed(elapsed)}, " + f"enter re-run, edit reload, ^C quit[/dim]" ) @@ -538,6 +592,6 @@ def entrypoint_failure(elapsed: float, err: str) -> None: console.print() console.print(f" [err]✗ {err}[/err]") console.print( - f" [err]✗[/err] [white]failed[/white] [dim]in {_fmt_elapsed(elapsed)} " - f"· enter re-run · edit reload · ^C quit[/dim]" + f" [err]✗[/err] [white]failed[/white] [dim]in {_fmt_elapsed(elapsed)}, " + f"enter re-run, edit reload, ^C quit[/dim]" ) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 3c5e89ec9..bbe930b58 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -401,7 +401,7 @@ def _show_url(url: str) -> None: set_credentials(api_key, overwrite=True) except (LoginError, ValueError) as exc: _fail(str(exc)) - ui.success("logged in · credentials saved to [dim]~/.runpod/config.toml[/dim]") + ui.success("logged in, credentials saved to [dim]~/.runpod/config.toml[/dim]") # ---------------------------------------------------------- app / env @@ -431,7 +431,7 @@ def app_list(): raise click.ClickException(str(exc)) from exc if not apps: - ui.console.print("\n no apps deployed · run [white]rp deploy[/white]\n") + ui.console.print("\n no apps deployed. run [white]rp deploy[/white]\n") return width = max(len(a["name"]) for a in apps) @@ -529,7 +529,7 @@ def registry_add(name, username, password): except Exception as exc: # noqa: BLE001 - surface engine errors cleanly raise click.ClickException(str(exc)) from exc ui.success( - f"registry credential [white]{name}[/white] created · " + f"registry credential [white]{name}[/white] created, " f'[dim]use registry_auth="{name}"[/dim]' ) @@ -547,7 +547,7 @@ def registry_list(): if not creds: ui.console.print( - "\n no registry credentials · run [white]rp registry add [/white]\n" + "\n no registry credentials. run [white]rp registry add [/white]\n" ) return ui.console.print() @@ -611,7 +611,7 @@ def secret_add(name, value, description): except Exception as exc: # noqa: BLE001 - surface engine errors cleanly raise click.ClickException(str(exc)) from exc ui.success( - f"secret [white]{name}[/white] created · " + f"secret [white]{name}[/white] created, " f"[dim]use runpod.Secret(\"{name}\")[/dim]" ) @@ -628,7 +628,7 @@ def secret_list(): raise click.ClickException(str(exc)) from exc if not secrets: - ui.console.print("\n no secrets · run [white]rp secret add [/white]\n") + ui.console.print("\n no secrets. run [white]rp secret add [/white]\n") return ui.console.print() width = max(len(s["name"]) for s in secrets) @@ -703,7 +703,7 @@ def env_list(app_name): envs = entry.get("flashEnvironments") or [] if not envs: ui.console.print( - f"\n no environments in [white]{name}[/white] · run [white]rp deploy[/white]\n" + f"\n no environments in [white]{name}[/white]. run [white]rp deploy[/white]\n" ) return ui.console.print() @@ -712,7 +712,7 @@ def env_list(app_name): build = (env_entry.get("activeBuildId") or "-")[:12] ui.console.print( f" [white]{env_entry['name']:<{width}}[/white] " - f"[dim]{build} · {_fmt_ts(env_entry.get('createdAt'))}[/dim]" + f"[dim]{build}, {_fmt_ts(env_entry.get('createdAt'))}[/dim]" ) ui.console.print() diff --git a/examples/apps/run_all.py b/tests/e2e/examples/run_all.py similarity index 91% rename from examples/apps/run_all.py rename to tests/e2e/examples/run_all.py index e55741931..f1bca092f 100644 --- a/examples/apps/run_all.py +++ b/tests/e2e/examples/run_all.py @@ -9,9 +9,9 @@ rp secret add ex-demo-secret --value anything usage: - python examples/apps/run_all.py # everything - python examples/apps/run_all.py 01 05 08 # by prefix - python examples/apps/run_all.py --jobs 4 # parallel + python tests/e2e/examples/run_all.py # everything + python tests/e2e/examples/run_all.py 01 05 08 # by prefix + python tests/e2e/examples/run_all.py --jobs 4 # parallel """ import argparse diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index 787366f65..1cdd77f61 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -45,7 +45,7 @@ def worker_ready(self, *a): class TestFormatWorkerCounts: def test_only_nonzero(self): s = format_worker_counts({"initializing": 1, "ready": 2, "throttled": 0}) - assert s == "1 initializing · 2 ready" + assert s == "1 initializing, 2 ready" def test_empty(self): assert format_worker_counts({"ready": 0}) == "no workers" From db500aa6e9137c30b073a7ebb9ef1a10822012e0 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:33:46 -0700 Subject: [PATCH 063/120] style: restore log stream pipe --- runpod/rp_cli/console.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 81ecd7388..8b766a123 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -132,7 +132,7 @@ def _padded(name: str) -> str: def _pipe(name: str) -> str: - return f"[accent.light]{_padded(name)}[/accent.light][dim]:[/dim]" + return f"[accent.light]{_padded(name)}[/accent.light] [dim]│[/dim]" def kind_badge(kind: str) -> str: @@ -444,7 +444,7 @@ def resource_removed(self, name: str) -> None: # -- request lifecycle (LiveTarget events) -- # next.js/wrangler-style event lines: leading status glyph, the # function call in normal weight, everything else dim, timing at - # the end. function stdout renders as `name: line` so concurrent + # the end. function stdout renders as `name │ line` so concurrent # calls interleave without losing attribution. def dispatch(self, name: str, label: str = "") -> None: From b9d5ef808cad9c612b9b794fa08f748a3e990047 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:11:54 -0700 Subject: [PATCH 064/120] feat: add endpoint config params, rp init, rp update, and build-only deploys --- runpod/apps/app.py | 43 +++ runpod/apps/deploy.py | 56 +++- runpod/apps/dev.py | 25 +- runpod/apps/init.py | 80 ++++++ runpod/apps/spec.py | 102 ++++++++ runpod/apps/targets.py | 2 + runpod/apps/tasks.py | 23 +- runpod/rp_cli/main.py | 132 +++++++++- runpod/rp_cli/update.py | 209 +++++++++++++++ runpod/runtimes/queue/worker.py | 22 +- tests/test_apps/test_config_params.py | 364 ++++++++++++++++++++++++++ tests/test_cli/test_init_command.py | 82 ++++++ tests/test_cli/test_update_command.py | 133 ++++++++++ 13 files changed, 1256 insertions(+), 17 deletions(-) create mode 100644 runpod/apps/init.py create mode 100644 runpod/rp_cli/update.py create mode 100644 tests/test_apps/test_config_params.py create mode 100644 tests/test_cli/test_init_command.py create mode 100644 tests/test_cli/test_update_command.py diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 7913cc0db..551cb0dbd 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -22,10 +22,13 @@ async def transcribe(audio_url: str): ... from .gpu import GpuLike from .handles import ApiHandle, FunctionHandle from .spec import ( + DEFAULT_SCALER_VALUE, ResourceKind, ResourceSpec, normalize_cpu, + normalize_cuda_version, normalize_gpu, + normalize_scaler_type, normalize_workers, ) from .targets import InvocationTarget, PodTarget, SentinelTarget @@ -94,12 +97,24 @@ def queue( image: Optional[str] = None, registry_auth: Optional[str] = None, model: Optional[Any] = None, + max_concurrency: int = 1, + execution_timeout_ms: int = 0, + flashboot: bool = True, + scaler_type: Optional[str] = None, + scaler_value: int = DEFAULT_SCALER_VALUE, + min_cuda_version: Optional[str] = None, + accelerate_downloads: bool = True, + container_disk_gb: Optional[int] = None, ) -> Callable[[Callable], FunctionHandle]: """declare a queue-based serverless endpoint from a function. image selects a custom base image for the workers; the deployed code still arrives via the build artifact and is booted by the runtime bootstrap, so any image with a python3 binary works. + + max_concurrency lets one worker process several jobs at once; + values above 1 only achieve real concurrency with async + functions. execution_timeout_ms caps a single job (0 = no cap). """ def decorator(fn: Callable) -> FunctionHandle: @@ -119,6 +134,14 @@ def decorator(fn: Callable) -> FunctionHandle: image=image, registry_auth=registry_auth, model=model, + max_concurrency=max_concurrency, + execution_timeout_ms=execution_timeout_ms, + flashboot=flashboot, + scaler_type=normalize_scaler_type(scaler_type), + scaler_value=scaler_value, + min_cuda_version=normalize_cuda_version(min_cuda_version), + accelerate_downloads=accelerate_downloads, + container_disk_gb=container_disk_gb, ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -141,6 +164,9 @@ def task( registry_auth: Optional[str] = None, model: Optional[Any] = None, datacenter: Optional[Union[str, List[str]]] = None, + min_cuda_version: Optional[str] = None, + accelerate_downloads: bool = True, + container_disk_gb: Optional[int] = None, ) -> Callable[[Callable], FunctionHandle]: """declare ephemeral pod compute from a function. @@ -164,6 +190,9 @@ def decorator(fn: Callable) -> FunctionHandle: registry_auth=registry_auth, model=model, datacenter=_datacenter_list(datacenter), + min_cuda_version=normalize_cuda_version(min_cuda_version), + accelerate_downloads=accelerate_downloads, + container_disk_gb=container_disk_gb, ) handle = FunctionHandle(self, fn, spec) self._register(spec.name, handle) @@ -188,6 +217,13 @@ def api( image: Optional[str] = None, registry_auth: Optional[str] = None, model: Optional[Any] = None, + execution_timeout_ms: int = 0, + flashboot: bool = True, + scaler_type: Optional[str] = None, + scaler_value: int = DEFAULT_SCALER_VALUE, + min_cuda_version: Optional[str] = None, + accelerate_downloads: bool = True, + container_disk_gb: Optional[int] = None, ) -> Callable[[Any], ApiHandle]: """declare a load-balanced serverless endpoint. @@ -228,6 +264,13 @@ def decorator(target: Any) -> ApiHandle: image=image, registry_auth=registry_auth, model=model, + execution_timeout_ms=execution_timeout_ms, + flashboot=flashboot, + scaler_type=normalize_scaler_type(scaler_type), + scaler_value=scaler_value, + min_cuda_version=normalize_cuda_version(min_cuda_version), + accelerate_downloads=accelerate_downloads, + container_disk_gb=container_disk_gb, ) handle = ApiHandle(self, target, spec) self._register(spec.name, handle) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 5660c4eeb..63901ae8e 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -215,6 +215,8 @@ def package_project( def _container_disk_gb(spec) -> int: + if spec.container_disk_gb: + return spec.container_disk_gb return CPU_CONTAINER_DISK_GB if spec.is_cpu else GPU_CONTAINER_DISK_GB @@ -271,17 +273,18 @@ def _deployed_endpoint_input( if tag and "RUNPOD_RUNTIME_TAG" not in template_env: template_env["RUNPOD_RUNTIME_TAG"] = tag + if spec.max_concurrency > 1: + template_env["RUNPOD_MAX_CONCURRENCY"] = str(spec.max_concurrency) + payload: Dict[str, Any] = { "name": spec.name, "flashEnvironmentId": environment_id, "workersMin": spec.workers[0], "workersMax": spec.workers[1], "idleTimeout": spec.idle_timeout, - "scalerType": ( - "REQUEST_COUNT" if spec.kind is ResourceKind.API else "QUEUE_DELAY" - ), - "scalerValue": 4, - "flashBootType": "FLASHBOOT", + "scalerType": spec.effective_scaler_type, + "scalerValue": spec.scaler_value, + "executionTimeoutMs": spec.execution_timeout_ms, "template": { "name": f"{app.name}-{spec.name}-template", "imageName": image_for_spec(spec, python_version=python_version), @@ -292,6 +295,8 @@ def _deployed_endpoint_input( ], }, } + if spec.flashboot: + payload["flashBootType"] = "FLASHBOOT" if spec.image: # custom image: inject the bootstrap; the vendored env in the @@ -331,6 +336,8 @@ def _deployed_endpoint_input( payload["gpuIds"] = gpu_ids_value(spec.gpu) payload["gpuCount"] = spec.gpu_count + if spec.min_cuda_version: + payload["minCudaVersion"] = spec.min_cuda_version if not spec.datacenter: # artifact delivery rides the flash volume (network # storage); machines outside storage-cluster regions fail @@ -432,6 +439,45 @@ def _phase(events, name: str, detail: str = "") -> None: handler(name, detail) +def build_artifact( + app: App, + project_root: Path, + *, + python_version: str = DEFAULT_PYTHON_VERSION, + exclude: Optional[List[str]] = None, + events: Optional[object] = None, + output: Optional[Path] = None, +) -> Path: + """vendor, build the manifest, and package one app's artifact.""" + _phase(events, "vendor", f"python {python_version}") + build: BuildResult = build_environment( + app, + project_root, + python_version=python_version, + exclude=exclude, + events=events, + ) + manifest = build_manifest( + app, + project_root, + python_version=python_version, + excluded_packages=build.excluded, + ) + _phase(events, "package") + tar_path = package_project( + project_root, manifest, output=output, env_dir=build.env_dir + ) + size_mb = tar_path.stat().st_size / (1024 * 1024) + if size_mb > MAX_ARTIFACT_MB: + raise BuildError( + f"artifact is {size_mb:.0f} MB (limit {MAX_ARTIFACT_MB} MB). " + f"exclude large packages with --exclude (they must then come " + f"from the worker image) or trim project files via .runpodignore" + ) + log.info("packaged %s (%.1f MB)", app.name, size_mb) + return tar_path + + async def deploy_app( app: App, project_root: Path, diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 1a65610f8..8a0d3cbed 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -142,15 +142,14 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: "workersMin": spec.workers[0], "workersMax": spec.workers[1], "idleTimeout": spec.idle_timeout, - "scalerType": ( - "REQUEST_COUNT" if spec.kind is ResourceKind.API else "QUEUE_DELAY" - ), - "scalerValue": 4, - "flashBootType": "FLASHBOOT", + "scalerType": spec.effective_scaler_type, + "scalerValue": spec.scaler_value, + "executionTimeoutMs": spec.execution_timeout_ms, "template": { "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), - "containerDiskInGb": 10 if spec.is_cpu else 30, + "containerDiskInGb": spec.container_disk_gb + or (10 if spec.is_cpu else 30), "dockerArgs": "", "env": [ {"key": GENERATION_ENV, "value": str(generation)}, @@ -163,6 +162,16 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: else [] ), {"key": "RUNPOD_DEV_APP", "value": app.name}, + *( + [ + { + "key": "RUNPOD_MAX_CONCURRENCY", + "value": str(spec.max_concurrency), + } + ] + if spec.max_concurrency > 1 + else [] + ), *( {"key": k, "value": v} for k, v in _render_env(spec.env).items() @@ -170,6 +179,8 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: ], }, } + if spec.flashboot: + payload["flashBootType"] = "FLASHBOOT" if spec.image: # custom image: inject the bootstrap so the worker runtime starts @@ -206,6 +217,8 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: payload["gpuIds"] = gpu_ids_value(spec.gpu) payload["gpuCount"] = spec.gpu_count + if spec.min_cuda_version: + payload["minCudaVersion"] = spec.min_cuda_version return payload diff --git a/runpod/apps/init.py b/runpod/apps/init.py new file mode 100644 index 000000000..b7a59edce --- /dev/null +++ b/runpod/apps/init.py @@ -0,0 +1,80 @@ +"""project scaffolding for `rp init`. + +writes a minimal app project: a main module with one queue function and +a local entrypoint, a requirements file, and a .runpodignore. existing +files are never overwritten unless the caller says so. +""" + +from pathlib import Path +from typing import Dict, List + +MAIN_TEMPLATE = '''"""{name}: a Runpod app. + +Run it live: + + rp dev main.py + +Deploy it: + + rp deploy +""" + +import runpod +from runpod import App + +app = App("{name}") + + +@app.queue(cpu="cpu3c-1-2") +def hello(name: str): + # this print streams back to your terminal during rp dev + print(f"running in the cloud, greeting {{name}}") + return f"hello {{name}}!" + + +@runpod.local_entrypoint +def main(): + # runs on your machine; hello() runs on a cloud worker + print(hello.remote("world")) +''' + +REQUIREMENTS_TEMPLATE = """# packages your functions need on the workers +# (also installed locally for rp dev) +""" + +RUNPODIGNORE_TEMPLATE = """# excluded from the deploy artifact +.git +.venv +__pycache__ +*.pyc +.env +""" + +PROJECT_FILES: Dict[str, str] = { + "main.py": MAIN_TEMPLATE, + "requirements.txt": REQUIREMENTS_TEMPLATE, + ".runpodignore": RUNPODIGNORE_TEMPLATE, +} + + +def detect_conflicts(project_dir: Path) -> List[str]: + """names of skeleton files that already exist in project_dir.""" + return [ + name for name in PROJECT_FILES if (project_dir / name).exists() + ] + + +def create_project( + project_dir: Path, name: str, *, overwrite: bool = False +) -> List[Path]: + """write the project skeleton; returns the files written.""" + project_dir.mkdir(parents=True, exist_ok=True) + written: List[Path] = [] + for filename, template in PROJECT_FILES.items(): + path = project_dir / filename + if path.exists() and not overwrite: + continue + content = template.format(name=name) if "{name}" in template else template + path.write_text(content, encoding="utf-8") + written.append(path) + return written diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 881875b6c..7e5cdb485 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -14,6 +14,27 @@ DEFAULT_WORKERS: Tuple[int, int] = (0, 3) +SCALER_TYPES = frozenset({"QUEUE_DELAY", "REQUEST_COUNT"}) +DEFAULT_SCALER_VALUE = 4 + +# cuda versions the platform can filter hosts by +CUDA_VERSIONS = frozenset( + { + "11.8", + "12.0", + "12.1", + "12.2", + "12.3", + "12.4", + "12.5", + "12.6", + "12.7", + "12.8", + "12.9", + "13.0", + } +) + class ResourceKind(str, Enum): QUEUE = "queue" @@ -98,6 +119,32 @@ def normalize_cpu( ) +def normalize_scaler_type(scaler_type: Optional[str]) -> Optional[str]: + """validate a scaler type string, tolerating lowercase.""" + if scaler_type is None: + return None + normalized = str(scaler_type).strip().upper() + if normalized not in SCALER_TYPES: + raise InvalidResourceError( + f"scaler_type must be one of {sorted(SCALER_TYPES)}, " + f"got {scaler_type!r}" + ) + return normalized + + +def normalize_cuda_version(version: Optional[str]) -> Optional[str]: + """validate a minimum cuda version string.""" + if version is None: + return None + normalized = str(version).strip() + if normalized not in CUDA_VERSIONS: + raise InvalidResourceError( + f"min_cuda_version must be one of " + f"{', '.join(sorted(CUDA_VERSIONS))}, got {version!r}" + ) + return normalized + + @dataclass class ResourceSpec: """declarative config for one app resource.""" @@ -118,6 +165,14 @@ class ResourceSpec: registry_auth: Optional[str] = None model: Optional[Any] = None schedule: Optional[str] = None + max_concurrency: int = 1 + execution_timeout_ms: int = 0 + flashboot: bool = True + scaler_type: Optional[str] = None + scaler_value: int = DEFAULT_SCALER_VALUE + min_cuda_version: Optional[str] = None + accelerate_downloads: bool = True + container_disk_gb: Optional[int] = None routes: List[RouteSpec] = field(default_factory=list) asgi_factory: Optional[str] = None @@ -128,11 +183,42 @@ def __post_init__(self) -> None: ) if not self.name: raise InvalidResourceError("resource name must not be empty") + if self.max_concurrency < 1: + raise InvalidResourceError( + f"resource '{self.name}': max_concurrency must be >= 1, " + f"got {self.max_concurrency}" + ) + if self.execution_timeout_ms < 0: + raise InvalidResourceError( + f"resource '{self.name}': execution_timeout_ms must be >= 0" + ) + if self.scaler_value < 1: + raise InvalidResourceError( + f"resource '{self.name}': scaler_value must be >= 1" + ) + if self.container_disk_gb is not None and self.container_disk_gb < 1: + raise InvalidResourceError( + f"resource '{self.name}': container_disk_gb must be >= 1" + ) + if self.min_cuda_version is not None and self.is_cpu: + raise InvalidResourceError( + f"resource '{self.name}': min_cuda_version has no effect " + f"on cpu resources" + ) @property def is_cpu(self) -> bool: return self.cpu is not None + @property + def effective_scaler_type(self) -> str: + """the scaler the endpoint deploys with: explicit or kind default.""" + if self.scaler_type: + return self.scaler_type + return ( + "REQUEST_COUNT" if self.kind is ResourceKind.API else "QUEUE_DELAY" + ) + def to_manifest(self) -> Dict[str, Any]: """serialize for the deploy manifest.""" data: Dict[str, Any] = { @@ -171,6 +257,22 @@ def to_manifest(self) -> Dict[str, Any]: data["model"] = model_reference(self.model) if self.schedule: data["schedule"] = self.schedule + if self.max_concurrency != 1: + data["maxConcurrency"] = self.max_concurrency + if self.execution_timeout_ms: + data["executionTimeoutMs"] = self.execution_timeout_ms + if not self.flashboot: + data["flashboot"] = False + if self.scaler_type: + data["scalerType"] = self.scaler_type + if self.scaler_value != DEFAULT_SCALER_VALUE: + data["scalerValue"] = self.scaler_value + if self.min_cuda_version: + data["minCudaVersion"] = self.min_cuda_version + if not self.accelerate_downloads: + data["accelerateDownloads"] = False + if self.container_disk_gb: + data["containerDiskGb"] = self.container_disk_gb if self.routes: data["routes"] = [ {"method": r.method, "path": r.path, "handler": r.handler_name} diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 66d3c2206..bc900d4d2 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -411,6 +411,7 @@ def build_payload( kwargs=serialize_kwargs(kwargs), dependencies=spec.dependencies, system_dependencies=spec.system_dependencies, + accelerate_downloads=spec.accelerate_downloads, ) return {"input": request.to_input()} @@ -588,6 +589,7 @@ def build_payload( kwargs=serialize_kwargs(kwargs), dependencies=spec.dependencies, system_dependencies=spec.system_dependencies, + accelerate_downloads=spec.accelerate_downloads, ) return request.to_input() diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 1f4e47221..8522c7715 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -55,6 +55,20 @@ def _bootstrap_command() -> str: return shell_launcher("RUNPOD_TASK_RUNNER_B64", "/task_runner.py") +def cuda_versions_at_least(minimum: str) -> List[str]: + """all platform cuda versions >= minimum, ascending.""" + from .spec import CUDA_VERSIONS + + def _key(version: str): + major, minor = version.split(".") + return (int(major), int(minor)) + + floor = _key(minimum) + return sorted( + (v for v in CUDA_VERSIONS if _key(v) >= floor), key=_key + ) + + def _device_names(gpu: Optional[List[str]]) -> List[str]: """pods take exact device names, not pool ids; expand pool ids (e.g. ADA_24 -> NVIDIA GeForce RTX 4090) and pass device names @@ -105,7 +119,8 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] spec, python_version=local_python_version() ), "ports": f"{TASK_PORT}/http", - "containerDiskInGb": 10 if spec.is_cpu else 30, + "containerDiskInGb": spec.container_disk_gb + or (10 if spec.is_cpu else 30), "terminateAfter": terminate_after, "supportPublicIp": True, } @@ -135,6 +150,12 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] else: pod["gpuTypeIdList"] = _device_names(spec.gpu) pod["gpuCount"] = spec.gpu_count + if spec.min_cuda_version: + # pods have no min-version filter; allow every version at + # or above the requested floor + pod["allowedCudaVersions"] = cuda_versions_at_least( + spec.min_cuda_version + ) return pod diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index bbe930b58..b337484d0 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -15,9 +15,19 @@ import click +# commands whose output would be polluted by (or that manage) the +# update notice +_UPDATE_CHECK_EXCLUDED = frozenset({"dev", "update"}) + + @click.group() -def cli(): +@click.pass_context +def cli(ctx): """Runpod CLI: deploy and manage apps, endpoints, and pods.""" + if ctx.invoked_subcommand not in _UPDATE_CHECK_EXCLUDED: + from runpod.rp_cli.update import start_background_check + + start_background_check() def _fail(message: str) -> None: @@ -35,6 +45,92 @@ def _app_source(found, project_root: Path) -> str: return str(source) +# ------------------------------------------------------------------ init + + +@cli.command() +@click.argument("project_name", required=False) +@click.option("--force", "-f", is_flag=True, help="Overwrite existing files.") +def init(project_name, force): + """Create a new app project (PROJECT_NAME, or '.' for the cwd).""" + from runpod.apps.init import create_project, detect_conflicts + from runpod.rp_cli import console as ui + + if project_name is None or project_name == ".": + project_dir = Path.cwd() + name = project_dir.name + else: + project_dir = Path(project_name) + name = project_name + + conflicts = detect_conflicts(project_dir) + if conflicts and not force: + listing = ", ".join(conflicts) + _fail( + f"{listing} already exist{'s' if len(conflicts) == 1 else ''} " + f"in {project_dir}. pass --force to overwrite." + ) + + written = create_project(project_dir, name, overwrite=force) + ui.success( + f"initialized [white]{name}[/white] " + f"[dim]{len(written)} files[/dim]" + ) + ui.console.print() + if project_dir != Path.cwd(): + ui.console.print(f" [dim]cd {project_name}[/dim]") + ui.console.print(" [dim]rp login[/dim]") + ui.console.print(" [dim]rp dev main.py[/dim]") + ui.console.print() + + +# ---------------------------------------------------------------- update + + +@cli.command() +@click.option( + "--version", "-V", "version_opt", default=None, help="Target version." +) +def update(version_opt): + """Update the runpod package to the latest (or a given) version.""" + from runpod.rp_cli import console as ui + from runpod.rp_cli.update import ( + compare_versions, + current_version, + fetch_pypi_metadata, + parse_version, + run_install, + ) + + current = current_version() + ui.console.print(f"current version: [white]{current}[/white]") + + with ui.console.status("[dim]checking pypi ...[/dim]"): + try: + latest, releases = fetch_pypi_metadata() + except (ConnectionError, RuntimeError) as exc: + _fail(str(exc)) + + target = version_opt or latest + if target not in releases: + _fail(f"version '{target}' not found on pypi") + if current == target: + ui.console.print(f"already on [white]{target}[/white], nothing to do") + return + if ( + current != "unknown" + and compare_versions(parse_version(target), parse_version(current)) < 0 + ): + ui.warn(f"downgrading from {current} to {target}") + + with ui.console.status(f"[dim]installing runpod {target} ...[/dim]"): + try: + run_install(target) + except Exception as exc: # noqa: BLE001 - surface installer errors cleanly + _fail(str(exc)) + ui.success(f"updated to [white]{target}[/white]") + + # ---------------------------------------------------------------- deploy @@ -54,11 +150,17 @@ def _app_source(found, project_root: Path) -> str: "(they must then come from the worker image). Torch packages are " "excluded automatically on builtin gpu images.", ) -def deploy(target, env, python_version, exclude): +@click.option( + "--build-only", + is_flag=True, + help="Build the artifact without deploying; writes " + "-artifact.tar.gz to the current directory.", +) +def deploy(target, env, python_version, exclude, build_only): """Package and deploy all apps found in TARGET (default: cwd).""" import logging - from runpod.apps.deploy import deploy_app + from runpod.apps.deploy import build_artifact, deploy_app from runpod.apps.discovery import DiscoveryError, discover_apps from runpod.rp_cli import console as ui @@ -82,6 +184,30 @@ def deploy(target, env, python_version, exclude): ' app = App("my-app")' ) + if build_only: + for found in apps: + ui.set_name_width(list(found.resources)) + events = ui.DeployEvents() + try: + artifact = build_artifact( + found, + project_root, + python_version=python_version, + exclude=exclude.split(",") if exclude else None, + events=events, + output=Path.cwd() / f"{found.name}-artifact.tar.gz", + ) + except Exception as exc: # noqa: BLE001 - surface engine errors cleanly + raise click.ClickException(str(exc)) from exc + finally: + events.close() + size_mb = artifact.stat().st_size / (1024 * 1024) + ui.success( + f"built [white]{found.name}[/white] " + f"[dim]{artifact.name} ({size_mb:.1f} MB)[/dim]" + ) + return + if len(apps) > 1: ui.deploy_plan( [ diff --git a/runpod/rp_cli/update.py b/runpod/rp_cli/update.py new file mode 100644 index 000000000..18a6f7a42 --- /dev/null +++ b/runpod/rp_cli/update.py @@ -0,0 +1,209 @@ +"""self-update for the rp cli and a passive new-version notice. + +`rp update` installs the latest (or a pinned) runpod release from pypi +using uv when available, pip otherwise. every other command starts a +daemon thread that checks pypi at most once per day (cached under +~/.runpod) and prints a one-line notice at exit when a newer version +exists. the check never blocks or crashes a command. +""" + +from __future__ import annotations + +import atexit +import json +import shutil +import subprocess +import sys +import threading +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional, Set, Tuple + +PYPI_URL = "https://pypi.org/pypi/runpod/json" +INSTALL_TIMEOUT_SECONDS = 120 +CACHE_PATH = Path.home() / ".runpod" / "update_check.json" +CHECK_INTERVAL_HOURS = 24 + +_newer_version: Optional[str] = None +_check_done = threading.Event() +_started = False +_start_lock = threading.Lock() + + +def current_version() -> str: + try: + from runpod import __version__ + + return __version__ + except Exception: # noqa: BLE001 - version must never break the cli + return "unknown" + + +def parse_version(version: str) -> Tuple[int, ...]: + """leading numeric release segments of a version string. + + dev/local suffixes (e.g. 1.8.0.dev3) are ignored so a dev build of + the next release does not count as older than the current one. + """ + parts = [] + for part in version.split("."): + if not part.isdigit(): + break + parts.append(int(part)) + return tuple(parts) + + +def compare_versions(a: Tuple[int, ...], b: Tuple[int, ...]) -> int: + """negative when a < b, zero when equal, positive when a > b.""" + width = max(len(a), len(b)) + a_padded = a + (0,) * (width - len(a)) + b_padded = b + (0,) * (width - len(b)) + return (a_padded > b_padded) - (a_padded < b_padded) + + +def fetch_pypi_metadata() -> Tuple[str, Set[str]]: + """(latest version, all release versions) from pypi.""" + try: + with urllib.request.urlopen(PYPI_URL, timeout=15) as resp: # noqa: S310 + data = json.loads(resp.read().decode()) + except urllib.error.URLError as exc: + if isinstance(exc, urllib.error.HTTPError): + raise RuntimeError( + f"pypi returned HTTP {exc.code}; try again later" + ) from exc + raise ConnectionError( + "could not reach pypi; check your network connection" + ) from exc + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + raise RuntimeError("pypi returned an unexpected response") from exc + + try: + latest = data["info"]["version"] + except (KeyError, TypeError) as exc: + raise RuntimeError("pypi response missing version info") from exc + return latest, set(data.get("releases", {}).keys()) + + +def install_command(version: str) -> list: + """the installer invocation, preferring uv when present.""" + package_spec = f"runpod=={version}" + if shutil.which("uv"): + return ["uv", "pip", "install", package_spec, "--quiet"] + return [sys.executable, "-m", "pip", "install", package_spec, "--quiet"] + + +def run_install(version: str) -> None: + cmd = install_command(version) + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=INSTALL_TIMEOUT_SECONDS, + ) + if result.returncode != 0: + installer = "uv" if cmd[0] == "uv" else "pip" + raise RuntimeError( + f"{installer} install failed (exit {result.returncode}): " + f"{result.stderr.strip()}" + ) + + +# -- passive background check ------------------------------------------- + + +def _read_cache() -> Optional[dict]: + try: + return json.loads(CACHE_PATH.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, ValueError): + return None + + +def _write_cache(latest_version: str) -> None: + try: + CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) + CACHE_PATH.write_text( + json.dumps( + { + "last_checked_utc": datetime.now( + timezone.utc + ).isoformat(), + "latest_version": latest_version, + } + ), + encoding="utf-8", + ) + except OSError: + pass + + +def _cache_fresh(cache: dict) -> bool: + try: + last_checked = datetime.fromisoformat(cache["last_checked_utc"]) + elapsed = ( + datetime.now(timezone.utc) - last_checked + ).total_seconds() / 3600 + return elapsed < CHECK_INTERVAL_HOURS + except (KeyError, ValueError, TypeError): + return False + + +def _run_check() -> None: + global _newer_version # noqa: PLW0603 + try: + current = current_version() + if current == "unknown": + return + + cache = _read_cache() + latest = None + if cache and _cache_fresh(cache): + latest = cache.get("latest_version") or None + if not latest: + latest, _ = fetch_pypi_metadata() + _write_cache(latest) + if not latest: + return + + if compare_versions(parse_version(latest), parse_version(current)) > 0: + _newer_version = latest + except Exception: # noqa: BLE001 - the check must never crash the cli + pass + finally: + _check_done.set() + + +def _print_update_notice() -> None: + # plain text: atexit runs after rich teardown + if not _check_done.is_set(): + return + if _newer_version: + print( + f"\na new version of runpod is available: {_newer_version}\n" + " run 'rp update' to upgrade.", + file=sys.stderr, + ) + + +def _is_interactive() -> bool: + for stream in (sys.stderr, sys.stdout): + try: + if stream is not None and stream.isatty(): + return True + except Exception: # noqa: BLE001 + pass + return False + + +def start_background_check() -> None: + """spawn the daemon check thread once per process (ttys only).""" + global _started # noqa: PLW0603 + with _start_lock: + if _started: + return + _started = True + if not _is_interactive(): + return + atexit.register(_print_update_notice) + threading.Thread(target=_run_check, daemon=True).start() diff --git a/runpod/runtimes/queue/worker.py b/runpod/runtimes/queue/worker.py index 145237cb0..512b859d6 100644 --- a/runpod/runtimes/queue/worker.py +++ b/runpod/runtimes/queue/worker.py @@ -108,13 +108,31 @@ def _live_handler(job: dict) -> dict: return execute_request(job.get("input") or {}) +def _max_concurrency() -> int: + """jobs one worker may run at once (RUNPOD_MAX_CONCURRENCY, min 1).""" + try: + return max(1, int(os.environ.get("RUNPOD_MAX_CONCURRENCY", "1"))) + except ValueError: + return 1 + + +def _worker_config(handler) -> dict: + config: dict = {"handler": handler} + concurrency = _max_concurrency() + if concurrency > 1: + config["concurrency_modifier"] = lambda _current: concurrency + return config + + def main() -> None: if _is_deployed(): handle = _load_deployed_handle() _run_init(handle) - runpod.serverless.start({"handler": _make_deployed_handler(handle)}) + runpod.serverless.start( + _worker_config(_make_deployed_handler(handle)) + ) else: - runpod.serverless.start({"handler": _live_handler}) + runpod.serverless.start(_worker_config(_live_handler)) if __name__ == "__main__": diff --git a/tests/test_apps/test_config_params.py b/tests/test_apps/test_config_params.py new file mode 100644 index 000000000..214a40c69 --- /dev/null +++ b/tests/test_apps/test_config_params.py @@ -0,0 +1,364 @@ +"""endpoint configuration parameters across spec, dev, and deploy payloads.""" + +import pytest + +import runpod +from runpod.apps import App +from runpod.apps.app import _clear_registry +from runpod.apps.dev import _endpoint_input +from runpod.apps.errors import InvalidResourceError +from runpod.apps.spec import ( + ResourceKind, + ResourceSpec, + normalize_cuda_version, + normalize_scaler_type, +) + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +class TestNormalizers: + def test_scaler_type_uppercases(self): + assert normalize_scaler_type("queue_delay") == "QUEUE_DELAY" + assert normalize_scaler_type("REQUEST_COUNT") == "REQUEST_COUNT" + + def test_scaler_type_none_passes(self): + assert normalize_scaler_type(None) is None + + def test_scaler_type_rejects_unknown(self): + with pytest.raises(InvalidResourceError): + normalize_scaler_type("SOMETHING") + + def test_cuda_version_valid(self): + assert normalize_cuda_version("12.8") == "12.8" + + def test_cuda_version_none_passes(self): + assert normalize_cuda_version(None) is None + + def test_cuda_version_rejects_unknown(self): + with pytest.raises(InvalidResourceError): + normalize_cuda_version("10.0") + + +class TestSpecValidation: + def test_max_concurrency_floor(self): + with pytest.raises(InvalidResourceError): + ResourceSpec( + kind=ResourceKind.QUEUE, name="q", max_concurrency=0 + ) + + def test_execution_timeout_floor(self): + with pytest.raises(InvalidResourceError): + ResourceSpec( + kind=ResourceKind.QUEUE, name="q", execution_timeout_ms=-1 + ) + + def test_scaler_value_floor(self): + with pytest.raises(InvalidResourceError): + ResourceSpec(kind=ResourceKind.QUEUE, name="q", scaler_value=0) + + def test_container_disk_floor(self): + with pytest.raises(InvalidResourceError): + ResourceSpec( + kind=ResourceKind.QUEUE, name="q", container_disk_gb=0 + ) + + def test_min_cuda_rejected_on_cpu(self): + with pytest.raises(InvalidResourceError): + ResourceSpec( + kind=ResourceKind.QUEUE, + name="q", + cpu=["cpu3c-1-2"], + min_cuda_version="12.8", + ) + + def test_effective_scaler_type_defaults_by_kind(self): + queue = ResourceSpec(kind=ResourceKind.QUEUE, name="q") + api = ResourceSpec(kind=ResourceKind.API, name="api") + assert queue.effective_scaler_type == "QUEUE_DELAY" + assert api.effective_scaler_type == "REQUEST_COUNT" + + def test_effective_scaler_type_explicit_wins(self): + spec = ResourceSpec( + kind=ResourceKind.QUEUE, name="q", scaler_type="REQUEST_COUNT" + ) + assert spec.effective_scaler_type == "REQUEST_COUNT" + + +class TestDecoratorPlumbing: + def test_queue_accepts_all_params(self): + app = App("a") + + @app.queue( + name="q", + gpu="4090", + max_concurrency=4, + execution_timeout_ms=60000, + flashboot=False, + scaler_type="request_count", + scaler_value=8, + min_cuda_version="12.4", + accelerate_downloads=False, + container_disk_gb=50, + ) + def q(): + pass + + spec = q.spec + assert spec.max_concurrency == 4 + assert spec.execution_timeout_ms == 60000 + assert spec.flashboot is False + assert spec.scaler_type == "REQUEST_COUNT" + assert spec.scaler_value == 8 + assert spec.min_cuda_version == "12.4" + assert spec.accelerate_downloads is False + assert spec.container_disk_gb == 50 + + def test_api_accepts_params(self): + app = App("a") + + @app.api( + name="api", + cpu="cpu3c-1-2", + execution_timeout_ms=30000, + scaler_value=2, + container_disk_gb=20, + ) + class Api: + @runpod.post("/x") + def x(self, body: dict): + return body + + assert Api.spec.execution_timeout_ms == 30000 + assert Api.spec.scaler_value == 2 + assert Api.spec.container_disk_gb == 20 + + def test_task_accepts_params(self): + app = App("a") + + @app.task( + name="t", + gpu="4090", + min_cuda_version="12.8", + accelerate_downloads=False, + container_disk_gb=100, + ) + def t(): + pass + + assert t.spec.min_cuda_version == "12.8" + assert t.spec.accelerate_downloads is False + assert t.spec.container_disk_gb == 100 + + def test_queue_invalid_scaler_fails_at_decoration(self): + app = App("a") + + with pytest.raises(InvalidResourceError): + + @app.queue(name="q", scaler_type="nope") + def q(): + pass + + +class TestManifestSerialization: + def test_non_defaults_serialized(self): + spec = ResourceSpec( + kind=ResourceKind.QUEUE, + name="q", + max_concurrency=4, + execution_timeout_ms=1000, + flashboot=False, + scaler_type="REQUEST_COUNT", + scaler_value=8, + min_cuda_version="12.8", + accelerate_downloads=False, + container_disk_gb=42, + ) + data = spec.to_manifest() + assert data["maxConcurrency"] == 4 + assert data["executionTimeoutMs"] == 1000 + assert data["flashboot"] is False + assert data["scalerType"] == "REQUEST_COUNT" + assert data["scalerValue"] == 8 + assert data["minCudaVersion"] == "12.8" + assert data["accelerateDownloads"] is False + assert data["containerDiskGb"] == 42 + + def test_defaults_omitted(self): + spec = ResourceSpec(kind=ResourceKind.QUEUE, name="q") + data = spec.to_manifest() + for key in ( + "maxConcurrency", + "executionTimeoutMs", + "flashboot", + "scalerType", + "scalerValue", + "minCudaVersion", + "accelerateDownloads", + "containerDiskGb", + ): + assert key not in data + + +class TestDevPayload: + def test_config_params_reach_payload(self): + app = App("a") + + @app.queue( + name="q", + gpu="4090", + max_concurrency=3, + execution_timeout_ms=90000, + scaler_type="request_count", + scaler_value=7, + min_cuda_version="12.6", + container_disk_gb=40, + ) + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["scalerType"] == "REQUEST_COUNT" + assert payload["scalerValue"] == 7 + assert payload["executionTimeoutMs"] == 90000 + assert payload["minCudaVersion"] == "12.6" + assert payload["template"]["containerDiskInGb"] == 40 + env = {e["key"]: e["value"] for e in payload["template"]["env"]} + assert env["RUNPOD_MAX_CONCURRENCY"] == "3" + + def test_flashboot_disabled_omits_flag(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", flashboot=False) + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert "flashBootType" not in payload + + def test_defaults_unchanged(self): + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + assert payload["scalerType"] == "QUEUE_DELAY" + assert payload["scalerValue"] == 4 + assert payload["executionTimeoutMs"] == 0 + assert payload["flashBootType"] == "FLASHBOOT" + assert payload["template"]["containerDiskInGb"] == 10 + env = {e["key"]: e["value"] for e in payload["template"]["env"]} + assert "RUNPOD_MAX_CONCURRENCY" not in env + + +class TestDeployPayload: + def test_config_params_reach_payload(self): + from runpod.apps.deploy import _deployed_endpoint_input + + app = App("dep") + + @app.queue( + name="que", + gpu="4090", + max_concurrency=2, + execution_timeout_ms=5000, + scaler_value=9, + min_cuda_version="12.8", + container_disk_gb=64, + ) + def que(): + pass + + payload = _deployed_endpoint_input(app, que.spec, "env-1", "b-1", "3.12") + assert payload["scalerValue"] == 9 + assert payload["executionTimeoutMs"] == 5000 + assert payload["minCudaVersion"] == "12.8" + assert payload["template"]["containerDiskInGb"] == 64 + env = {e["key"]: e["value"] for e in payload["template"]["env"]} + assert env["RUNPOD_MAX_CONCURRENCY"] == "2" + + def test_flashboot_default_present(self): + from runpod.apps.deploy import _deployed_endpoint_input + + app = App("dep") + + @app.queue(name="que", cpu="cpu3c-1-2") + def que(): + pass + + payload = _deployed_endpoint_input(app, que.spec, "env-1", "b-1", "3.12") + assert payload["flashBootType"] == "FLASHBOOT" + + def test_flashboot_disabled_omitted(self): + from runpod.apps.deploy import _deployed_endpoint_input + + app = App("dep") + + @app.queue(name="que", cpu="cpu3c-1-2", flashboot=False) + def que(): + pass + + payload = _deployed_endpoint_input(app, que.spec, "env-1", "b-1", "3.12") + assert "flashBootType" not in payload + + +class TestTaskPayload: + def test_min_cuda_expands_to_allowed_versions(self): + from runpod.apps.tasks import _pod_input, cuda_versions_at_least + + app = App("a") + + @app.task(name="t", gpu="4090", min_cuda_version="12.8") + def t(): + pass + + pod = _pod_input(t.spec, "tok", "t") + assert pod["allowedCudaVersions"] == ["12.8", "12.9", "13.0"] + assert cuda_versions_at_least("12.9") == ["12.9", "13.0"] + + def test_container_disk_override(self): + from runpod.apps.tasks import _pod_input + + app = App("a") + + @app.task(name="t", cpu="cpu3c-1-2", container_disk_gb=25) + def t(): + pass + + pod = _pod_input(t.spec, "tok", "t") + assert pod["containerDiskInGb"] == 25 + + +class TestAccelerateDownloads: + def test_flag_travels_in_function_request(self): + from runpod.apps.targets import LiveTarget + + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", accelerate_downloads=False) + def q(x: int): + return x + + target = LiveTarget("ep-1", "q") + payload = target.build_payload(q._fn, q.spec, (1,), {}) + assert payload["input"]["accelerate_downloads"] is False + + def test_default_true(self): + from runpod.apps.targets import LiveTarget + + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2") + def q(x: int): + return x + + target = LiveTarget("ep-1", "q") + payload = target.build_payload(q._fn, q.spec, (1,), {}) + assert payload["input"]["accelerate_downloads"] is True diff --git a/tests/test_cli/test_init_command.py b/tests/test_cli/test_init_command.py new file mode 100644 index 000000000..48d969eaa --- /dev/null +++ b/tests/test_cli/test_init_command.py @@ -0,0 +1,82 @@ +"""rp init: project scaffolding.""" + +from pathlib import Path + +from click.testing import CliRunner + +from runpod.apps.init import create_project, detect_conflicts +from runpod.rp_cli.main import cli + + +class TestCreateProject: + def test_writes_skeleton(self, tmp_path): + written = create_project(tmp_path, "my-app") + names = {p.name for p in written} + assert names == {"main.py", "requirements.txt", ".runpodignore"} + main = (tmp_path / "main.py").read_text() + assert 'App("my-app")' in main + assert "@runpod.local_entrypoint" in main + + def test_skeleton_module_is_valid_python(self, tmp_path): + create_project(tmp_path, "my-app") + compile((tmp_path / "main.py").read_text(), "main.py", "exec") + + def test_existing_files_kept_without_overwrite(self, tmp_path): + (tmp_path / "main.py").write_text("original") + written = create_project(tmp_path, "my-app") + assert (tmp_path / "main.py").read_text() == "original" + assert Path(tmp_path / "requirements.txt") not in written or True + assert (tmp_path / "requirements.txt").exists() + + def test_overwrite_replaces_files(self, tmp_path): + (tmp_path / "main.py").write_text("original") + create_project(tmp_path, "my-app", overwrite=True) + assert 'App("my-app")' in (tmp_path / "main.py").read_text() + + def test_creates_directory(self, tmp_path): + target = tmp_path / "new-project" + create_project(target, "new-project") + assert (target / "main.py").exists() + + +class TestDetectConflicts: + def test_empty_dir_no_conflicts(self, tmp_path): + assert detect_conflicts(tmp_path) == [] + + def test_existing_files_reported(self, tmp_path): + (tmp_path / "main.py").write_text("x") + assert detect_conflicts(tmp_path) == ["main.py"] + + +class TestInitCommand: + def test_init_new_project(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + runner = CliRunner() + result = runner.invoke(cli, ["init", "demo"]) + assert result.exit_code == 0, result.output + assert (tmp_path / "demo" / "main.py").exists() + + def test_init_current_directory(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + runner = CliRunner() + result = runner.invoke(cli, ["init", "."]) + assert result.exit_code == 0, result.output + assert (tmp_path / "main.py").exists() + assert tmp_path.name in (tmp_path / "main.py").read_text() + + def test_init_conflicts_fail_without_force(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "main.py").write_text("keep me") + runner = CliRunner() + result = runner.invoke(cli, ["init", "."]) + assert result.exit_code != 0 + assert "main.py" in result.output + assert (tmp_path / "main.py").read_text() == "keep me" + + def test_init_force_overwrites(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "main.py").write_text("old") + runner = CliRunner() + result = runner.invoke(cli, ["init", ".", "--force"]) + assert result.exit_code == 0, result.output + assert "App(" in (tmp_path / "main.py").read_text() diff --git a/tests/test_cli/test_update_command.py b/tests/test_cli/test_update_command.py new file mode 100644 index 000000000..4ed658057 --- /dev/null +++ b/tests/test_cli/test_update_command.py @@ -0,0 +1,133 @@ +"""rp update and the passive update check.""" + +import json +from unittest.mock import patch + +from click.testing import CliRunner + +from runpod.rp_cli import update as upd +from runpod.rp_cli.main import cli + + +class TestParseVersion: + def test_release(self): + assert upd.parse_version("1.8.0") == (1, 8, 0) + + def test_dev_suffix_ignored(self): + assert upd.parse_version("1.8.0.dev3") == (1, 8, 0) + + def test_two_part(self): + assert upd.parse_version("2.0") == (2, 0) + + +class TestCompareVersions: + def test_orders(self): + assert upd.compare_versions((1, 0), (2, 0)) < 0 + assert upd.compare_versions((2, 1), (2, 0)) > 0 + + def test_pads_shorter(self): + assert upd.compare_versions((2, 0), (2, 0, 0)) == 0 + + +class TestInstallCommand: + def test_prefers_uv(self): + with patch.object(upd.shutil, "which", return_value="/usr/bin/uv"): + cmd = upd.install_command("1.2.3") + assert cmd[0] == "uv" + assert "runpod==1.2.3" in cmd + + def test_falls_back_to_pip(self): + with patch.object(upd.shutil, "which", return_value=None): + cmd = upd.install_command("1.2.3") + assert cmd[1:4] == ["-m", "pip", "install"] + + +class TestUpdateCommand: + def test_already_current(self): + runner = CliRunner() + with patch.object( + upd, "fetch_pypi_metadata", return_value=("9.9.9", {"9.9.9"}) + ), patch.object(upd, "current_version", return_value="9.9.9"), patch( + "runpod.rp_cli.main.cli.callback" + ): + result = runner.invoke(cli, ["update"]) + assert result.exit_code == 0, result.output + assert "nothing to do" in result.output + + def test_unknown_version_fails(self): + runner = CliRunner() + with patch.object( + upd, "fetch_pypi_metadata", return_value=("9.9.9", {"9.9.9"}) + ): + result = runner.invoke(cli, ["update", "--version", "0.0.0.404"]) + assert result.exit_code != 0 + assert "not found" in result.output + + def test_installs_target(self): + runner = CliRunner() + with patch.object( + upd, "fetch_pypi_metadata", return_value=("9.9.9", {"9.9.9"}) + ), patch.object(upd, "current_version", return_value="1.0.0"), patch.object( + upd, "run_install" + ) as install: + result = runner.invoke(cli, ["update"]) + assert result.exit_code == 0, result.output + install.assert_called_once_with("9.9.9") + + +class TestBackgroundCheckCache: + def test_fresh_cache_skips_fetch(self, tmp_path, monkeypatch): + cache = tmp_path / "update_check.json" + monkeypatch.setattr(upd, "CACHE_PATH", cache) + from datetime import datetime, timezone + + cache.write_text( + json.dumps( + { + "last_checked_utc": datetime.now(timezone.utc).isoformat(), + "latest_version": "99.0.0", + } + ) + ) + monkeypatch.setattr(upd, "_newer_version", None) + with patch.object(upd, "fetch_pypi_metadata") as fetch, patch.object( + upd, "current_version", return_value="1.0.0" + ): + upd._run_check() + fetch.assert_not_called() + assert upd._newer_version == "99.0.0" + + def test_stale_cache_refetches(self, tmp_path, monkeypatch): + cache = tmp_path / "update_check.json" + monkeypatch.setattr(upd, "CACHE_PATH", cache) + cache.write_text( + json.dumps( + { + "last_checked_utc": "2000-01-01T00:00:00+00:00", + "latest_version": "0.1.0", + } + ) + ) + monkeypatch.setattr(upd, "_newer_version", None) + with patch.object( + upd, "fetch_pypi_metadata", return_value=("99.0.0", {"99.0.0"}) + ), patch.object(upd, "current_version", return_value="1.0.0"): + upd._run_check() + assert upd._newer_version == "99.0.0" + assert json.loads(cache.read_text())["latest_version"] == "99.0.0" + + def test_current_version_newer_no_notice(self, tmp_path, monkeypatch): + monkeypatch.setattr(upd, "CACHE_PATH", tmp_path / "update_check.json") + monkeypatch.setattr(upd, "_newer_version", None) + with patch.object( + upd, "fetch_pypi_metadata", return_value=("1.0.0", {"1.0.0"}) + ), patch.object(upd, "current_version", return_value="2.0.0"): + upd._run_check() + assert upd._newer_version is None + + def test_check_never_raises(self, tmp_path, monkeypatch): + monkeypatch.setattr(upd, "CACHE_PATH", tmp_path / "update_check.json") + with patch.object( + upd, "fetch_pypi_metadata", side_effect=RuntimeError("boom") + ), patch.object(upd, "current_version", return_value="1.0.0"): + upd._run_check() From 0f5ca897c66ed2d5dd9356f61df9b7858248951f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:11:59 -0700 Subject: [PATCH 065/120] docs: simplify readme --- README.md | 544 +++++------------------------------------------------- 1 file changed, 45 insertions(+), 499 deletions(-) diff --git a/README.md b/README.md index 1d1a3f575..dc719d3a8 100644 --- a/README.md +++ b/README.md @@ -1,543 +1,89 @@
-

Runpod | Python Library

-[![PyPI Package](https://badge.fury.io/py/runpod.svg)](https://badge.fury.io/py/runpod) -  -[![Downloads](https://static.pepy.tech/personalized-badge/runpod?period=total&units=international_system&left_color=grey&right_color=blue&left_text=Downloads)](https://pepy.tech/project/runpod) +

Runpod Python

-[![CI | End-to-End Runpod Python Tests](https://github.com/runpod/runpod-python/actions/workflows/CI-e2e.yml/badge.svg)](https://github.com/runpod/runpod-python/actions/workflows/CI-e2e.yml) +**Define GPU functions in Python. Run them in the cloud with one decorator.** +[![PyPI Package](https://badge.fury.io/py/runpod.svg)](https://badge.fury.io/py/runpod) +[![Downloads](https://static.pepy.tech/personalized-badge/runpod?period=total&units=international_system&left_color=grey&right_color=blue&left_text=downloads)](https://pepy.tech/project/runpod) [![CI | Unit Tests](https://github.com/runpod/runpod-python/actions/workflows/CI-pytests.yml/badge.svg)](https://github.com/runpod/runpod-python/actions/workflows/CI-pytests.yml) -  -[![CI | CodeQL](https://github.com/runpod/runpod-python/actions/workflows/CI-codeql.yml/badge.svg)](https://github.com/runpod/runpod-python/actions/workflows/CI-codeql.yml) - -
- -The official Python SDK for [Runpod](https://runpod.io): define GPU functions in Python, run them in the cloud with one decorator, and deploy them as production endpoints. - -```python -import runpod -from runpod import App - -app = App("my-app") - -@app.queue(gpu="4090") -def embed(text: str): - import torch - ... - return vector +[![License](https://img.shields.io/badge/License-MIT-blue)](LICENSE) -@runpod.local_entrypoint -def main(): - print(embed.remote("hello world")) # runs on a cloud 4090 -``` +[Documentation](https://docs.runpod.io) • [Examples](examples/apps) • [Discord](https://discord.gg/pJ3P2DbUUq) -```bash -pip install runpod -rp login # browser auth -rp dev main.py # live dev session: edit, re-run, logs stream back -rp deploy # production endpoints -``` - -## Table of Contents - -- [Installation](#--installation) -- [Apps: Python Functions in the Cloud](#--apps-python-functions-in-the-cloud) - - [Resources: queue, api, task](#resources-queue-api-task) - - [Calling Functions](#calling-functions) - - [Hardware Selection](#hardware-selection) - - [Dependencies and Images](#dependencies-and-images) - - [Volumes](#volumes) - - [Secrets](#secrets) - - [Cached Models](#cached-models) - - [Private Registries](#private-registries) - - [The Dev Loop](#the-dev-loop) - - [Deploying](#deploying) - - [CLI Reference](#cli-reference) -- [Serverless Worker (SDK)](#--serverless-worker-sdk) -- [API Language Library](#--api-language-library-graphql-wrapper) -- [Directory](#--directory) -- [Community and Contributing](#--community-and-contributing) - -## 💻 | Installation - -```bash -# stable -pip install runpod - -# or with uv -uv add runpod - -# latest development version -pip install git+https://github.com/runpod/runpod-python.git -``` - -*Python 3.10 or higher is required.* + -Installing the package also installs the `rp` CLI (also available as `runpod`). Authenticate once: +## Installation ```bash -rp login +pip install runpod # or: uv add runpod +rp login # authenticate once ``` -## 🚀 | Apps: Python Functions in the Cloud +Requires Python 3.10+. Installing the package also installs the `rp` CLI. -An **App** is a named collection of functions that run on Runpod. Decorate plain Python functions; the SDK handles packaging, provisioning, scaling, and routing. - -### Resources: queue, api, task - -Three decorators cover the three execution models: +## Example ```python import runpod -from runpod import App +from runpod import App, Model, Secret, Volume app = App("inference") -# queue: autoscaling job queue (scale-to-zero serverless workers) -@app.queue(gpu="4090") -def transcribe(audio_url: str): - ... - return text - -# api: load-balanced http service from a class -from runpod import init, get, post - -@app.api(cpu="cpu5c-2-4") -class Counter: - @init - def setup(self): # runs once per worker, before traffic - self.count = 0 - - @post("/bump") - async def bump(self, body: dict): - self.count += body.get("by", 1) - return {"count": self.count} - - @get("/value") - async def value(self): - return {"count": self.count} - -# task: one ephemeral pod per call, runs to completion, then terminates -@app.task(gpu="H100", gpu_count=2) -def train(steps: int = 1000): - ... - return {"loss": final_loss} -``` - -Pick by shape of work: - -| | `@app.queue` | `@app.api` | `@app.task` | -|---|---|---|---| -| execution | job queue | http routes | dedicated pod per call | -| scaling | 0..N workers | 0..N workers | one pod, then gone | -| best for | inference jobs | rest services, streaming | training runs, batch jobs | -| duration | seconds to minutes | request-scoped | minutes to hours | - -`@app.api` also accepts a zero-argument function returning an ASGI app (FastAPI, Starlette) if you'd rather bring your own framework. - -### Calling Functions - -Every decorated function keeps its normal Python identity and gains remote invocation: - -```python -transcribe.remote(url) # run remotely, block for the result -await transcribe.remote.aio(url) # async variant -transcribe.spawn(url) # fire and forget -> Job -transcribe.local(url) # run in this process (no cloud) - -job = transcribe.spawn(url) -result = job.result() # collect later -``` - -Functions can call each other — from your laptop, or from inside another worker: - -```python -@app.queue(cpu="cpu3c-1-2") -def orchestrate(x: int): - return {"embedding": embed.remote(x), "meta": lookup.remote(x)} -``` - -`@runpod.local_entrypoint` marks the function `rp dev` runs as your client-side script. - -### Hardware Selection - -GPUs resolve from pool ids, exact device names, or shorthand fragments: - -```python -@app.queue(gpu="4090") # NVIDIA GeForce RTX 4090 -@app.queue(gpu="H100") # all H100 variants -@app.queue(gpu="B200", gpu_count=8) # multi-gpu -@app.queue(gpu=GpuGroup.ADA_24) # pool id enum -@app.queue(gpu=["4090", "5090"]) # any of several -@app.queue(cpu="cpu5c-2-4") # cpu instance flavor -``` - -Unknown names fail at import time with the full list of valid options — not at provision time with a cryptic API error. +models = Volume("models", size=100) +llama = Model("meta-llama/Llama-3.1-8B-Instruct") -### Dependencies and Images -```python +# an autoscaling job queue on cloud H100s: weights pre-cached, +# dependencies vendored at deploy time, scale-to-zero when idle @app.queue( - cpu="cpu3c-1-2", - dependencies=["pyfiglet", "numpy>=2"], # pip packages - system_dependencies=["ffmpeg"], # apt packages + gpu="H100", + workers=(0, 3), + dependencies=["vllm"], + volume=models, + model=llama, + env={"HF_TOKEN": Secret("hf-token")}, ) -def render(text: str): ... -``` - -At deploy time the SDK resolves your dependency closure into the build artifact, so cold starts never run pip. Prefer a fully custom image: - -```python -@app.queue(image="ghcr.io/me/mine:latest", registry_auth="my-ghcr") -def custom(): ... -``` - -Custom images must include `python3`; the worker runtime bootstraps itself. - -### Volumes - -Network volumes persist across calls and are shared between resources: - -```python -from runpod import Volume - -models = Volume("models", size=100) # created on first use - -@app.task(gpu="H100", volume=models) -def train(): - torch.save(state, models.path / "run-1/model.pt") - -@app.queue(gpu="4090", volume=models) -def infer(prompt: str): - sd = torch.load(models.path / "run-1/model.pt") -``` - -A volume lives in one datacenter, and everything attached to it must schedule there. The SDK solves placement over the whole app: it intersects each resource's hardware availability per datacenter, picks the one where the most-constrained resource has the best stock, and errors with a per-resource breakdown when no datacenter can host everyone. - -`volume.path` resolves to the platform mount inside the worker (`/workspace` on task pods, `/runpod-volume` on endpoint workers). Endpoints may take a list of volumes (one per datacenter) to serve from multiple regions. - -### Secrets - -Platform-encrypted values, decrypted only inside the worker: - -```bash -rp secret add hf-token # prompts for the value -``` - -```python -from runpod import Secret - -@app.queue(gpu="4090", env={"HF_TOKEN": Secret("hf-token")}) -def download(): ... # worker sees the real value in $HF_TOKEN -``` - -The value never passes through the SDK. Deploys fail fast when a referenced secret does not exist. - -### Cached Models - -Reference HuggingFace weights and the platform stages them on the host **before** your worker starts — cold starts skip the download entirely: - -```python -from runpod import Model - -llama = Model("meta-llama/Llama-3.1-8B-Instruct") - -@app.queue(gpu="H100", model=llama, env={"HF_TOKEN": Secret("hf-token")}) def chat(prompt: str): - llm = vllm.LLM(model=str(llama.path)) # weights already on disk -``` - -Weights also appear in the standard HuggingFace cache layout, so `transformers` and `vllm` find them with zero configuration. Gated repos authenticate with an `HF_TOKEN` env var (a `Secret` works). - -### Private Registries - -```bash -rp registry add my-ghcr # prompts for username/password -``` - -```python -@app.queue(image="ghcr.io/me/private:latest", registry_auth="my-ghcr") -def infer(): ... -``` - -### The Dev Loop - -```bash -rp dev main.py -``` - -provisions temporary live endpoints for your app, runs your `@runpod.local_entrypoint`, and streams everything back: - -``` -dev inference · main.py - transcribe queue 4090 se3u174ensbubc ↗ - - ▸ main() local entrypoint - ○ transcribe() - ● transcribe() running on worker 1x74v8dg5e9i - transcribe │ loading model... - transcribe │ transcribing 42s of audio - ✓ transcribe() in 12.4s - - ✓ done in 12.6s · enter re-run · edit reload · ^C quit -``` + import vllm -- **edit a file** → session reloads, shows a diff of what changed, next call runs the new code -- **enter** → re-run the entrypoint -- **worker stdout streams live** into your terminal, attributed per function -- **ctrl-c** → everything is cleaned up; dev sessions leave nothing behind + llm = vllm.LLM(model=str(llama.path)) # weights already on disk + return llm.generate(prompt) -Code ships with each request during dev, so there is no build step between edits. -### Deploying - -```bash -rp deploy # every app found in the current directory -rp deploy main.py # one file -rp deploy -e staging # a named environment -``` - -Deploys package your project and its resolved dependency environment into a build artifact, upload it once, and reconcile one endpoint per queue/api resource (tasks provision per call). Workers cold-start from the artifact — no dependency resolution at runtime. +# one ephemeral pod per call: provisions, runs to completion, terminates +@app.task(gpu="H100", gpu_count=2, volume=models) +def finetune(steps: int = 1000): + ... + return {"loss": final_loss} -``` -deploy inference → default · transcribe -✓ resolving dependencies 89 packages -✓ packaging artifact -✓ uploading build ━━━━━━━━━━ 53.3 MB -✓ reconciling endpoints -✓ inference/default is live 44.9s - transcribe gumuj47vi1xu4y ↗ +@runpod.local_entrypoint +def main(): + print(chat.remote("why is the sky blue?")) # blocks for the result + job = finetune.spawn(steps=500) # fire and forget -> Job ``` -Manage what's deployed: - ```bash -rp app list # all apps and environments -rp env get default -a inference # endpoints in an environment -rp undeploy -a inference # tear down an environment's endpoints -rp app delete inference # remove the app entirely -``` - -### CLI Reference - -| command | | -|---|---| -| `rp login` | browser auth (or `--api-key` to paste a key) | -| `rp dev ` | live dev session | -| `rp deploy [target]` | build and deploy apps | -| `rp undeploy` | tear down an environment | -| `rp app list/get/delete` | app management | -| `rp env list/get/create/delete` | environment management | -| `rp secret add/list/rm` | platform secrets | -| `rp registry add/list/rm` | registry credentials | -| `rp logs [-f]` | pod logs (snapshot or follow) | -| `rp pod / ssh / exec / config` | pod management | - -## ⚡ | Serverless Worker (SDK) - -This package is also what you use to build the worker side of a custom serverless endpoint: a handler function that processes jobs. - -```python -# my_worker.py - -import runpod - -def is_even(job): - job_input = job["input"] - the_number = job_input["number"] - - if not isinstance(the_number, int): - return {"error": "Silly human, you need to pass an integer."} - - if the_number % 2 == 0: - return True - - return False - -runpod.serverless.start({"handler": is_even}) +rp dev main.py # live dev session: edit, re-run, logs stream back +rp deploy # deploy production endpoints ``` -Make sure that this file is run when your container starts. This can be accomplished by calling it in the docker command when you set up a template at [console.runpod.io/serverless/user/templates](https://console.runpod.io/serverless/user/templates) or by setting it as the default command in your Dockerfile. - -See our [blog post](https://www.runpod.io/blog/build-basic-serverless-api) for creating a basic Serverless API, or view the [detailed docs](https://docs.runpod.io/serverless-ai/custom-apis) for more information. +Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for complete projects and [docs.runpod.io](https://docs.runpod.io) for the full guide. -### Local Test Worker +## Contributing -You can test your worker locally before deploying it to Runpod: +Pull requests and issues are welcome — see the [contributing guide](CONTRIBUTING.md) to get started. ```bash -python my_worker.py --rp_serve_api -``` - -### Worker Fitness Checks - -Fitness checks validate your worker environment at startup before processing jobs. If any check fails, the worker exits immediately, allowing your orchestrator to restart it. - -```python -# my_worker.py - -import runpod -import torch - -@runpod.serverless.register_fitness_check -def check_gpu_available(): - """Verify GPU is available.""" - if not torch.cuda.is_available(): - raise RuntimeError("GPU not available") - -@runpod.serverless.register_fitness_check -def check_disk_space(): - """Verify sufficient disk space.""" - import shutil - stat = shutil.disk_usage("/") - free_gb = stat.free / (1024**3) - if free_gb < 10: - raise RuntimeError(f"Insufficient disk space: {free_gb:.2f}GB free") - -def handler(job): - job_input = job["input"] - # Your handler code here - return {"output": "success"} - -runpod.serverless.start({"handler": handler}) +git clone https://github.com/runpod/runpod-python.git +cd runpod-python +make setup +make test ``` -Built-in system fitness checks (GPU health, disk, memory, CUDA, network) run automatically on serverless workers. - -## 📚 | API Language Library (GraphQL Wrapper) - -When interacting with the Runpod API you can use this library to make requests. - -```python -import runpod - -runpod.api_key = "your_runpod_api_key_found_under_settings" -``` - -### Endpoints - -Interact with existing endpoints by id: - -```python -import runpod - -endpoint = runpod.Endpoint("ENDPOINT_ID") - -run_request = endpoint.run( - {"your_model_input_key": "your_model_input_value"} -) - -# Check the status of the endpoint run request -print(run_request.status()) - -# Get the output of the endpoint run request, blocking until complete -print(run_request.output()) -``` - -```python -import runpod - -endpoint = runpod.Endpoint("ENDPOINT_ID") - -run_request = endpoint.run_sync( - {"your_model_input_key": "your_model_input_value"} -) - -# Returns the job results if completed within 90 seconds, otherwise, returns the job status. -print(run_request) -``` - -#### Per-Endpoint API Keys - -Each `Endpoint` instance can use its own API key, useful for multi-tenant scenarios: - -```python -import runpod - -# Endpoint using a specific API key -endpoint1 = runpod.Endpoint("ENDPOINT_ID_1", api_key="CUSTOMER_A_KEY") -endpoint2 = runpod.Endpoint("ENDPOINT_ID_2", api_key="CUSTOMER_B_KEY") - -result = endpoint2.run_sync({"input": "data"}) -``` - -#### Thread-Safe Operations - -Each `Endpoint` instance maintains its own API key, making concurrent operations safe: - -```python -import threading -import runpod - -def process_request(api_key, endpoint_id, input_data): - # Each thread gets its own Endpoint instance - endpoint = runpod.Endpoint(endpoint_id, api_key=api_key) - return endpoint.run_sync(input_data) - -threads = [] -for customer in customers: - t = threading.Thread( - target=process_request, - args=(customer["api_key"], customer["endpoint_id"], customer["input"]) - ) - threads.append(t) - t.start() -``` - -### GPU Cloud (Pods) - -```python -import runpod - -runpod.api_key = "your_runpod_api_key_found_under_settings" - -# Get all my pods -pods = runpod.get_pods() - -# Get a specific pod -pod = runpod.get_pod(pod.id) - -# Create a pod with GPU -pod = runpod.create_pod("test", "runpod/stack", "NVIDIA GeForce RTX 3070") - -# Create a pod with CPU -pod = runpod.create_pod("test", "runpod/stack", instance_id="cpu3c-2-4") - -# Stop the pod -runpod.stop_pod(pod.id) - -# Resume the pod -runpod.resume_pod(pod.id) - -# Terminate the pod -runpod.terminate_pod(pod.id) -``` - -## 📁 | Directory - -```BASH -. -├── docs # Documentation -├── examples # Examples -├── runpod # Package source code -│ ├── api # Language library - API (GraphQL) -│ ├── apps # Apps SDK (App, queue/api/task, volumes, secrets) -│ ├── cli # Legacy CLI functions -│ ├── endpoint # Language library - Endpoints -│ ├── rp_cli # The rp / runpod CLI -│ ├── runtimes # Worker runtime images (queue, api, task) -│ └── serverless # SDK - Serverless Worker -└── tests # Package tests -``` - -## 🤝 | Community and Contributing - -We welcome both pull requests and issues on [GitHub](https://github.com/runpod/runpod-python). Bug fixes and new features are encouraged, but please read our [contributing guide](CONTRIBUTING.md) first. - From d0ef53cdb688d2d9599f7b8fec8fa865abd72435 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:14:07 -0700 Subject: [PATCH 066/120] refactor: split showcase examples from the e2e example suite --- README.md | 2 +- examples/apps/README.md | 60 ++++------------ examples/apps/gpu_inference.py | 44 ++++++++++++ examples/apps/hello_world.py | 32 +++++++++ examples/apps/train_and_eval.py | 71 +++++++++++++++++++ examples/apps/web_service.py | 42 +++++++++++ .../e2e/examples}/01_hello_world.py | 2 +- .../e2e/examples}/02_invocation_styles.py | 2 +- .../apps => tests/e2e/examples}/03_gpu.py | 2 +- .../e2e/examples}/04_dependencies.py | 2 +- .../e2e/examples}/05_api_service.py | 2 +- .../apps => tests/e2e/examples}/06_tasks.py | 2 +- .../e2e/examples}/07_pipelines.py | 2 +- .../apps => tests/e2e/examples}/08_volumes.py | 2 +- .../apps => tests/e2e/examples}/09_secrets.py | 2 +- .../e2e/examples}/10_cached_models.py | 2 +- .../e2e/examples}/11_train_eval.py | 2 +- .../e2e/examples}/12_custom_image.py | 2 +- tests/e2e/examples/README.md | 26 +++++++ 19 files changed, 242 insertions(+), 59 deletions(-) create mode 100644 examples/apps/gpu_inference.py create mode 100644 examples/apps/hello_world.py create mode 100644 examples/apps/train_and_eval.py create mode 100644 examples/apps/web_service.py rename {examples/apps => tests/e2e/examples}/01_hello_world.py (87%) rename {examples/apps => tests/e2e/examples}/02_invocation_styles.py (91%) rename {examples/apps => tests/e2e/examples}/03_gpu.py (93%) rename {examples/apps => tests/e2e/examples}/04_dependencies.py (92%) rename {examples/apps => tests/e2e/examples}/05_api_service.py (94%) rename {examples/apps => tests/e2e/examples}/06_tasks.py (92%) rename {examples/apps => tests/e2e/examples}/07_pipelines.py (93%) rename {examples/apps => tests/e2e/examples}/08_volumes.py (94%) rename {examples/apps => tests/e2e/examples}/09_secrets.py (93%) rename {examples/apps => tests/e2e/examples}/10_cached_models.py (94%) rename {examples/apps => tests/e2e/examples}/11_train_eval.py (97%) rename {examples/apps => tests/e2e/examples}/12_custom_image.py (89%) create mode 100644 tests/e2e/examples/README.md diff --git a/README.md b/README.md index dc719d3a8..47efd626a 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ rp dev main.py # live dev session: edit, re-run, logs stream back rp deploy # deploy production endpoints ``` -Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for complete projects and [docs.runpod.io](https://docs.runpod.io) for the full guide. +Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for runnable examples and [docs.runpod.io](https://docs.runpod.io) for the full guide. ## Contributing diff --git a/examples/apps/README.md b/examples/apps/README.md index f5b445938..9abdbffa0 100644 --- a/examples/apps/README.md +++ b/examples/apps/README.md @@ -1,51 +1,19 @@ -# apps examples +# app examples -each file is a self-contained app demonstrating one feature, runnable -directly and asserting on its own results — the set doubles as an -end-to-end test suite against production. - -| example | shows | -|---|---| -| `01_hello_world` | one queue function, one `.remote()` call | -| `02_invocation_styles` | `.remote()`, `.remote.aio()`, `.spawn()`, `.local()` | -| `03_gpu` | gpu shorthand selection, env vars, torch/cuda | -| `04_dependencies` | pip `dependencies` and apt `system_dependencies` | -| `05_api_service` | `@app.api` class with `@init`/`@get`/`@post`, stateful workers | -| `06_tasks` | ephemeral pods (cpu and gpu) via `@app.task` | -| `07_pipelines` | workers calling sibling resources with nested `.remote()` | -| `08_volumes` | shared network volume between tasks, auto placement | -| `09_secrets` | platform secrets as env vars (needs `rp secret add ex-demo-secret`) | -| `10_cached_models` | platform-staged huggingface weights via `Model` | -| `11_train_eval` | train on one gpu pod, save to a volume, eval on another | -| `12_custom_image` | custom container image with runtime bootstrap | - -## running - -one example: - -```bash -rp dev examples/apps/01_hello_world.py --once -``` - -the whole suite: +each file is a self-contained app showing one way to use the sdk. +run any of them live with `rp dev`: ```bash -python examples/apps/run_all.py # sequential -python examples/apps/run_all.py --jobs 4 # parallel -python examples/apps/run_all.py 01 07 # subset by prefix +rp dev examples/apps/hello_world.py ``` -`09_secrets` needs a secret to exist first: - -```bash -rp secret add ex-demo-secret --value anything -``` - -`12_custom_image` bootstraps the worker runtime onto a bare image by -installing the runpod package at cold start. until the version with -`runpod.runtimes` is on pypi, point the bootstrap at a tarball of the -branch (plain https, not git+ — slim images have no git): - -```bash -export RUNPOD_PACKAGE_SPEC=https://github.com/runpod/runpod-python/archive/refs/heads/feat/apps-sdk.tar.gz -``` +| example | shows | +|---|---| +| `hello_world` | one queue function, one `.remote()` call | +| `gpu_inference` | gpu selection, dependencies, a cuda benchmark | +| `web_service` | `@app.api` class with routes and per-worker state | +| `train_and_eval` | gpu tasks sharing checkpoints through a volume | + +edit a file while the session is running, then press enter to re-run — +workers pick up the new code automatically. the exhaustive +feature-by-feature suite lives in [`tests/e2e/examples`](../../tests/e2e/examples). diff --git a/examples/apps/gpu_inference.py b/examples/apps/gpu_inference.py new file mode 100644 index 000000000..5e5bacb5e --- /dev/null +++ b/examples/apps/gpu_inference.py @@ -0,0 +1,44 @@ +"""Run a function on a GPU. + +Ask for hardware by name — "4090", "H100", "B200" all work, as do +pool ids like GpuGroup.ADA_24 and exact device names. Dependencies +listed on the decorator are ready before your function runs, so the +worker needs no custom image. + + rp dev examples/apps/gpu_inference.py +""" + +import runpod +from runpod import App + +app = App("gpu-demo") + + +@app.queue(gpu="4090", dependencies=["numpy"]) +def matmul_benchmark(size: int = 4096): + import time + + import torch + + device = torch.cuda.get_device_name(0) + print(f"running on {device}") + + a = torch.randn(size, size, device="cuda") + b = torch.randn(size, size, device="cuda") + + torch.cuda.synchronize() + start = time.time() + for _ in range(10): + a @ b + torch.cuda.synchronize() + elapsed = time.time() - start + + tflops = 10 * 2 * size**3 / elapsed / 1e12 + print(f"{size}x{size} matmul: {tflops:.1f} TFLOPS") + return {"device": device, "tflops": round(tflops, 1)} + + +@runpod.local_entrypoint +def main(): + result = matmul_benchmark.remote() + print("benchmark:", result) diff --git a/examples/apps/hello_world.py b/examples/apps/hello_world.py new file mode 100644 index 000000000..0bee9c5c5 --- /dev/null +++ b/examples/apps/hello_world.py @@ -0,0 +1,32 @@ +"""Your first app: a Python function that runs in the cloud. + +An App is a named collection of functions. Decorating a function with +@app.queue turns it into a remote resource — calling .remote() sends +the call to a cloud worker and blocks until the result comes back. + +Try it: + + rp dev examples/apps/hello_world.py + +Edit the greeting while the session is running, then press enter to +re-run — the worker picks up your new code automatically. +""" + +import runpod +from runpod import App + +app = App("hello") + + +@app.queue(cpu="cpu3c-1-2") +def hello(name: str): + # this print shows up in your terminal, streamed from the worker + print(f"running in the cloud, greeting {name}") + return f"hello {name}!" + + +@runpod.local_entrypoint +def main(): + # runs on your machine; hello() runs on a cloud worker + greeting = hello.remote("world") + print(greeting) diff --git a/examples/apps/train_and_eval.py b/examples/apps/train_and_eval.py new file mode 100644 index 000000000..db7b4c443 --- /dev/null +++ b/examples/apps/train_and_eval.py @@ -0,0 +1,71 @@ +"""A training workflow: tasks, volumes, and functions working together. + +@app.task gives each call its own dedicated pod that terminates when +the function returns — right for work that runs minutes to hours. +A Volume persists files between calls and across resources: train() +writes a checkpoint, evaluate() reads it from a different pod. The +volume is created on first use, and both tasks are automatically +placed in its datacenter. + + rp dev examples/apps/train_and_eval.py +""" + +import runpod +from runpod import App, Volume + +app = App("trainer") + +checkpoints = Volume("checkpoints", size=10) + + +@app.task(gpu="4090", volume=checkpoints) +def train(steps: int = 200): + import torch + import torch.nn as nn + + device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"training on {device}") + + # learn y = 3x + 1 from noisy samples + x = torch.randn(2048, 1, device=device) + y = 3 * x + 1 + torch.randn_like(x) * 0.05 + + model = nn.Linear(1, 1).to(device) + opt = torch.optim.SGD(model.parameters(), lr=0.05) + for step in range(1, steps + 1): + loss = ((model(x) - y) ** 2).mean() + opt.zero_grad() + loss.backward() + opt.step() + if step % 50 == 0: + print(f"step {step}/{steps} loss={loss.item():.5f}") + + # checkpoints.path is where the volume is mounted on this pod + run_dir = checkpoints.path / "linear" + run_dir.mkdir(parents=True, exist_ok=True) + torch.save(model.state_dict(), run_dir / "model.pt") + print(f"saved checkpoint to {run_dir}") + return {"checkpoint": "linear", "final_loss": round(loss.item(), 5)} + + +@app.task(gpu="4090", volume=checkpoints) +def evaluate(checkpoint: str): + import torch + import torch.nn as nn + + model = nn.Linear(1, 1) + model.load_state_dict( + torch.load(checkpoints.path / checkpoint / "model.pt") + ) + weight, bias = model.weight.item(), model.bias.item() + print(f"learned: y = {weight:.3f}x + {bias:.3f}") + return {"weight": round(weight, 2), "bias": round(bias, 2)} + + +@runpod.local_entrypoint +def main(): + out = train.remote() + print("train finished:", out) + + fit = evaluate.remote(out["checkpoint"]) + print("evaluation:", fit) diff --git a/examples/apps/web_service.py b/examples/apps/web_service.py new file mode 100644 index 000000000..fe6f6baa6 --- /dev/null +++ b/examples/apps/web_service.py @@ -0,0 +1,42 @@ +"""A load-balanced HTTP service from a plain class. + +@app.api turns a class into a web service: mark methods with @get and +@post to expose routes, and @init to run setup once per worker before +it takes traffic. Workers keep state between requests — this counter +lives in memory on the worker. + + rp dev examples/apps/web_service.py + +Deployed services get a stable URL; in dev you call routes through the +class itself, as shown in the entrypoint. +""" + +import runpod +from runpod import App, get, init, post + +app = App("web") + + +@app.api(cpu="cpu3c-1-2") +class Counter: + @init + def setup(self): + # runs once when a worker starts, before any request + self.count = 0 + print("worker ready, counter at 0") + + @post("/bump") + async def bump(self, body: dict): + self.count += body.get("by", 1) + return {"count": self.count} + + @get("/value") + async def value(self): + return {"count": self.count} + + +@runpod.local_entrypoint +def main(): + print(Counter.post("/bump", {"by": 3})) # {'count': 3} + print(Counter.post("/bump", {"by": 2})) # {'count': 5} + print(Counter.get("/value")) # {'count': 5} diff --git a/examples/apps/01_hello_world.py b/tests/e2e/examples/01_hello_world.py similarity index 87% rename from examples/apps/01_hello_world.py rename to tests/e2e/examples/01_hello_world.py index 36d96506c..5b64bb121 100644 --- a/examples/apps/01_hello_world.py +++ b/tests/e2e/examples/01_hello_world.py @@ -1,6 +1,6 @@ """the smallest possible app: one queue function, one remote call. - rp dev examples/apps/01_hello_world.py --once + rp dev tests/e2e/examples/01_hello_world.py --once """ import runpod diff --git a/examples/apps/02_invocation_styles.py b/tests/e2e/examples/02_invocation_styles.py similarity index 91% rename from examples/apps/02_invocation_styles.py rename to tests/e2e/examples/02_invocation_styles.py index 803daf048..30ae03a37 100644 --- a/examples/apps/02_invocation_styles.py +++ b/tests/e2e/examples/02_invocation_styles.py @@ -1,6 +1,6 @@ """every way to call a function: remote, async, spawn, local. - rp dev examples/apps/02_invocation_styles.py --once + rp dev tests/e2e/examples/02_invocation_styles.py --once """ import asyncio diff --git a/examples/apps/03_gpu.py b/tests/e2e/examples/03_gpu.py similarity index 93% rename from examples/apps/03_gpu.py rename to tests/e2e/examples/03_gpu.py index 044dae88a..993bd8857 100644 --- a/examples/apps/03_gpu.py +++ b/tests/e2e/examples/03_gpu.py @@ -1,6 +1,6 @@ """gpu selection: shorthand names resolve to real devices. - rp dev examples/apps/03_gpu.py --once + rp dev tests/e2e/examples/03_gpu.py --once """ import runpod diff --git a/examples/apps/04_dependencies.py b/tests/e2e/examples/04_dependencies.py similarity index 92% rename from examples/apps/04_dependencies.py rename to tests/e2e/examples/04_dependencies.py index 9cc7fba16..a7ad41f36 100644 --- a/examples/apps/04_dependencies.py +++ b/tests/e2e/examples/04_dependencies.py @@ -1,6 +1,6 @@ """pip and apt dependencies, available on the worker without a custom image. - rp dev examples/apps/04_dependencies.py --once + rp dev tests/e2e/examples/04_dependencies.py --once """ import runpod diff --git a/examples/apps/05_api_service.py b/tests/e2e/examples/05_api_service.py similarity index 94% rename from examples/apps/05_api_service.py rename to tests/e2e/examples/05_api_service.py index 47d1c8a05..f5b460639 100644 --- a/examples/apps/05_api_service.py +++ b/tests/e2e/examples/05_api_service.py @@ -1,6 +1,6 @@ """a load-balanced http service from a class with route markers. - rp dev examples/apps/05_api_service.py --once + rp dev tests/e2e/examples/05_api_service.py --once """ import runpod diff --git a/examples/apps/06_tasks.py b/tests/e2e/examples/06_tasks.py similarity index 92% rename from examples/apps/06_tasks.py rename to tests/e2e/examples/06_tasks.py index afff8fe81..e60b00765 100644 --- a/examples/apps/06_tasks.py +++ b/tests/e2e/examples/06_tasks.py @@ -1,6 +1,6 @@ """tasks: one ephemeral pod per call, terminated when done. - rp dev examples/apps/06_tasks.py --once + rp dev tests/e2e/examples/06_tasks.py --once """ import runpod diff --git a/examples/apps/07_pipelines.py b/tests/e2e/examples/07_pipelines.py similarity index 93% rename from examples/apps/07_pipelines.py rename to tests/e2e/examples/07_pipelines.py index c859b4533..74519dc96 100644 --- a/examples/apps/07_pipelines.py +++ b/tests/e2e/examples/07_pipelines.py @@ -1,6 +1,6 @@ """functions calling functions: workers invoke sibling resources remotely. - rp dev examples/apps/07_pipelines.py --once + rp dev tests/e2e/examples/07_pipelines.py --once """ import runpod diff --git a/examples/apps/08_volumes.py b/tests/e2e/examples/08_volumes.py similarity index 94% rename from examples/apps/08_volumes.py rename to tests/e2e/examples/08_volumes.py index e9a7bc7e2..4b6364eb1 100644 --- a/examples/apps/08_volumes.py +++ b/tests/e2e/examples/08_volumes.py @@ -3,7 +3,7 @@ the volume is created on first use; placement co-locates both tasks in the volume's datacenter automatically. - rp dev examples/apps/08_volumes.py --once + rp dev tests/e2e/examples/08_volumes.py --once """ import runpod diff --git a/examples/apps/09_secrets.py b/tests/e2e/examples/09_secrets.py similarity index 93% rename from examples/apps/09_secrets.py rename to tests/e2e/examples/09_secrets.py index fec3ff73c..242060c26 100644 --- a/examples/apps/09_secrets.py +++ b/tests/e2e/examples/09_secrets.py @@ -3,7 +3,7 @@ requires the secret to exist first: rp secret add ex-demo-secret --value "s3cret" - rp dev examples/apps/09_secrets.py --once + rp dev tests/e2e/examples/09_secrets.py --once """ import runpod diff --git a/examples/apps/10_cached_models.py b/tests/e2e/examples/10_cached_models.py similarity index 94% rename from examples/apps/10_cached_models.py rename to tests/e2e/examples/10_cached_models.py index 923118920..0032503dc 100644 --- a/examples/apps/10_cached_models.py +++ b/tests/e2e/examples/10_cached_models.py @@ -1,6 +1,6 @@ """platform-cached model weights: staged on the host before the worker starts. - rp dev examples/apps/10_cached_models.py --once + rp dev tests/e2e/examples/10_cached_models.py --once """ import runpod diff --git a/examples/apps/11_train_eval.py b/tests/e2e/examples/11_train_eval.py similarity index 97% rename from examples/apps/11_train_eval.py rename to tests/e2e/examples/11_train_eval.py index b7e1eaf3b..c988ba46c 100644 --- a/examples/apps/11_train_eval.py +++ b/tests/e2e/examples/11_train_eval.py @@ -1,6 +1,6 @@ """a real workflow: train on one gpu pod, save to a volume, eval on another. - rp dev examples/apps/11_train_eval.py --once + rp dev tests/e2e/examples/11_train_eval.py --once """ import runpod diff --git a/examples/apps/12_custom_image.py b/tests/e2e/examples/12_custom_image.py similarity index 89% rename from examples/apps/12_custom_image.py rename to tests/e2e/examples/12_custom_image.py index 4697764f1..77782b069 100644 --- a/examples/apps/12_custom_image.py +++ b/tests/e2e/examples/12_custom_image.py @@ -1,7 +1,7 @@ """a custom container image: the sdk bootstraps its runtime onto any image with python3. - rp dev examples/apps/12_custom_image.py --once + rp dev tests/e2e/examples/12_custom_image.py --once """ import runpod diff --git a/tests/e2e/examples/README.md b/tests/e2e/examples/README.md new file mode 100644 index 000000000..86c531346 --- /dev/null +++ b/tests/e2e/examples/README.md @@ -0,0 +1,26 @@ +# e2e example suite + +self-asserting apps exercising every sdk feature against production. +each file runs via `rp dev --once` and fails loudly when a +result is wrong, so the set doubles as a release gate. + +```bash +python tests/e2e/examples/run_all.py # everything +python tests/e2e/examples/run_all.py --jobs 4 # parallel +python tests/e2e/examples/run_all.py 01 07 # subset by prefix +``` + +prerequisites: + +```bash +rp login +rp secret add ex-demo-secret --value anything # for 09_secrets +``` + +`12_custom_image` installs the runpod package at cold start; until the +release with `runpod.runtimes` is on pypi, point the bootstrap at the +branch tarball (plain https, not git+ — slim images have no git): + +```bash +export RUNPOD_PACKAGE_SPEC=https://github.com/runpod/runpod-python/archive/refs/heads/feat/apps-sdk.tar.gz +``` From 5a301bd6ca4b537f4994bf0800474bbf4e0b9bbf Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:46:03 -0700 Subject: [PATCH 067/120] fix: anonymous login polling and local-python deploy default --- runpod/api/graphql.py | 16 ++++++---------- runpod/apps/api.py | 10 +++++----- runpod/rp_cli/main.py | 24 +++++++++++++++++++++--- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/runpod/api/graphql.py b/runpod/api/graphql.py index 02a033b72..367874250 100644 --- a/runpod/api/graphql.py +++ b/runpod/api/graphql.py @@ -86,7 +86,7 @@ async def run_graphql_query_async( api_key: Optional[str] = None, variables: Optional[Dict[str, Any]] = None, timeout: float = 60.0, - allow_anonymous: bool = False, + anonymous: bool = False, ) -> Dict[str, Any]: """ Async variant of run_graphql_query, sharing the same url, headers, @@ -97,18 +97,14 @@ async def run_graphql_query_async( api_key: Optional API key to use for this query. variables: Optional GraphQL variables to send with the query. timeout: Total request timeout in seconds. - allow_anonymous: Send the request without credentials when no - key is available (pre-login flows). + anonymous: Send the request without credentials even when a key + is available. The browser-login flow requires this: the + server only releases the granted key to guest requests, so + a stale stored key would make the poll spin forever. """ import aiohttp # pylint: disable=import-outside-toplevel - if allow_anonymous: - try: - effective_api_key: Optional[str] = _resolve_api_key(api_key) - except error.AuthenticationError: - effective_api_key = None - else: - effective_api_key = _resolve_api_key(api_key) + effective_api_key = None if anonymous else _resolve_api_key(api_key) client_timeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(timeout=client_timeout) as session: diff --git a/runpod/apps/api.py b/runpod/apps/api.py index d796557ab..e973f30e9 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -449,9 +449,7 @@ async def create_auth_request(self) -> Dict[str, Any]: } } """ - response = await run_graphql_query_async( - mutation, api_key=self._api_key, allow_anonymous=True - ) + response = await run_graphql_query_async(mutation, anonymous=True) return response["data"].get("createFlashAuthRequest") or {} async def get_auth_request_status(self, request_id: str) -> Dict[str, Any]: @@ -465,11 +463,13 @@ async def get_auth_request_status(self, request_id: str) -> Dict[str, Any]: } } """ + # anonymous by design: the server only releases the granted key + # to guest requests, so authenticating the poll (e.g. with a + # stale stored key) would return apiKey=null forever response = await run_graphql_query_async( query, - api_key=self._api_key, variables={"flashAuthRequestId": request_id}, - allow_anonymous=True, + anonymous=True, ) return response["data"].get("flashAuthRequestStatus") or {} diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index b337484d0..84251faad 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -139,9 +139,9 @@ def update(version_opt): @click.option("--env", "-e", "env", default=None, help="Target environment name.") @click.option( "--python-version", - default="3.12", - show_default=True, - help="Worker python version for dependency wheels (3.10-3.14).", + default=None, + help="Worker python version for dependency wheels (3.10-3.14). " + "Defaults to the python running the cli.", ) @click.option( "--exclude", @@ -166,6 +166,24 @@ def deploy(target, env, python_version, exclude, build_only): logging.getLogger("runpod.apps").setLevel(logging.WARNING) + if python_version is None: + from runpod.apps.images import ( + DEFAULT_PYTHON_VERSION, + local_python_version, + ) + + try: + python_version = local_python_version() + except RuntimeError: + # deployed calls are plain json (no pickle compat needed), + # so an unsupported local interpreter falls back instead of + # failing like dev does + ui.warn( + f"local python has no runtime image; building for " + f"{DEFAULT_PYTHON_VERSION} (override with --python-version)" + ) + python_version = DEFAULT_PYTHON_VERSION + target = (target or Path.cwd()).resolve() if not target.exists(): _fail(f"{target} does not exist") From c657720e02d94cabcc819eb8c7b3de9914569c7b Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:50:10 -0700 Subject: [PATCH 068/120] fix: retry proxy propagation 404s when polling task results --- runpod/apps/tasks.py | 28 ++++++++---- tests/test_apps/test_tasks.py | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 8522c7715..db7fe8944 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -272,16 +272,28 @@ async def submit(self, request: Dict[str, Any]) -> None: return async def poll_result(self) -> Optional[Dict[str, Any]]: - """fetch /result; returns the response dict once DONE, else None.""" + """fetch /result; returns the response dict once DONE, else None. + + the proxy answers 404 for pods an edge node has not learned + about yet (the same race /submit tolerates), so 404s are + retried briefly; a persistent 404 means the pod is gone and + surfaces as an error. + """ url = f"{_proxy_url(self.pod_id)}/result" + attempts = 6 async with aiohttp.ClientSession() as session: - async with session.get( - url, - headers=self._headers, - timeout=aiohttp.ClientTimeout(total=30), - ) as resp: - resp.raise_for_status() - data = await resp.json() + for attempt in range(attempts): + async with session.get( + url, + headers=self._headers, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + if resp.status == 404 and attempt < attempts - 1: + await asyncio.sleep(2 * (attempt + 1)) + continue + resp.raise_for_status() + data = await resp.json() + break if data.get("status") == "DONE": return data.get("response") return None diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index cf449d7c0..d8cf9b3fd 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -276,6 +276,88 @@ def t(): instance.submit.assert_awaited_once() +class TestPollResult: + """poll_result must tolerate proxy propagation 404s.""" + + def _server(self, responses): + """a local server that pops one canned (status, body) per hit.""" + import http.server + import threading + + hits = {"count": 0} + + class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + hits["count"] += 1 + status, body = responses[min(hits["count"], len(responses)) - 1] + payload = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), H) + httpd.daemon_threads = True + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return httpd, hits + + def _poll(self, url): + import asyncio + + from runpod.apps.tasks import TaskExecution + + spec = ResourceSpec(kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"]) + execution = TaskExecution(spec) + execution.pod_id = "fake" + with ( + patch("runpod.apps.tasks._proxy_url", return_value=url), + patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()), + ): + return asyncio.run(execution.poll_result()) + + def test_retries_transient_404_then_returns_result(self): + httpd, hits = self._server( + [ + (404, {"error": "not found"}), + (404, {"error": "not found"}), + (200, {"status": "DONE", "response": {"success": True}}), + ] + ) + try: + url = f"http://127.0.0.1:{httpd.server_port}" + assert self._poll(url) == {"success": True} + assert hits["count"] == 3 + finally: + httpd.shutdown() + httpd.server_close() + + def test_persistent_404_raises(self): + import aiohttp + + httpd, hits = self._server([(404, {"error": "not found"})]) + try: + url = f"http://127.0.0.1:{httpd.server_port}" + with pytest.raises(aiohttp.ClientResponseError): + self._poll(url) + assert hits["count"] == 6 + finally: + httpd.shutdown() + httpd.server_close() + + def test_running_returns_none(self): + httpd, _ = self._server([(200, {"status": "RUNNING", "response": None})]) + try: + url = f"http://127.0.0.1:{httpd.server_port}" + assert self._poll(url) is None + finally: + httpd.shutdown() + httpd.server_close() + + class TestGpuPoolExpansion: def test_pool_id_expands_to_device_names(self): from runpod.apps.tasks import _device_names From 3c6e9a718d65094fc278681191649e544a172a35 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:08:22 -0700 Subject: [PATCH 069/120] fix: install resource dependencies on live api sync --- runpod/apps/handles.py | 6 ++++- runpod/apps/targets.py | 25 ++++++++++++------ runpod/runtimes/api/server.py | 18 +++++++++++++ tests/test_apps/test_api_server.py | 41 ++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 8e6c4c57e..768fb185e 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -181,7 +181,11 @@ async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: # server-side; other targets ignore the reference setter = getattr(target, "attach_source", None) if setter is not None: - setter(self._handle.__wrapped__, self._handle.spec.name) + setter( + self._handle.__wrapped__, + self._handle.spec.name, + self._handle.spec, + ) return await target.request(self._method, path, body, **kwargs) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index bc900d4d2..a9ddfd317 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -361,18 +361,23 @@ def __init__( self.metrics_key = metrics_key self._source_target: Optional[Any] = None self._source_resource: str = "" + self._source_spec: Optional[ResourceSpec] = None # source hash the worker last confirmed; None forces a sync self._synced_hash: Optional[str] = None - def attach_source(self, target: Any, resource: str) -> None: + def attach_source( + self, target: Any, resource: str, spec: Optional[ResourceSpec] = None + ) -> None: """register the object whose module backs this api resource. live api endpoints materialize their routes from module source pushed over /_runpod/sync (once per source change), keeping dev - behavior identical to deployed. + behavior identical to deployed. the spec rides along so the + worker can install the resource's dependencies before @init. """ self._source_target = target self._source_resource = resource + self._source_spec = spec async def _sync_source(self, timeout: float) -> None: """push the current module source to the worker if it changed.""" @@ -387,12 +392,16 @@ async def _sync_source(self, timeout: float) -> None: if digest == self._synced_hash: return url = f"https://{self.endpoint_id}.{_lb_domain()}/_runpod/sync" - await _post_json( - url, - {"source": source, "resource": self._source_resource}, - _headers(), - timeout, - ) + payload: Dict[str, Any] = { + "source": source, + "resource": self._source_resource, + } + if self._source_spec is not None: + payload["dependencies"] = self._source_spec.dependencies + payload["system_dependencies"] = ( + self._source_spec.system_dependencies + ) + await _post_json(url, payload, _headers(), timeout) self._synced_hash = digest def build_payload( diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py index 411494dac..8637c06d5 100644 --- a/runpod/runtimes/api/server.py +++ b/runpod/runtimes/api/server.py @@ -164,6 +164,23 @@ async def ping(): return app +def _install_live_dependencies(request: dict) -> None: + """install a live resource's dependencies before its module runs. + + dev workers boot on the bare runtime image; deployed workers get + dependencies baked into the build artifact, so live mode installs + them at sync time to match. + """ + from runpod.runtimes.task.runner import _install, _install_system + + error = _install_system(request.get("system_dependencies")) + if error: + raise RuntimeError(error) + error = _install(request.get("dependencies"), "dependencies") + if error: + raise RuntimeError(error) + + async def _materialize_live_api(source: str, resource: str) -> Any: """exec shipped module source and build the api app for a resource. @@ -235,6 +252,7 @@ async def sync(request: dict): resource = request.get("resource") or "" digest = hashlib.sha256(source.encode()).hexdigest() if digest != self._hash: + _install_live_dependencies(request) self._inner = await _materialize_live_api(source, resource) self._hash = digest return {"status": "synced", "hash": digest} diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py index 28307add4..054c2359d 100644 --- a/tests/test_apps/test_api_server.py +++ b/tests/test_apps/test_api_server.py @@ -198,6 +198,47 @@ def test_sync_materializes_routes(self): # core routes still work after materialization assert client.get("/ping").status_code == 200 + def test_sync_installs_dependencies_before_materializing(self): + from unittest.mock import patch + + from fastapi.testclient import TestClient + + from runpod.runtimes.api.server import _build_live_app + + source = ( + "import runpod\n" + "from runpod.apps.markers import get\n" + "app = runpod.App('live-deps')\n" + "@app.api(cpu='cpu3c-1-2', dependencies=['somepkg'])\n" + "class D:\n" + " @get('/ok')\n" + " async def ok(self):\n" + " return {'ok': True}\n" + ) + client = TestClient(_build_live_app()) + with ( + patch( + "runpod.runtimes.task.runner._install", return_value=None + ) as install, + patch( + "runpod.runtimes.task.runner._install_system", + return_value=None, + ) as install_system, + ): + r = client.post( + "/_runpod/sync", + json={ + "source": source, + "resource": "D", + "dependencies": ["somepkg"], + "system_dependencies": ["libfoo"], + }, + ) + assert r.status_code == 200 + install.assert_called_once_with(["somepkg"], "dependencies") + install_system.assert_called_once_with(["libfoo"]) + assert client.get("/ok").json() == {"ok": True} + def test_resync_same_source_keeps_state(self): from fastapi.testclient import TestClient From 389dfa57e7b7c6b62eef37edb94f3a2526765ba1 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:39:43 -0700 Subject: [PATCH 070/120] perf: cache dependency layers and skip gpu-base rebuilds for code-only changes --- .dockerignore | 16 ++++++++++++++++ .github/workflows/runtimes.yml | 25 ++++++++++++++++++++++--- runpod/runtimes/api/Dockerfile | 13 +++++++++++-- runpod/runtimes/queue/Dockerfile | 13 +++++++++++-- runpod/runtimes/task/Dockerfile | 12 ++++++++++-- 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0dcfa1acb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +# keep the runtime image build context to what pip install needs: +# package source, metadata, and the files setup.py/MANIFEST.in read +.git +.github +.venv +build +benchmark_results +docs +examples +job_files +scripts +tests +uv.lock +**/__pycache__ +**/*.pyc +*.egg-info diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index a36f97da4..7ef5e4c89 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -19,6 +19,21 @@ env: TAG_SUFFIX: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/apps-sdk') && 'latest' || 'dev' }} jobs: + changes: + runs-on: ubuntu-latest + outputs: + gpu-base: ${{ steps.filter.outputs.gpu-base }} + steps: + - uses: actions/checkout@v7 + + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + gpu-base: + - 'runpod/runtimes/gpu-base/**' + - '.github/workflows/runtimes.yml' + test-runner: runs-on: ubuntu-latest steps: @@ -38,7 +53,11 @@ jobs: build-gpu-base: runs-on: ubuntu-latest - needs: [test-runner] + # the torch layer is multi-GB; only rebuild when the gpu-base + # definition itself changes. runtime images build FROM the + # registry tag, so the previously published base keeps working + needs: [changes, test-runner] + if: needs.changes.outputs.gpu-base == 'true' strategy: # a transient registry/mirror failure on one leg must not cancel # the other python versions @@ -79,8 +98,8 @@ jobs: build-runtimes: runs-on: ubuntu-latest - # gpu legs build FROM the registry's gpu-base tag, so a failed - # gpu-base rebuild (e.g. torch mirror outage) must not block + # gpu legs build FROM the registry's gpu-base tag, so a skipped or + # failed gpu-base rebuild (e.g. torch mirror outage) must not block # runtime images: the previously published base keeps working needs: [test-runner, build-gpu-base] if: always() && needs.test-runner.result == 'success' diff --git a/runpod/runtimes/api/Dockerfile b/runpod/runtimes/api/Dockerfile index 3509bd730..ded53a686 100644 --- a/runpod/runtimes/api/Dockerfile +++ b/runpod/runtimes/api/Dockerfile @@ -12,9 +12,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* -# install the runpod package (worker loop + runtimes) from the build context +# dependency layer: cached until requirements.txt changes, so code-only +# edits skip the full dependency install +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle "uvicorn>=0.30" \ + && rm /tmp/requirements.txt + +# install the runpod package (worker loop + runtimes) from the build +# context. the context has no .git, so setuptools-scm needs a pretend +# version COPY . /src -RUN pip install --no-cache-dir /src cloudpickle "uvicorn>=0.30" && rm -rf /src +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ + pip install --no-cache-dir --no-deps /src && rm -rf /src EXPOSE 80 CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/runpod/runtimes/queue/Dockerfile b/runpod/runtimes/queue/Dockerfile index df7870957..bbaecee68 100644 --- a/runpod/runtimes/queue/Dockerfile +++ b/runpod/runtimes/queue/Dockerfile @@ -12,8 +12,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* -# install the runpod package (worker loop + runtimes) from the build context +# dependency layer: cached until requirements.txt changes, so code-only +# edits skip the full dependency install +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle \ + && rm /tmp/requirements.txt + +# install the runpod package (worker loop + runtimes) from the build +# context. the context has no .git, so setuptools-scm needs a pretend +# version COPY . /src -RUN pip install --no-cache-dir /src cloudpickle && rm -rf /src +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ + pip install --no-cache-dir --no-deps /src && rm -rf /src CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/runpod/runtimes/task/Dockerfile b/runpod/runtimes/task/Dockerfile index a1a9f18d5..1e981d9ea 100644 --- a/runpod/runtimes/task/Dockerfile +++ b/runpod/runtimes/task/Dockerfile @@ -11,10 +11,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* +# dependency layer: cached until requirements.txt changes, so code-only +# edits skip the full dependency install +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle \ + && rm /tmp/requirements.txt + # install the runpod package: task payloads carry the user's full module -# source, which imports runpod (decorators, nested .remote calls) +# source, which imports runpod (decorators, nested .remote calls). +# the context has no .git, so setuptools-scm needs a pretend version COPY . /src -RUN pip install --no-cache-dir /src cloudpickle && rm -rf /src +RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ + pip install --no-cache-dir --no-deps /src && rm -rf /src EXPOSE 8080 CMD ["python", "-m", "runpod.runtimes.task.runner"] From 5b5e96a6c3edd97491eeb949d94f070d306867e8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:44:58 -0700 Subject: [PATCH 071/120] refactor: drop the config wizard and exec command group --- runpod/cli/entry.py | 5 - runpod/cli/groups/config/commands.py | 45 ------- runpod/cli/groups/exec/__init__.py | 1 - runpod/cli/groups/exec/commands.py | 27 ----- runpod/cli/groups/exec/functions.py | 15 --- runpod/cli/groups/exec/helpers.py | 37 ------ runpod/rp_cli/main.py | 4 +- .../test_cli_groups/test_config_commands.py | 114 ------------------ .../test_cli_groups/test_exec_commands.py | 59 --------- .../test_cli_groups/test_exec_functions.py | 29 ----- .../test_cli_groups/test_exec_helpers.py | 74 ------------ 11 files changed, 1 insertion(+), 409 deletions(-) delete mode 100644 runpod/cli/groups/config/commands.py delete mode 100644 runpod/cli/groups/exec/__init__.py delete mode 100644 runpod/cli/groups/exec/commands.py delete mode 100644 runpod/cli/groups/exec/functions.py delete mode 100644 runpod/cli/groups/exec/helpers.py delete mode 100644 tests/test_cli/test_cli_groups/test_config_commands.py delete mode 100644 tests/test_cli/test_cli_groups/test_exec_commands.py delete mode 100644 tests/test_cli/test_cli_groups/test_exec_functions.py delete mode 100644 tests/test_cli/test_cli_groups/test_exec_helpers.py diff --git a/runpod/cli/entry.py b/runpod/cli/entry.py index 4794b05e8..f2ddb9c21 100644 --- a/runpod/cli/entry.py +++ b/runpod/cli/entry.py @@ -6,8 +6,6 @@ import click -from .groups.config.commands import config_wizard -from .groups.exec.commands import exec_cli from .groups.pod.commands import pod_cli from .groups.ssh.commands import ssh_cli @@ -17,8 +15,5 @@ def runpod_cli(): """A collection of CLI functions for Runpod.""" -runpod_cli.add_command(config_wizard) # runpod config - runpod_cli.add_command(ssh_cli) # runpod ssh runpod_cli.add_command(pod_cli) # runpod pod -runpod_cli.add_command(exec_cli) # runpod exec diff --git a/runpod/cli/groups/config/commands.py b/runpod/cli/groups/config/commands.py deleted file mode 100644 index cae08c1b6..000000000 --- a/runpod/cli/groups/config/commands.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Commands for the config command group -""" - -import sys - -import click - -from .functions import check_credentials, set_credentials - - -@click.command("config", help="Configures the Runpod CLI with the user's API key.") -@click.argument("api-key", required=False, default=None) -@click.option( - "--profile", default="default", help="The profile to set the credentials for." -) -@click.option("--check", is_flag=True, help="Check if credentials are already set.") -def config_wizard(api_key, profile, check): - """Starts the configuration wizard to set up the Runpod CLI. - If credentials are already set, prompts the user to overwrite them. - """ - valid, error = check_credentials(profile) - - if check and valid: - click.echo("Credentials already set for profile: " + profile) - sys.exit(0) - elif check and not valid: - click.echo(error) - sys.exit(1) - - if valid: - click.confirm( - f"Credentials already set for profile: {profile}. Overwrite?", abort=True - ) - - if api_key is None: - click.echo("Please enter your Runpod API Key.") - click.echo("You can find it at https://console.runpod.io/user/settings") - api_key = click.prompt( - " > Runpod API Key", hide_input=False, confirmation_prompt=False - ) - - set_credentials(api_key, profile, overwrite=True) - - click.echo(f"Credentials set for profile: {profile} in ~/.runpod/config.toml") diff --git a/runpod/cli/groups/exec/__init__.py b/runpod/cli/groups/exec/__init__.py deleted file mode 100644 index b3a934dc6..000000000 --- a/runpod/cli/groups/exec/__init__.py +++ /dev/null @@ -1 +0,0 @@ -""" A collection of CLI execution tools """ diff --git a/runpod/cli/groups/exec/commands.py b/runpod/cli/groups/exec/commands.py deleted file mode 100644 index 1d372269c..000000000 --- a/runpod/cli/groups/exec/commands.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Runpod | CLI | Exec | Commands -""" - -import click - -from .functions import python_over_ssh -from .helpers import get_session_pod - - -@click.group("exec", help="Execute commands in a pods.") -def exec_cli(): - """A collection of CLI functions for Exec.""" - - -@exec_cli.command("python") -@click.option("--pod_id", default=None, help="The pod ID to run the command on.") -@click.argument("file", type=click.Path(exists=True), required=True) -def remote_python(pod_id, file): - """ - Runs a remote Python shell. - """ - if pod_id is None: - pod_id = get_session_pod() - - click.echo("Running remote Python shell...") - python_over_ssh(pod_id, file) diff --git a/runpod/cli/groups/exec/functions.py b/runpod/cli/groups/exec/functions.py deleted file mode 100644 index f5bc0a8f2..000000000 --- a/runpod/cli/groups/exec/functions.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Runpod | CLI | Exec | Functions -""" - -from runpod.cli.utils import ssh_cmd - - -def python_over_ssh(pod_id, file): - """ - Runs a Python file over SSH. - """ - ssh = ssh_cmd.SSHConnection(pod_id) - ssh.put_file(file, f"/root/{file}") - ssh.run_commands([f"python3.10 /root/{file}"]) - ssh.close() diff --git a/runpod/cli/groups/exec/helpers.py b/runpod/cli/groups/exec/helpers.py deleted file mode 100644 index 687c2b978..000000000 --- a/runpod/cli/groups/exec/helpers.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Helper functions for the runpod cli group exec command""" - -import os - -import click - -from runpod import get_pod - -POD_ID_FILE = os.path.join(os.path.expanduser("~"), ".runpod", "pod_id") - - -def get_session_pod(): - """Returns the pod_id for the current session. - - Session pod is used to execute commands and run scripts remotely. - - - If the pod_id is already set, return it. - - If the pod_id is not set, prompt the user for it. - - Save the pod_id to a file so that the user doesn't have to enter it again. - """ - pod_id = None - - if os.path.exists(POD_ID_FILE): - with open(POD_ID_FILE, "r", encoding="UTF-8") as pod_file: - pod_id = pod_file.read().strip() - - # Confirm that the pod_id is valid - if get_pod(pod_id) is not None: - return pod_id - - # If file doesn't exist or is empty, prompt user for the pod_id - pod_id = click.prompt("Please provide the pod ID") - os.makedirs(os.path.dirname(POD_ID_FILE), exist_ok=True) - with open(POD_ID_FILE, "w", encoding="UTF-8") as pod_file: - pod_file.write(pod_id) - - return pod_id diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 84251faad..8d7f19cca 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -990,12 +990,10 @@ def undeploy(app_name, env_name, yes): def _mount_groups() -> None: - from runpod.cli.groups.config.commands import config_wizard - from runpod.cli.groups.exec.commands import exec_cli from runpod.cli.groups.pod.commands import pod_cli from runpod.cli.groups.ssh.commands import ssh_cli - for command in (config_wizard, ssh_cli, pod_cli, exec_cli): + for command in (ssh_cli, pod_cli): cli.add_command(command) diff --git a/tests/test_cli/test_cli_groups/test_config_commands.py b/tests/test_cli/test_cli_groups/test_config_commands.py deleted file mode 100644 index 16da467aa..000000000 --- a/tests/test_cli/test_cli_groups/test_config_commands.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -Runpod | Tests | CLI | Commands -""" - -import unittest -from unittest.mock import patch - -from click.testing import CliRunner - -from runpod.cli.entry import runpod_cli - - -class TestCommands(unittest.TestCase): - """A collection of tests for the CLI commands.""" - - def setUp(self): - self.runner = CliRunner() - - def test_config_wizard(self): - """Tests the config command.""" - with patch("click.echo") as mock_echo, patch( - "runpod.cli.groups.config.commands.set_credentials" - ) as mock_set_credentials, patch( - "runpod.cli.groups.config.commands.check_credentials" - ) as mock_check_creds, patch( - "click.confirm", return_value=True - ) as mock_confirm, patch( - "click.prompt", return_value="KEY" - ) as mock_prompt: - - # Assuming credentials aren't set (doesn't prompt for overwrite) - mock_check_creds.return_value = (False, None) - - # Successful Call with Direct Key - result = self.runner.invoke( - runpod_cli, ["config", "--profile", "test", "KEY"] - ) - assert result.exit_code == 0 - mock_set_credentials.assert_called_with("KEY", "test", overwrite=True) - assert mock_echo.call_count == 1 - - # Successful Call with Prompted Key (since direct key isn't provided) - result = self.runner.invoke(runpod_cli, ["config", "--profile", "test"]) - assert result.exit_code == 0 - mock_set_credentials.assert_called_with("KEY", "test", overwrite=True) - mock_prompt.assert_called_with( - " > Runpod API Key", hide_input=False, confirmation_prompt=False - ) # pylint: disable=line-too-long - - # Simulating existing credentials, prompting for overwrite - mock_check_creds.return_value = (True, None) - result = self.runner.invoke(runpod_cli, ["config", "--profile", "test"]) - mock_confirm.assert_called_with( - "Credentials already set for profile: test. Overwrite?", abort=True - ) - - # Unsuccessful Call - mock_set_credentials.side_effect = ValueError() - result = self.runner.invoke( - runpod_cli, ["config", "--profile", "test", "KEY"] - ) - assert result.exit_code == 1 - - def test_check_flag(self): - """Tests the --check flag.""" - with patch( - "runpod.cli.groups.config.commands.check_credentials" - ) as mock_check_creds: - # Assuming credentials are set - mock_check_creds.return_value = (True, None) - result = self.runner.invoke( - runpod_cli, ["config", "--check", "--profile", "test"] - ) - assert result.exit_code == 0 - - # Assuming credentials aren't set - mock_check_creds.return_value = (False, "Credentials not found.") - result = self.runner.invoke( - runpod_cli, ["config", "--check", "--profile", "test"] - ) - assert result.exit_code == 1 - - def test_output_messages(self): - """Tests the output messages for the config command.""" - with patch("click.echo") as mock_echo, patch( - "runpod.cli.groups.config.commands.set_credentials" - ) as mock_set_credentials, patch( - "runpod.cli.groups.config.commands.check_credentials", - return_value=(False, None), - ) as mock_check_creds: # pylint: disable=line-too-long - result = self.runner.invoke( - runpod_cli, ["config", "KEY", "--profile", "test"] - ) - mock_set_credentials.assert_called_with("KEY", "test", overwrite=True) - mock_echo.assert_any_call( - "Credentials set for profile: test in ~/.runpod/config.toml" - ) - assert result.exit_code == 0 - assert mock_check_creds.call_count == 1 - - def test_api_key_prompt(self): - """Tests the API key prompt.""" - with patch("click.prompt", return_value="KEY") as mock_prompt, patch( - "runpod.cli.groups.config.commands.set_credentials" - ) as mock_set_credentials, patch( - "runpod.cli.groups.config.commands.check_credentials", - return_value=(False, None) - ): - result = self.runner.invoke(runpod_cli, ["config", "--profile", "test"]) - mock_prompt.assert_called_with( - " > Runpod API Key", hide_input=False, confirmation_prompt=False - ) # pylint: disable=line-too-long - mock_set_credentials.assert_called_with("KEY", "test", overwrite=True) - assert result.exit_code == 0 diff --git a/tests/test_cli/test_cli_groups/test_exec_commands.py b/tests/test_cli/test_cli_groups/test_exec_commands.py deleted file mode 100644 index d8a0edf6e..000000000 --- a/tests/test_cli/test_cli_groups/test_exec_commands.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Tests for Runpod CLI exec commands. -""" - -import tempfile -import unittest -from unittest.mock import patch - -import click -from click.testing import CliRunner - -from runpod.cli.entry import runpod_cli - - -class TestExecCommands(unittest.TestCase): - """Tests for Runpod CLI exec commands.""" - def setUp(self): - self.runner = CliRunner() - self.runner = click.testing.CliRunner() - - def test_remote_python_with_provided_pod_id(self): - """Tests the remote_python command when pod_id is provided directly.""" - with tempfile.NamedTemporaryFile() as temp_file, patch( - "runpod.cli.groups.exec.commands.python_over_ssh" - ) as mock_python_over_ssh: - result = self.runner.invoke( - runpod_cli, - ["exec", "python", "--pod_id", "sample_pod_id", temp_file.name], - ) - assert result.exit_code == 0 - mock_python_over_ssh.assert_called_with("sample_pod_id", temp_file.name) - - def test_remote_python_without_provided_pod_id_stored(self): - """Tests the remote_python command when pod_id is retrieved from storage.""" - with tempfile.NamedTemporaryFile() as temp_file, patch( - "runpod.cli.groups.exec.commands.python_over_ssh" - ) as mock_python_over_ssh, patch( - "runpod.cli.groups.exec.commands.get_session_pod", - return_value="stored_pod_id", - ) as mock_get_pod_id: # pylint: disable=line-too-long - mock_python_over_ssh.return_value = None - result = self.runner.invoke(runpod_cli, ["exec", "python", temp_file.name]) - assert result.exit_code == 0 - mock_get_pod_id.assert_called_once() - mock_python_over_ssh.assert_called_with("stored_pod_id", temp_file.name) - - def test_remote_python_without_provided_pod_id_prompt(self): - """Tests the remote_python command when pod_id is prompted to user.""" - with tempfile.NamedTemporaryFile() as temp_file, patch( - "runpod.cli.groups.exec.commands.python_over_ssh" - ) as mock_python_over_ssh, patch( - "runpod.cli.groups.exec.commands.get_session_pod", - return_value="prompted_pod_id", - ) as mock_get_pod_id: # pylint: disable=line-too-long - mock_python_over_ssh.return_value = None - result = self.runner.invoke(runpod_cli, ["exec", "python", temp_file.name]) - assert result.exit_code == 0 - mock_get_pod_id.assert_called_once() - mock_python_over_ssh.assert_called_with("prompted_pod_id", temp_file.name) diff --git a/tests/test_cli/test_cli_groups/test_exec_functions.py b/tests/test_cli/test_cli_groups/test_exec_functions.py deleted file mode 100644 index 511ebe30e..000000000 --- a/tests/test_cli/test_cli_groups/test_exec_functions.py +++ /dev/null @@ -1,29 +0,0 @@ -""" Tests for CLI group `exec functions` """ - -import unittest -from unittest.mock import MagicMock, patch - -from runpod.cli.groups.exec import functions - - -class TestExecFunctions(unittest.TestCase): - """Tests for CLI group `exec functions`""" - - @patch("runpod.cli.groups.exec.functions.ssh_cmd.SSHConnection") - def test_python_over_ssh(self, mock_ssh_connection): - """ - Test `python_over_ssh` - """ - mock_ssh = MagicMock() - mock_ssh_connection.return_value = mock_ssh - - pod_id = "pod_id" - file_name = "file_name" - - functions.python_over_ssh(pod_id, file_name) - - self.assertTrue(functions.python_over_ssh) - mock_ssh_connection.assert_called_once_with(pod_id) - mock_ssh.put_file.assert_called_once_with(file_name, f"/root/{file_name}") - mock_ssh.run_commands.assert_called_once_with([f"python3.10 /root/{file_name}"]) - mock_ssh.close.assert_called_once_with() diff --git a/tests/test_cli/test_cli_groups/test_exec_helpers.py b/tests/test_cli/test_cli_groups/test_exec_helpers.py deleted file mode 100644 index 248a684c1..000000000 --- a/tests/test_cli/test_cli_groups/test_exec_helpers.py +++ /dev/null @@ -1,74 +0,0 @@ -""" Unit tests for the runpod.cli.groups.exec.helpers module. """ - -import unittest -from unittest.mock import mock_open, patch - -from runpod.cli.groups.exec.helpers import POD_ID_FILE, get_session_pod - - -class TestGetSessionPod(unittest.TestCase): - """Unit tests for get_session_pod""" - - def setUp(self): - self.mocked_pod_id = "sample_pod_id" - - @patch("os.path.exists") - @patch("runpod.cli.groups.exec.helpers.get_pod") - def test_existing_pod_id_file_and_valid_pod_id(self, mock_get_pod, mock_exists): - """Test get_session_pod when the pod_id file exists and the pod_id is valid""" - mock_exists.return_value = True - mock_get_pod.return_value = True - with patch("builtins.open", mock_open(read_data=self.mocked_pod_id)): - result = get_session_pod() - self.assertEqual(result, self.mocked_pod_id) - - @patch("os.path.exists") - @patch("runpod.cli.groups.exec.helpers.get_pod") - def test_existing_pod_id_file_and_invalid_pod_id(self, mock_get_pod, mock_exists): - """Test get_session_pod when the pod_id file exists and the pod_id is invalid""" - mock_exists.return_value = True - mock_get_pod.return_value = None - with patch("builtins.open", mock_open(read_data="invalid_pod_id")): - with patch("click.prompt", return_value=self.mocked_pod_id): - result = get_session_pod() - self.assertEqual(result, self.mocked_pod_id) - - @patch("os.path.exists") - @patch("runpod.cli.groups.exec.helpers.get_pod") - def test_no_pod_id_file(self, mock_get_pod, mock_exists): - """Test get_session_pod when the pod_id file doesn't exist""" - mock_exists.return_value = False - mock_get_pod.return_value = None - with patch("click.prompt", return_value=self.mocked_pod_id): - result = get_session_pod() - self.assertEqual(result, self.mocked_pod_id) - - @patch("os.path.exists") - @patch("runpod.cli.groups.exec.helpers.get_pod") - def test_pod_id_file_written_to_when_not_existing(self, mock_get_pod, mock_exists): - """Test get_session_pod when the pod_id file doesn't exist""" - mock_exists.return_value = False - mock_get_pod.return_value = None - mocked = mock_open() - with patch("builtins.open", mocked): - with patch("click.prompt", return_value=self.mocked_pod_id): - get_session_pod() - mocked.assert_called_once_with(POD_ID_FILE, "w", encoding="UTF-8") - handle = mocked() - handle.write.assert_called_once_with(self.mocked_pod_id) - - @patch("os.path.exists") - @patch("runpod.cli.groups.exec.helpers.get_pod") - def test_pod_id_file_written_to_when_invalid_pod_id_in_file( - self, mock_get_pod, mock_exists - ): - """Test get_session_pod when the pod_id file exists and the pod_id is invalid""" - mock_exists.return_value = True - mock_get_pod.return_value = None - mocked = mock_open(read_data="invalid_pod_id") - with patch("builtins.open", mocked): - with patch("click.prompt", return_value=self.mocked_pod_id): - get_session_pod() - mocked.assert_called_with(POD_ID_FILE, "w", encoding="UTF-8") - handle = mocked() - handle.write.assert_called_with(self.mocked_pod_id) From 08d4ab920b06fdac3d38df88b3e3e60249aefa98 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:53:46 -0700 Subject: [PATCH 072/120] refactor: standardize cli verbs to add/list/get/delete --- docs/cli/references/command_line_interface.md | 39 +++++++------------ docs/cli/start_here.md | 20 ++++------ runpod/apps/registry.py | 2 +- runpod/apps/secret.py | 2 +- runpod/cli/groups/pod/commands.py | 10 ++--- runpod/cli/groups/ssh/commands.py | 8 ++-- runpod/rp_cli/main.py | 20 +++++----- .../test_cli_groups/test_pod_commands.py | 2 +- 8 files changed, 43 insertions(+), 60 deletions(-) diff --git a/docs/cli/references/command_line_interface.md b/docs/cli/references/command_line_interface.md index d72567acc..80ca0cba1 100644 --- a/docs/cli/references/command_line_interface.md +++ b/docs/cli/references/command_line_interface.md @@ -4,42 +4,29 @@ Note: This CLI is not the same as runpodctl and provides a different set of feat ```bash # Auth -runpod config +rp login -runpod ssh list-keys -runpod ssh add-key +# SSH keys (used by rp pod connect) +rp ssh list +rp ssh add -runpod pod list -runpod pod create -runpod pod connect - -runpod exec python file.py +# Pods +rp pod list +rp pod create +rp pod connect ``` ## Overview ```bash -runpod --help -``` - -### Configure - -```bash -$ runpod config -Profile [default]: -Runpod API Key [None]: YOUR_RUNPOD_API_KEY +rp --help ``` -### Launch Pod +### Authenticate ```bash -runpod launch --help -runpod launch pod --template-file template.yaml +rp login # browser approval +rp login --api-key YOUR_KEY # store a key directly ``` -### Launch Endpoint - -```bash -runpod launch endpoint --help -runpod launch endpoint --template-file template.yaml -``` +Credentials are stored in `~/.runpod/config.toml`. diff --git a/docs/cli/start_here.md b/docs/cli/start_here.md index 5ee16c91f..85ccb7ef2 100644 --- a/docs/cli/start_here.md +++ b/docs/cli/start_here.md @@ -1,23 +1,19 @@ -# [BETA] | Runpod Python CLI Reference +# Runpod CLI Reference Note: This CLI is not the same as runpodctl and provides a different set of features. ## Getting Started -![runpod --help](demos/help.gif) +![rp --help](demos/help.gif) -### Configure +### Authenticate -![runpod config](demos/config.gif) +Store your Runpod API key by running `rp login` (browser approval) or `rp login --api-key YOUR_KEY`. Credentials are stored in `~/.runpod/config.toml`. -Store your Runpod API key by running `runpod config`. Optionally you can also call the command with your API key `runpod config YOUR_API_KEY` or include the `--profile` to save multiple keys (stored under "default" profile is not specified) Credentials are stored in `~/.runpod/config.toml`. +### SSH keys -![runpod ssh add-key](demos/ssh.gif) +Add an SSH key to your account by running `rp ssh add`. To use an existing key pass `--key` or `--key-file`. Keys are stored in `~/.runpod/ssh/`. If no key is specified a new one is generated and stored. Pods trust these keys, so `rp pod connect ` works once a key is added. -Add a SSH key to you account by running `runpod ssh add-key`. To specify and existing key pass in `--key` or `--key-file` to use a file. Keys are stored in `~/.runpod/ssh/`. If no key is specified a new one will be generated and stored. +## Apps -## Runpod Project - -A "project" is the start of a serverless worker. To get started call `runpod project new`, you will be asked a few questions about the project you are creating, a project folder will be created. You can now navigate into your repo and run `runpod project start`. - -Once you are finished developing you can run `runpod project deploy` to deploy your project and create an endpoint. +An app is a collection of Python functions that run on Runpod. Scaffold one with `rp init`, iterate on it live with `rp dev main.py`, and ship it with `rp deploy`. See the [README](../../README.md) for the full workflow. diff --git a/runpod/apps/registry.py b/runpod/apps/registry.py index 336fa283c..cb054e1da 100644 --- a/runpod/apps/registry.py +++ b/runpod/apps/registry.py @@ -5,7 +5,7 @@ @app.queue(image="ghcr.io/me/private:latest", registry_auth="my-ghcr") def infer(): ... -create and manage them with `rp registry add/list/rm`. resolution +create and manage them with `rp registry add/list/delete`. resolution (name -> containerRegistryAuthId) happens when the resource provisions. """ diff --git a/runpod/apps/secret.py b/runpod/apps/secret.py index 0c6399955..d1742f968 100644 --- a/runpod/apps/secret.py +++ b/runpod/apps/secret.py @@ -8,7 +8,7 @@ def download(): ... the worker sees a plain HF_TOKEN env var with the decrypted value. -create and manage secrets with `rp secret add/list/rm`. +create and manage secrets with `rp secret add/list/delete`. """ import re diff --git a/runpod/cli/groups/pod/commands.py b/runpod/cli/groups/pod/commands.py index e6f37c81c..62aa96db9 100644 --- a/runpod/cli/groups/pod/commands.py +++ b/runpod/cli/groups/pod/commands.py @@ -92,8 +92,8 @@ def sync_pods(source_pod_id, dest_pod_id, source_workspace, dest_workspace): 1. SSH Key Setup: • You must have an SSH key configured in your Runpod account - • If you don't have one, create it with: runpod ssh add-key - • List your keys with: runpod ssh list-keys + • If you don't have one, create it with: rp ssh add + • List your keys with: rp ssh list 2. Pod Configuration: • Both pods must have SSH access enabled @@ -130,10 +130,10 @@ def sync_pods(source_pod_id, dest_pod_id, source_workspace, dest_workspace): click.echo("❌ No SSH keys found in your Runpod account!") click.echo("") click.echo("🔑 To create an SSH key, run:") - click.echo(" runpod ssh add-key") + click.echo(" rp ssh add") click.echo("") click.echo("📖 For more help, see:") - click.echo(" runpod ssh add-key --help") + click.echo(" rp ssh add --help") return else: click.echo(f"✅ Found {len(user_keys)} SSH key(s) in your account") @@ -234,7 +234,7 @@ def sync_pods(source_pod_id, dest_pod_id, source_workspace, dest_workspace): click.echo("") click.echo("🔧 Troubleshooting tips:") click.echo("• Ensure both pods have SSH access enabled") - click.echo("• Check that your SSH key is added to your Runpod account: runpod ssh list-keys") + click.echo("• Check that your SSH key is added to your Runpod account: rp ssh list") click.echo("• For running pods, you may need to add PUBLIC_KEY env var and restart") click.echo("• Verify the source and destination paths exist") finally: diff --git a/runpod/cli/groups/ssh/commands.py b/runpod/cli/groups/ssh/commands.py index bf0d19abf..fe62e1a2b 100644 --- a/runpod/cli/groups/ssh/commands.py +++ b/runpod/cli/groups/ssh/commands.py @@ -8,12 +8,12 @@ from .functions import generate_ssh_key_pair, get_user_pub_keys -@click.group("ssh", help="Manage and configure SSH keys for secure access to pods.") +@click.group("ssh", help="Manage the SSH keys pods trust (connect with `rp pod connect`).") def ssh_cli(): - """Manage and configure SSH keys.""" + """Manage account SSH keys.""" -@ssh_cli.command("list-keys") +@ssh_cli.command("list") def list_keys(): """ Lists the SSH keys for the current user. @@ -25,7 +25,7 @@ def list_keys(): click.echo(table) -@ssh_cli.command("add-key") +@ssh_cli.command("add") @click.option("--key", default=None, help="The public key to add.") @click.option( "--key-file", default=None, help="The file containing the public key to add." diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 8d7f19cca..13ee4bbfc 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -700,15 +700,15 @@ def registry_list(): ui.console.print() -@registry.command(name="rm") +@registry.command(name="delete") @click.argument("name") @click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") -def registry_rm(name, yes): +def registry_delete(name, yes): """Delete a registry credential by name.""" from runpod.apps.api import AppsApiClient from runpod.rp_cli import console as ui - async def _rm(): + async def _delete(): client = AppsApiClient() creds = await client.list_registry_auths() match = next((c for c in creds if c["name"] == name), None) @@ -719,7 +719,7 @@ async def _rm(): if not yes: click.confirm(f"delete registry credential '{name}'?", abort=True) try: - asyncio.run(_rm()) + asyncio.run(_delete()) except click.ClickException: raise except Exception as exc: # noqa: BLE001 - surface engine errors cleanly @@ -784,15 +784,15 @@ def secret_list(): ui.console.print() -@secret.command(name="rm") +@secret.command(name="delete") @click.argument("name") @click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") -def secret_rm(name, yes): +def secret_delete(name, yes): """Delete a secret by name.""" from runpod.apps.api import AppsApiClient from runpod.rp_cli import console as ui - async def _rm(): + async def _delete(): client = AppsApiClient() secrets = await client.list_secrets() match = next((s for s in secrets if s["name"] == name), None) @@ -803,7 +803,7 @@ async def _rm(): if not yes: click.confirm(f"delete secret '{name}'?", abort=True) try: - asyncio.run(_rm()) + asyncio.run(_delete()) except click.ClickException: raise except Exception as exc: # noqa: BLE001 - surface engine errors cleanly @@ -891,10 +891,10 @@ def env_get(env_name, app_name): ui.console.print() -@env.command(name="create") +@env.command(name="add") @click.argument("env_name") @click.option("--app", "-a", "app_name", default=None, help="App name.") -def env_create(env_name, app_name): +def env_add(env_name, app_name): """Create a new environment in an app.""" from runpod.apps.api import AppsApiClient from runpod.apps.manage import get_app diff --git a/tests/test_cli/test_cli_groups/test_pod_commands.py b/tests/test_cli/test_cli_groups/test_pod_commands.py index 1eefbe791..2ce00b2b8 100644 --- a/tests/test_cli/test_cli_groups/test_pod_commands.py +++ b/tests/test_cli/test_cli_groups/test_pod_commands.py @@ -184,7 +184,7 @@ def test_sync_pods_no_ssh_keys(self, mock_ssh_connection, mock_echo, mock_uuid): # Verify error message was shown mock_echo.assert_any_call("❌ No SSH keys found in your Runpod account!") mock_echo.assert_any_call("🔑 To create an SSH key, run:") - mock_echo.assert_any_call(" runpod ssh add-key") + mock_echo.assert_any_call(" rp ssh add") @patch("runpod.cli.groups.pod.commands.uuid.uuid4") @patch("runpod.cli.groups.pod.commands.click.echo") From 5372f90647f3896bc3d7d9d8c30255a2b8de186f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:56:18 -0700 Subject: [PATCH 073/120] feat: rp ssh POD_ID opens a pod terminal --- docs/cli/references/command_line_interface.md | 9 ++-- docs/cli/start_here.md | 6 ++- runpod/cli/groups/pod/commands.py | 4 +- runpod/cli/groups/ssh/commands.py | 35 +++++++++++++++- .../test_cli_groups/test_ssh_dispatch.py | 41 +++++++++++++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 tests/test_cli/test_cli_groups/test_ssh_dispatch.py diff --git a/docs/cli/references/command_line_interface.md b/docs/cli/references/command_line_interface.md index 80ca0cba1..e3b5232e7 100644 --- a/docs/cli/references/command_line_interface.md +++ b/docs/cli/references/command_line_interface.md @@ -6,14 +6,15 @@ Note: This CLI is not the same as runpodctl and provides a different set of feat # Auth rp login -# SSH keys (used by rp pod connect) -rp ssh list -rp ssh add +# SSH +rp ssh add # add a key to your account +rp ssh list # list account keys +rp ssh POD_ID # open a terminal on a pod # Pods rp pod list rp pod create -rp pod connect +rp pod connect POD_ID ``` ## Overview diff --git a/docs/cli/start_here.md b/docs/cli/start_here.md index 85ccb7ef2..99902938b 100644 --- a/docs/cli/start_here.md +++ b/docs/cli/start_here.md @@ -10,9 +10,11 @@ Note: This CLI is not the same as runpodctl and provides a different set of feat Store your Runpod API key by running `rp login` (browser approval) or `rp login --api-key YOUR_KEY`. Credentials are stored in `~/.runpod/config.toml`. -### SSH keys +### SSH -Add an SSH key to your account by running `rp ssh add`. To use an existing key pass `--key` or `--key-file`. Keys are stored in `~/.runpod/ssh/`. If no key is specified a new one is generated and stored. Pods trust these keys, so `rp pod connect ` works once a key is added. +Add an SSH key to your account by running `rp ssh add`. To use an existing key pass `--key` or `--key-file`. Keys are stored in `~/.runpod/ssh/`. If no key is specified a new one is generated and stored. + +Once a key is added, open a terminal on any pod with `rp ssh ` (or the equivalent `rp pod connect `). ## Apps diff --git a/runpod/cli/groups/pod/commands.py b/runpod/cli/groups/pod/commands.py index 62aa96db9..0c88ff84b 100644 --- a/runpod/cli/groups/pod/commands.py +++ b/runpod/cli/groups/pod/commands.py @@ -69,7 +69,9 @@ def create_new_pod( @click.argument("pod_id") def connect_to_pod(pod_id): """ - Connects to a pod. + Open an interactive terminal on a pod over SSH. + + Requires an SSH key on your account (rp ssh add). """ click.echo(f"Connecting to pod {pod_id}...") ssh = ssh_cmd.SSHConnection(pod_id) diff --git a/runpod/cli/groups/ssh/commands.py b/runpod/cli/groups/ssh/commands.py index fe62e1a2b..6041b084f 100644 --- a/runpod/cli/groups/ssh/commands.py +++ b/runpod/cli/groups/ssh/commands.py @@ -8,9 +8,40 @@ from .functions import generate_ssh_key_pair, get_user_pub_keys -@click.group("ssh", help="Manage the SSH keys pods trust (connect with `rp pod connect`).") +class SSHGroup(click.Group): + """dispatches unknown first arguments to connect. + + `rp ssh ` opens a terminal on the pod, exactly like plain + `ssh `; named subcommands (add, list, connect) still resolve + normally. + """ + + def resolve_command(self, ctx, args): + try: + return super().resolve_command(ctx, args) + except click.UsageError: + return "connect", self.commands["connect"], args + + +@click.group( + "ssh", + cls=SSHGroup, + help="SSH into pods and manage the keys they trust.", + invoke_without_command=False, +) def ssh_cli(): - """Manage account SSH keys.""" + """SSH into pods and manage account SSH keys.""" + + +@ssh_cli.command("connect") +@click.argument("pod_id") +def connect(pod_id): + """Open an interactive terminal on a pod (also: rp ssh POD_ID).""" + from runpod.cli.utils import ssh_cmd + + click.echo(f"Connecting to pod {pod_id}...") + ssh = ssh_cmd.SSHConnection(pod_id) + ssh.launch_terminal() @ssh_cli.command("list") diff --git a/tests/test_cli/test_cli_groups/test_ssh_dispatch.py b/tests/test_cli/test_cli_groups/test_ssh_dispatch.py new file mode 100644 index 000000000..fa74cc130 --- /dev/null +++ b/tests/test_cli/test_cli_groups/test_ssh_dispatch.py @@ -0,0 +1,41 @@ +"""rp ssh: pod-id dispatch and subcommand resolution.""" + +from unittest.mock import MagicMock, patch + +from click.testing import CliRunner + +from runpod.rp_cli.main import cli + + +class TestSSHDispatch: + def test_pod_id_routes_to_connect(self): + runner = CliRunner() + with patch( + "runpod.cli.utils.ssh_cmd.SSHConnection" + ) as connection: + connection.return_value = MagicMock() + result = runner.invoke(cli, ["ssh", "pod-abc123"]) + assert "Connecting to pod pod-abc123" in result.output + connection.assert_called_once_with("pod-abc123") + + def test_connect_subcommand(self): + runner = CliRunner() + with patch( + "runpod.cli.utils.ssh_cmd.SSHConnection" + ) as connection: + connection.return_value = MagicMock() + result = runner.invoke(cli, ["ssh", "connect", "pod-abc123"]) + assert "Connecting to pod pod-abc123" in result.output + + def test_named_subcommands_still_resolve(self): + runner = CliRunner() + with patch( + "runpod.cli.groups.ssh.commands.get_user_pub_keys", return_value=[] + ): + result = runner.invoke(cli, ["ssh", "list"]) + assert result.exit_code == 0 + + def test_bare_ssh_shows_help(self): + runner = CliRunner() + result = runner.invoke(cli, ["ssh"]) + assert "Commands:" in result.output From b3c5fd7f8286dfee16610e13ab7852d52238800e Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:58:56 -0700 Subject: [PATCH 074/120] fix: clean errors for unknown pods and hide the ssh connect alias --- runpod/cli/groups/pod/commands.py | 5 ++++- runpod/cli/groups/ssh/commands.py | 11 +++++++---- runpod/cli/utils/rp_info.py | 2 ++ tests/test_cli/test_cli_utils/test_info.py | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/runpod/cli/groups/pod/commands.py b/runpod/cli/groups/pod/commands.py index 0c88ff84b..805744ef5 100644 --- a/runpod/cli/groups/pod/commands.py +++ b/runpod/cli/groups/pod/commands.py @@ -74,7 +74,10 @@ def connect_to_pod(pod_id): Requires an SSH key on your account (rp ssh add). """ click.echo(f"Connecting to pod {pod_id}...") - ssh = ssh_cmd.SSHConnection(pod_id) + try: + ssh = ssh_cmd.SSHConnection(pod_id) + except (ValueError, TimeoutError) as exc: + raise click.ClickException(str(exc)) from exc ssh.launch_terminal() diff --git a/runpod/cli/groups/ssh/commands.py b/runpod/cli/groups/ssh/commands.py index 6041b084f..1c6e43764 100644 --- a/runpod/cli/groups/ssh/commands.py +++ b/runpod/cli/groups/ssh/commands.py @@ -26,21 +26,24 @@ def resolve_command(self, ctx, args): @click.group( "ssh", cls=SSHGroup, - help="SSH into pods and manage the keys they trust.", + help="SSH into a pod (rp ssh POD_ID) and manage the keys pods trust.", invoke_without_command=False, ) def ssh_cli(): """SSH into pods and manage account SSH keys.""" -@ssh_cli.command("connect") +@ssh_cli.command("connect", hidden=True) @click.argument("pod_id") def connect(pod_id): - """Open an interactive terminal on a pod (also: rp ssh POD_ID).""" + """Open an interactive terminal on a pod.""" from runpod.cli.utils import ssh_cmd click.echo(f"Connecting to pod {pod_id}...") - ssh = ssh_cmd.SSHConnection(pod_id) + try: + ssh = ssh_cmd.SSHConnection(pod_id) + except (ValueError, TimeoutError) as exc: + raise click.ClickException(str(exc)) from exc ssh.launch_terminal() diff --git a/runpod/cli/utils/rp_info.py b/runpod/cli/utils/rp_info.py index 545f79166..6464bfb6d 100644 --- a/runpod/cli/utils/rp_info.py +++ b/runpod/cli/utils/rp_info.py @@ -18,6 +18,8 @@ def get_pod_ssh_ip_port(pod_id, timeout=300): while time.time() - start_time < timeout and (pod_ip is None or pod_port is None): pod = get_pod(pod_id) + if not pod: + raise ValueError(f"pod '{pod_id}' not found") desired_status = pod.get("desiredStatus", None) runtime = pod.get("runtime", None) diff --git a/tests/test_cli/test_cli_utils/test_info.py b/tests/test_cli/test_cli_utils/test_info.py index 8f2d4ecc0..464c96a86 100644 --- a/tests/test_cli/test_cli_utils/test_info.py +++ b/tests/test_cli/test_cli_utils/test_info.py @@ -41,5 +41,5 @@ def test_get_pod_ssh_ip_port_timeout(self): mock_get_pod.return_value = {} - with pytest.raises(TimeoutError): + with pytest.raises(ValueError, match="not found"): get_pod_ssh_ip_port("pod_id", timeout=0.1) From 90ce2717b0f497f96147c87b225fa417ff6b601d Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:08:55 -0700 Subject: [PATCH 075/120] fix: reject platform-cached models on task resources --- runpod/apps/app.py | 5 +++-- runpod/apps/spec.py | 6 ++++++ tests/test_apps/test_model.py | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 551cb0dbd..112946bfd 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -162,7 +162,6 @@ def task( env: Optional[Dict[str, Any]] = None, image: Optional[str] = None, registry_auth: Optional[str] = None, - model: Optional[Any] = None, datacenter: Optional[Union[str, List[str]]] = None, min_cuda_version: Optional[str] = None, accelerate_downloads: bool = True, @@ -173,6 +172,9 @@ def task( tasks have no standing infrastructure: `.remote()` provisions a pod, runs the body, returns the result, and terminates the pod. they never require `rp deploy` (except to register a schedule). + + platform-cached models (`model=`) are only available on queue + and api resources; tasks download weights themselves. """ def decorator(fn: Callable) -> FunctionHandle: @@ -188,7 +190,6 @@ def decorator(fn: Callable) -> FunctionHandle: env=env, image=image, registry_auth=registry_auth, - model=model, datacenter=_datacenter_list(datacenter), min_cuda_version=normalize_cuda_version(min_cuda_version), accelerate_downloads=accelerate_downloads, diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 7e5cdb485..e87e9b036 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -205,6 +205,12 @@ def __post_init__(self) -> None: f"resource '{self.name}': min_cuda_version has no effect " f"on cpu resources" ) + if self.model is not None and self.kind is ResourceKind.TASK: + raise InvalidResourceError( + f"resource '{self.name}': platform-cached models are only " + f"available on queue and api resources; tasks download " + f"weights themselves" + ) @property def is_cpu(self) -> bool: diff --git a/tests/test_apps/test_model.py b/tests/test_apps/test_model.py index d21496a6f..5a762d4e3 100644 --- a/tests/test_apps/test_model.py +++ b/tests/test_apps/test_model.py @@ -66,3 +66,23 @@ def chat(prompt: str): chat.spec.to_manifest()["model"] == "meta-llama/Llama-3.1-8B-Instruct" ) + + def test_task_decorator_rejects_model(self): + from runpod.apps.app import App + + app = App("modeltest") + + with pytest.raises(TypeError, match="model"): + + @app.task(name="train", gpu="4090", model=Model("org/name")) + def train(): + pass + + def test_task_spec_rejects_model(self): + from runpod.apps.errors import InvalidResourceError + from runpod.apps.spec import ResourceKind, ResourceSpec + + with pytest.raises(InvalidResourceError, match="queue and api"): + ResourceSpec( + kind=ResourceKind.TASK, name="train", model=Model("org/name") + ) From 6c85a6c42df44e49d84ba0309dea5f11494d33c8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:20:25 -0700 Subject: [PATCH 076/120] fix: ignore fastapi testclient deprecation warning in pytest --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index 1c2c135e2..3e3bb1d68 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,9 @@ filterwarnings = # only warn. Don't fail the suite on them until the deps update. ignore:'asyncio\.get_event_loop_policy' is deprecated:DeprecationWarning ignore:'asyncio\.iscoroutinefunction' is deprecated:DeprecationWarning + # fastapi's bundled TestClient imports starlette.testclient, which warns + # about httpx until fastapi migrates to httpx2 + ignore:Using `httpx` with `starlette.testclient` is deprecated.* python_files = tests.py test_*.py *_test.py norecursedirs = venv *.egg-info .git build tests/e2e asyncio_mode = auto From 08017fb35d3a69ab5cec7b89cc65222cb4b3e8f5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:39:09 -0700 Subject: [PATCH 077/120] test: cover the apps api client, targets, monitor, logs, and queue worker --- tests/test_apps/test_api_client.py | 375 +++++++++++++++++++++ tests/test_apps/test_logs.py | 140 ++++++++ tests/test_apps/test_monitor.py | 131 ++++++- tests/test_apps/test_queue_worker.py | 205 +++++++++++ tests/test_apps/test_targets.py | 315 +++++++++++++++++ tests/test_cli/test_management_commands.py | 357 ++++++++++++++++++++ 6 files changed, 1522 insertions(+), 1 deletion(-) create mode 100644 tests/test_apps/test_api_client.py create mode 100644 tests/test_apps/test_logs.py create mode 100644 tests/test_apps/test_queue_worker.py create mode 100644 tests/test_apps/test_targets.py create mode 100644 tests/test_cli/test_management_commands.py diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py new file mode 100644 index 000000000..095ad90f5 --- /dev/null +++ b/tests/test_apps/test_api_client.py @@ -0,0 +1,375 @@ +"""unit tests for the apps control-plane client.""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import aiohttp +import pytest + +from runpod.apps.api import AppsApiClient +from runpod.error import QueryError + + +def _respond(data): + return AsyncMock(return_value={"data": data}) + + +def _client_with(data): + client = AppsApiClient(api_key="test-key") + return client, patch( + "runpod.apps.api.run_graphql_query_async", _respond(data) + ) + + +class TestExecuteRetry: + async def test_returns_data(self): + client, patcher = _client_with({"ok": 1}) + with patcher: + assert await client._execute("query {}") == {"ok": 1} + + async def test_retries_transport_errors(self): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock( + side_effect=[ + aiohttp.ClientError("reset"), + OSError("dns"), + {"data": {"ok": 1}}, + ] + ) + with ( + patch("runpod.apps.api.run_graphql_query_async", transport), + patch("asyncio.sleep", AsyncMock()), + ): + assert await client._execute("query {}") == {"ok": 1} + assert transport.await_count == 3 + + async def test_exhausted_retries_raise(self): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock(side_effect=aiohttp.ClientError("down")) + with ( + patch("runpod.apps.api.run_graphql_query_async", transport), + patch("asyncio.sleep", AsyncMock()), + ): + with pytest.raises(aiohttp.ClientError): + await client._execute("query {}") + assert transport.await_count == 4 + + async def test_graphql_errors_propagate_immediately(self): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock(side_effect=QueryError("bad query", "query {}")) + with patch("runpod.apps.api.run_graphql_query_async", transport): + with pytest.raises(QueryError): + await client._execute("query {}") + assert transport.await_count == 1 + + +class TestEndpoints: + async def test_save_endpoint(self): + client, patcher = _client_with( + {"saveEndpoint": {"id": "ep1", "name": "chat"}} + ) + with patcher: + result = await client.save_endpoint({"name": "chat"}) + assert result["id"] == "ep1" + + async def test_delete_endpoint(self): + client, patcher = _client_with({"deleteEndpoint": True}) + with patcher: + assert await client.delete_endpoint("ep1") is True + + async def test_list_my_endpoints(self): + client, patcher = _client_with( + {"myself": {"endpoints": [{"id": "ep1"}]}} + ) + with patcher: + assert await client.list_my_endpoints() == [{"id": "ep1"}] + + +class TestTaskPods: + async def test_deploy_gpu_pod(self): + client = AppsApiClient(api_key="test-key") + transport = _respond( + {"podFindAndDeployOnDemand": {"id": "pod1", "desiredStatus": "RUNNING"}} + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + result = await client.deploy_task_pod( + {"gpuTypeIdList": ["NVIDIA GeForce RTX 4090"]}, is_cpu=False + ) + assert result["id"] == "pod1" + + async def test_deploy_cpu_pod_converts_instance_ids(self): + client = AppsApiClient(api_key="test-key") + transport = _respond({"deployCpuPod": {"id": "pod2"}}) + with patch("runpod.apps.api.run_graphql_query_async", transport): + result = await client.deploy_task_pod( + {"instanceIds": ["cpu3c-2-4", "cpu3g-2-8"]}, is_cpu=True + ) + assert result["id"] == "pod2" + sent = transport.call_args[1]["variables"]["input"] + assert sent["instanceId"] == "cpu3c-2-4" + assert "instanceIds" not in sent + + async def test_terminate_pod(self): + client = AppsApiClient(api_key="test-key") + transport = _respond({"podTerminate": None}) + with patch("runpod.apps.api.run_graphql_query_async", transport): + await client.terminate_pod("pod1") + sent = transport.call_args[1]["variables"]["input"] + assert sent == {"podId": "pod1"} + + +class TestAppLifecycle: + async def test_get_app_by_name(self): + client, patcher = _client_with( + {"flashAppByName": {"id": "app1", "name": "demo"}} + ) + with patcher: + app = await client.get_app_by_name("demo") + assert app["id"] == "app1" + + async def test_create_app(self): + client, patcher = _client_with( + {"createFlashApp": {"id": "app1", "name": "demo"}} + ) + with patcher: + assert (await client.create_app("demo"))["id"] == "app1" + + async def test_create_environment(self): + client, patcher = _client_with( + {"createFlashEnvironment": {"id": "env1", "name": "prod"}} + ) + with patcher: + result = await client.create_environment("app1", "prod") + assert result["id"] == "env1" + + async def test_list_apps(self): + client, patcher = _client_with( + {"myself": {"flashApps": [{"id": "app1"}]}} + ) + with patcher: + assert await client.list_apps() == [{"id": "app1"}] + + async def test_list_apps_empty(self): + client, patcher = _client_with({"myself": {"flashApps": None}}) + with patcher: + assert await client.list_apps() == [] + + async def test_delete_app(self): + client, patcher = _client_with({"deleteFlashApp": True}) + with patcher: + assert await client.delete_app("app1") is True + + async def test_delete_environment(self): + client, patcher = _client_with({"deleteFlashEnvironment": True}) + with patcher: + assert await client.delete_environment("env1") is True + + async def test_get_environment_by_name(self): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock( + side_effect=[ + {"data": {"flashAppByName": {"id": "app1"}}}, + {"data": {"flashEnvironmentByName": {"id": "env1"}}}, + ] + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + env = await client.get_environment_by_name("demo", "prod") + assert env["id"] == "env1" + + async def test_get_environment_missing_app(self): + client, patcher = _client_with({"flashAppByName": None}) + with patcher: + assert await client.get_environment_by_name("demo", "prod") is None + + async def test_get_environment_not_found(self): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock( + side_effect=[ + {"data": {"flashAppByName": {"id": "app1"}}}, + QueryError("environment not found", "query {}"), + ] + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + assert await client.get_environment_by_name("demo", "prod") is None + + +class TestStock: + async def test_gpu_stock_status(self): + client, patcher = _client_with( + {"gpuTypes": [{"lowestPrice": {"stockStatus": "High"}}]} + ) + with patcher: + status = await client.gpu_stock_status( + "NVIDIA GeForce RTX 4090", "US-KS-2" + ) + assert status == "High" + + async def test_gpu_stock_no_data(self): + client, patcher = _client_with({"gpuTypes": []}) + with patcher: + assert ( + await client.gpu_stock_status("X", "US-KS-2") is None + ) + + async def test_gpu_stock_pods_flag(self): + client = AppsApiClient(api_key="test-key") + transport = _respond({"gpuTypes": []}) + with patch("runpod.apps.api.run_graphql_query_async", transport): + await client.gpu_stock_status("X", "US-KS-2", pods=True) + sent = transport.call_args[1]["variables"]["lowestPriceInput"] + assert sent["includeAiApi"] is False + + async def test_cpu_stock_status(self): + client = AppsApiClient(api_key="test-key") + transport = _respond( + {"cpuFlavors": [{"specifics": {"stockStatus": "Low"}}]} + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + status = await client.cpu_stock_status("cpu3c-2-4", "US-KS-2") + assert status == "Low" + sent = transport.call_args[1]["variables"] + assert sent["cpuFlavorInput"] == {"id": "cpu3c"} + + +class TestVolumesRegistrySecrets: + async def test_list_network_volumes(self): + client, patcher = _client_with( + {"myself": {"networkVolumes": [{"id": "v1"}]}} + ) + with patcher: + assert await client.list_network_volumes() == [{"id": "v1"}] + + async def test_create_network_volume(self): + client, patcher = _client_with( + {"createNetworkVolume": {"id": "v1", "name": "data"}} + ) + with patcher: + result = await client.create_network_volume("data", 10, "US-KS-2") + assert result["id"] == "v1" + + async def test_registry_auth_crud(self): + client, patcher = _client_with( + { + "myself": {"containerRegistryCreds": [{"id": "r1"}]}, + "saveRegistryAuth": {"id": "r1", "name": "dh"}, + "deleteRegistryAuth": True, + } + ) + with patcher: + assert await client.list_registry_auths() == [{"id": "r1"}] + assert ( + await client.create_registry_auth("dh", "user", "pass") + )["id"] == "r1" + assert await client.delete_registry_auth("r1") is True + + async def test_secret_crud(self): + client, patcher = _client_with( + { + "myself": {"secrets": [{"id": "s1"}]}, + "secretCreate": {"id": "s1", "name": "tok"}, + "secretDelete": True, + } + ) + with patcher: + assert await client.list_secrets() == [{"id": "s1"}] + assert (await client.create_secret("tok", "v"))["id"] == "s1" + assert await client.delete_secret("s1") is True + + +class TestAuthRequests: + async def test_create_auth_request_is_anonymous(self): + client = AppsApiClient(api_key="test-key") + transport = _respond( + {"createFlashAuthRequest": {"id": "req1", "status": "PENDING"}} + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + result = await client.create_auth_request() + assert result["id"] == "req1" + assert transport.call_args[1]["anonymous"] is True + + async def test_status_poll_is_anonymous(self): + client = AppsApiClient(api_key="test-key") + transport = _respond( + {"flashAuthRequestStatus": {"id": "req1", "status": "APPROVED"}} + ) + with patch("runpod.apps.api.run_graphql_query_async", transport): + result = await client.get_auth_request_status("req1") + assert result["status"] == "APPROVED" + assert transport.call_args[1]["anonymous"] is True + + +class TestArtifacts: + async def test_prepare_artifact_upload(self): + client, patcher = _client_with( + { + "prepareFlashArtifactUpload": { + "uploadUrl": "https://s3", + "objectKey": "k", + } + } + ) + with patcher: + result = await client.prepare_artifact_upload("app1", 123) + assert result["objectKey"] == "k" + + async def test_finalize_artifact_upload(self): + client, patcher = _client_with( + {"finalizeFlashArtifactUpload": {"id": "b1", "manifest": {}}} + ) + with patcher: + result = await client.finalize_artifact_upload("app1", "k", {}) + assert result["id"] == "b1" + + async def test_deploy_build(self): + client, patcher = _client_with( + {"deployBuildToEnvironment": {"id": "env1", "name": "prod"}} + ) + with patcher: + result = await client.deploy_build("env1", "b1") + assert result["id"] == "env1" + + +class TestUploadTarball: + async def test_upload_reports_progress(self, tmp_path): + tar = tmp_path / "app.tar.gz" + tar.write_bytes(b"x" * 2048) + client = AppsApiClient(api_key="test-key") + progress = MagicMock() + + put = AsyncMock() + with patch.object(client, "_put_tarball", put): + await client.upload_tarball("https://s3", str(tar), progress) + put.assert_awaited_once() + + # drain the reader to drive progress callbacks + reader = put.call_args[0][1] + async for _ in reader: + pass + progress.assert_called_with(2048, 2048) + + async def test_upload_retries_then_succeeds(self, tmp_path): + tar = tmp_path / "app.tar.gz" + tar.write_bytes(b"x") + client = AppsApiClient(api_key="test-key") + + put = AsyncMock(side_effect=[OSError("broken pipe"), None]) + with ( + patch.object(client, "_put_tarball", put), + patch("asyncio.sleep", AsyncMock()), + ): + await client.upload_tarball("https://s3", str(tar)) + assert put.await_count == 2 + + async def test_upload_exhausted_raises(self, tmp_path): + tar = tmp_path / "app.tar.gz" + tar.write_bytes(b"x") + client = AppsApiClient(api_key="test-key") + + put = AsyncMock(side_effect=aiohttp.ClientError("reset")) + with ( + patch.object(client, "_put_tarball", put), + patch("asyncio.sleep", AsyncMock()), + ): + with pytest.raises(aiohttp.ClientError): + await client.upload_tarball("https://s3", str(tar)) + assert put.await_count == 4 diff --git a/tests/test_apps/test_logs.py b/tests/test_apps/test_logs.py new file mode 100644 index 000000000..f32b661b7 --- /dev/null +++ b/tests/test_apps/test_logs.py @@ -0,0 +1,140 @@ +"""unit tests for pod log access.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from runpod.apps import logs as pod_logs_mod +from runpod.apps.logs import pod_logs, stream_pod_logs, tail_summary + + +class TestTailSummary: + def test_empty(self): + assert tail_summary({}) == "(no logs available)" + assert tail_summary({"container": [], "system": []}) == ( + "(no logs available)" + ) + + def test_container_only(self): + out = tail_summary({"container": ["a", "b"], "system": []}) + assert "--- container (last 2) ---" in out + assert "a" in out and "b" in out + assert "system" not in out + + def test_tail_limit(self): + entries = [f"line{i}" for i in range(50)] + out = tail_summary({"container": entries}, lines=3) + assert "line49" in out + assert "line46" not in out + assert "(last 3)" in out + + def test_system_before_container(self): + out = tail_summary({"container": ["c"], "system": ["s"]}) + assert out.index("system") < out.index("container") + + +def _mock_session(response): + """a ClientSession whose get() context manager yields `response`.""" + get_cm = MagicMock() + get_cm.__aenter__ = AsyncMock(return_value=response) + get_cm.__aexit__ = AsyncMock(return_value=False) + session = MagicMock() + session.get = MagicMock(return_value=get_cm) + session_cm = MagicMock() + session_cm.__aenter__ = AsyncMock(return_value=session) + session_cm.__aexit__ = AsyncMock(return_value=False) + return session_cm, session + + +class TestPodLogs: + async def test_snapshot_normalizes_nulls(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "test-key") + response = MagicMock() + response.raise_for_status = MagicMock() + response.json = AsyncMock( + return_value={"container": ["hello"], "system": None} + ) + session_cm, session = _mock_session(response) + + with patch("aiohttp.ClientSession", return_value=session_cm): + result = await pod_logs("pod123") + + assert result == {"container": ["hello"], "system": []} + url = session.get.call_args[0][0] + assert "pod123" in url + headers = session.get.call_args[1]["headers"] + assert headers["Authorization"] == "Bearer test-key" + + async def test_log_type_param(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "test-key") + response = MagicMock() + response.raise_for_status = MagicMock() + response.json = AsyncMock(return_value={}) + session_cm, session = _mock_session(response) + + with patch("aiohttp.ClientSession", return_value=session_cm): + await pod_logs("pod123", log_type="system") + + assert session.get.call_args[1]["params"] == {"type": "system"} + + +class _FakeContent: + def __init__(self, lines): + self._lines = list(lines) + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._lines: + raise StopAsyncIteration + return self._lines.pop(0) + + +class TestStreamPodLogs: + async def test_parses_sse_frames(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "test-key") + frames = [ + b": heartbeat\n", + b"data: " + json.dumps( + {"source": "container", "line": "hi", "ts": "t1"} + ).encode() + b"\n", + b"garbage\n", + b"data: not-json\n", + b"data: " + json.dumps( + {"source": "system", "line": "boot", "ts": "t2"} + ).encode() + b"\n", + ] + response = MagicMock() + response.raise_for_status = MagicMock() + response.content = _FakeContent(frames) + session_cm, session = _mock_session(response) + + with patch("aiohttp.ClientSession", return_value=session_cm): + events = [e async for e in stream_pod_logs("pod123")] + + assert events == [ + {"source": "container", "line": "hi", "ts": "t1"}, + {"source": "system", "line": "boot", "ts": "t2"}, + ] + params = session.get.call_args[1]["params"] + assert params["stream"] == "true" + assert params["tail"] == "100" + + async def test_since_param(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "test-key") + response = MagicMock() + response.raise_for_status = MagicMock() + response.content = _FakeContent([]) + session_cm, session = _mock_session(response) + + with patch("aiohttp.ClientSession", return_value=session_cm): + async for _ in stream_pod_logs( + "pod123", since="2026-01-01T00:00:00Z", tail=5 + ): + pass + + params = session.get.call_args[1]["params"] + assert params["since"] == "2026-01-01T00:00:00Z" + assert params["tail"] == "5" diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index 1cdd77f61..c1cb27192 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -1,7 +1,7 @@ """worker monitor: metrics transitions, log attach, event emission.""" import asyncio -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -142,3 +142,132 @@ def test_non_frame_json_passes_through(self): # user code printing json without the sdk frame shape assert _filter_line('{"result": 5}') == '{"result": 5}' + + +class TestPodLogStream: + async def test_follow_emits_and_dedups(self): + from runpod.apps.monitor import PodLogStream + + sink = Sink() + events = [ + {"ts": "t1", "line": "hello"}, + {"ts": "t1", "line": "hello"}, # duplicate frame + {"ts": "t2", "line": ""}, # blank filtered + {"ts": "t3", "line": "world"}, + ] + + calls = {"n": 0} + + async def fake_stream(pod_id, **kwargs): + if calls["n"]: + # second attach: park forever so stop() cancels us + await asyncio.sleep(3600) + calls["n"] += 1 + for event in events: + yield event + + stream = PodLogStream("pod1", "calc", sink) + with patch("runpod.apps.logs.stream_pod_logs", fake_stream): + stream.attach() + await asyncio.sleep(0.05) + await stream.stop() + + logs = [e for e in sink.events if e[0] == "worker_log"] + assert logs == [ + ("worker_log", "calc", "hello"), + ("worker_log", "calc", "world"), + ] + + async def test_stop_snapshots_when_stream_never_attached(self): + from datetime import datetime, timedelta, timezone + + from runpod.apps.monitor import PodLogStream + + sink = Sink() + + async def failing_stream(pod_id, **kwargs): + raise RuntimeError("403") + yield # pragma: no cover + + stream = PodLogStream("pod1", "calc", sink) + after = ( + datetime.now(timezone.utc) + timedelta(seconds=5) + ).isoformat() + snapshot = { + "container": [f"{after} recovered output"], + "system": [], + } + with ( + patch("runpod.apps.logs.stream_pod_logs", failing_stream), + patch( + "runpod.apps.logs.pod_logs", + AsyncMock(return_value=snapshot), + ), + ): + stream.attach() + await asyncio.sleep(0.05) + await stream.stop() + + logs = [e for e in sink.events if e[0] == "worker_log"] + assert logs == [("worker_log", "calc", "recovered output")] + + async def test_snapshot_skips_lines_before_since(self): + from runpod.apps.monitor import PodLogStream + + sink = Sink() + stale = "2000-01-01T00:00:00.000000000Z old line" + snapshot = {"container": [stale], "system": []} + stream = PodLogStream("pod1", "calc", sink) + with patch( + "runpod.apps.logs.pod_logs", + AsyncMock(return_value=snapshot), + ): + await stream._snapshot() + + assert sink.events == [] + + async def test_attach_idempotent(self): + from runpod.apps.monitor import PodLogStream + + async def parked(pod_id, **kwargs): + await asyncio.sleep(3600) + yield # pragma: no cover + + stream = PodLogStream("pod1", "calc", Sink()) + with patch("runpod.apps.logs.stream_pod_logs", parked): + stream.attach() + first = stream._task + stream.attach() + assert stream._task is first + stream._lines_emitted = 1 # skip the snapshot fallback + await stream.stop() + assert stream._task is None + + +class TestMonitorLifecycle: + async def test_start_without_metrics_key_is_noop(self): + monitor = WorkerMonitor("ep1", "calc", Sink()) + await monitor.start() + assert monitor._tasks == [] + await monitor.stop() + + async def test_metrics_polling_reports_counts(self): + import runpod + + sink = Sink() + monitor = WorkerMonitor("ep1", "calc", sink, metrics_key="mk") + payload = {"workers": {"initializing": 1, "ready": 0}} + + async def fake_get(url, headers, timeout): + assert headers["Authorization"] == "Bearer mk" + return payload + + with patch("runpod.apps.targets._get_json", fake_get): + await monitor.start() + await asyncio.sleep(0.05) + await monitor.stop() + + statuses = [e for e in sink.events if e[0] == "worker_status"] + assert statuses + assert statuses[0][2]["initializing"] == 1 + diff --git a/tests/test_apps/test_queue_worker.py b/tests/test_apps/test_queue_worker.py new file mode 100644 index 000000000..1630c2ac1 --- /dev/null +++ b/tests/test_apps/test_queue_worker.py @@ -0,0 +1,205 @@ +"""unit tests for the generic queue worker runtime.""" + +import inspect +import json +from unittest.mock import MagicMock, patch + +import pytest + +from runpod.runtimes.queue import worker + + +class TestModeDetection: + def test_resource_name_flash_env(self, monkeypatch): + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + assert worker._resource_name() == "chat" + + def test_resource_name_runpod_env(self, monkeypatch): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.setenv("RUNPOD_RESOURCE_NAME", "embed") + assert worker._resource_name() == "embed" + + def test_resource_name_empty(self, monkeypatch): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) + assert worker._resource_name() == "" + + def test_is_deployed_requires_name_and_manifest(self, monkeypatch, tmp_path): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) + monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) + assert not worker._is_deployed() + + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + assert not worker._is_deployed() + + (tmp_path / worker.MANIFEST_NAME).write_text("{}") + assert worker._is_deployed() + + +class TestDeployedHandle: + def _write_manifest(self, tmp_path, resources): + (tmp_path / worker.MANIFEST_NAME).write_text( + json.dumps({"resources": resources}) + ) + + def test_load_deployed_handle(self, monkeypatch, tmp_path): + module_src = ( + "import runpod\n" + "app = runpod.App('t')\n" + "@app.queue(name='chat', gpu='4090')\n" + "def chat(prompt: str):\n" + " return {'echo': prompt}\n" + ) + (tmp_path / "user_mod_a.py").write_text(module_src) + self._write_manifest( + tmp_path, [{"name": "chat", "module": "user_mod_a"}] + ) + monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + + handle = worker._load_deployed_handle() + assert handle.spec.name == "chat" + + def test_load_deployed_handle_missing_resource(self, monkeypatch, tmp_path): + self._write_manifest(tmp_path, [{"name": "other", "module": "x"}]) + monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + + with pytest.raises(RuntimeError, match="not in manifest"): + worker._load_deployed_handle() + + def test_load_deployed_handle_missing_handle(self, monkeypatch, tmp_path): + (tmp_path / "user_mod_b.py").write_text("x = 1\n") + self._write_manifest( + tmp_path, [{"name": "chat", "module": "user_mod_b"}] + ) + monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + + with pytest.raises(RuntimeError, match="no @app.queue"): + worker._load_deployed_handle() + + +class TestDeployedHandler: + def _handle(self, fn): + handle = MagicMock() + handle._fn = fn + return handle + + async def test_sync_function(self): + handler = worker._make_deployed_handler( + self._handle(lambda a, b: {"sum": a + b}) + ) + result = await handler({"input": {"a": 1, "b": 2}}) + assert result == {"sum": 3} + + async def test_async_function(self): + async def fn(x): + return x * 2 + + handler = worker._make_deployed_handler(self._handle(fn)) + assert await handler({"input": {"x": 5}}) == 10 + + async def test_empty_marker_stripped(self): + handler = worker._make_deployed_handler( + self._handle(lambda: "ok") + ) + assert await handler({"input": {"__empty": True}}) == "ok" + + async def test_no_input(self): + handler = worker._make_deployed_handler( + self._handle(lambda: "ok") + ) + assert await handler({}) == "ok" + + +class TestInitHook: + def test_no_init(self): + handle = MagicMock(spec=[]) + worker._run_init(handle) # no _init_fn attr: no-op + + def test_sync_init(self): + calls = [] + handle = MagicMock() + handle._init_fn = lambda: calls.append(1) + worker._run_init(handle) + assert calls == [1] + + def test_async_init(self): + calls = [] + + async def init(): + calls.append(1) + + handle = MagicMock() + handle._init_fn = init + worker._run_init(handle) + assert calls == [1] + + +class TestConcurrency: + def test_default(self, monkeypatch): + monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) + assert worker._max_concurrency() == 1 + + def test_env_value(self, monkeypatch): + monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "8") + assert worker._max_concurrency() == 8 + + def test_invalid_value(self, monkeypatch): + monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "banana") + assert worker._max_concurrency() == 1 + + def test_floor_of_one(self, monkeypatch): + monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "0") + assert worker._max_concurrency() == 1 + + def test_worker_config_single(self, monkeypatch): + monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) + config = worker._worker_config(lambda job: job) + assert "concurrency_modifier" not in config + + def test_worker_config_concurrent(self, monkeypatch): + monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "4") + config = worker._worker_config(lambda job: job) + assert config["concurrency_modifier"](1) == 4 + + +class TestMain: + def test_live_mode(self, monkeypatch): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) + with patch("runpod.serverless.start") as start: + worker.main() + config = start.call_args[0][0] + assert config["handler"] is worker._live_handler + + def test_deployed_mode(self, monkeypatch, tmp_path): + module_src = ( + "import runpod\n" + "app = runpod.App('t')\n" + "@app.queue(name='greet', gpu='4090')\n" + "def greet(name: str):\n" + " return f'hi {name}'\n" + ) + (tmp_path / "user_mod_c.py").write_text(module_src) + (tmp_path / worker.MANIFEST_NAME).write_text( + json.dumps({"resources": [{"name": "greet", "module": "user_mod_c"}]}) + ) + monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "greet") + + with patch("runpod.serverless.start") as start: + worker.main() + handler = start.call_args[0][0]["handler"] + assert inspect.iscoroutinefunction(handler) + + def test_live_handler_delegates(self): + with patch( + "runpod.runtimes.task.runner.execute_request", + return_value={"ok": True}, + ) as execute: + result = worker._live_handler({"input": {"foo": 1}}) + assert result == {"ok": True} + execute.assert_called_once_with({"foo": 1}) diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py new file mode 100644 index 000000000..e4b31c5c1 --- /dev/null +++ b/tests/test_apps/test_targets.py @@ -0,0 +1,315 @@ +"""unit tests for invocation targets and their helpers.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import AsyncMock, patch + +import pytest + +from runpod.apps.errors import RemoteExecutionError +from runpod.apps.spec import ResourceKind, ResourceSpec +from runpod.apps.targets import ( + SENTINEL_ID, + LiveTarget, + SentinelTarget, + _api_key, + _headers, + _lb_domain, + _wait_terminal, + args_to_input, + unwrap_job_output, +) + +# http.server's per-request sockets are collected lazily; the unraisable +# checker flags them as ResourceWarnings non-deterministically +pytestmark = pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" +) + + +class TestApiKey: + def test_env_var(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "sk-env") + assert _api_key() == "sk-env" + + def test_module_fallback(self, monkeypatch): + import runpod + + monkeypatch.delenv("RUNPOD_API_KEY", raising=False) + monkeypatch.setattr(runpod, "api_key", "sk-module") + assert _api_key() == "sk-module" + + def test_missing_raises(self, monkeypatch): + import runpod + + monkeypatch.delenv("RUNPOD_API_KEY", raising=False) + monkeypatch.setattr(runpod, "api_key", None) + with pytest.raises(RuntimeError, match="rp login"): + _api_key() + + +class TestUrlHelpers: + def test_lb_domain(self, monkeypatch): + import runpod + + monkeypatch.setattr( + runpod, "endpoint_url_base", "https://api.runpod.ai/v2" + ) + assert _lb_domain() == "api.runpod.ai" + + def test_headers(self, monkeypatch): + monkeypatch.setenv("RUNPOD_API_KEY", "sk-test") + headers = _headers({"X-Extra": "1"}) + assert headers["Authorization"] == "Bearer sk-test" + assert headers["X-Extra"] == "1" + + +class TestArgsToInput: + def test_positional_mapping(self): + def fn(a, b, c=3): + pass + + assert args_to_input(fn, (1, 2), {}) == {"a": 1, "b": 2} + + def test_kwargs_merge(self): + def fn(a, b): + pass + + assert args_to_input(fn, (1,), {"b": 2}) == {"a": 1, "b": 2} + + def test_too_many_positional(self): + def fn(a): + pass + + with pytest.raises(TypeError, match="positional"): + args_to_input(fn, (1, 2), {}) + + def test_empty_marker(self): + def fn(): + pass + + assert args_to_input(fn, (), {}) == {"__empty": True} + + +class TestUnwrapJobOutput: + def test_completed(self): + assert unwrap_job_output( + {"status": "COMPLETED", "output": {"x": 1}} + ) == {"x": 1} + + def test_failed_status(self): + with pytest.raises(RemoteExecutionError, match="boom"): + unwrap_job_output({"status": "FAILED", "error": "boom"}) + + def test_error_in_output(self): + with pytest.raises(RemoteExecutionError, match="oops"): + unwrap_job_output( + {"status": "COMPLETED", "output": {"error": "oops"}} + ) + + def test_missing_output_returns_data(self): + data = {"status": "COMPLETED", "value": 7} + assert unwrap_job_output(data) == data + + +class TestWaitTerminal: + async def test_immediate_terminal(self): + data = {"id": "j1", "status": "COMPLETED", "output": 1} + result = await _wait_terminal("http://x", data, {}, timeout=5) + assert result is data + + async def test_polls_to_completion(self): + polls = [ + {"id": "j1", "status": "IN_PROGRESS"}, + {"id": "j1", "status": "COMPLETED", "output": 2}, + ] + seen = [] + with ( + patch( + "runpod.apps.targets._get_json", + AsyncMock(side_effect=polls), + ), + patch("asyncio.sleep", AsyncMock()), + ): + result = await _wait_terminal( + "http://x", + {"id": "j1", "status": "IN_QUEUE"}, + {}, + timeout=30, + on_status=seen.append, + ) + assert result["status"] == "COMPLETED" + assert len(seen) == 3 + + async def test_no_job_id_returns_data(self): + data = {"status": "IN_QUEUE"} + result = await _wait_terminal("http://x", data, {}, timeout=5) + assert result is data + + async def test_timeout(self): + with ( + patch( + "runpod.apps.targets._get_json", + AsyncMock(return_value={"id": "j1", "status": "IN_PROGRESS"}), + ), + patch("asyncio.sleep", AsyncMock()), + patch("time.monotonic", side_effect=[0, 100, 200]), + ): + with pytest.raises(TimeoutError, match="did not complete"): + await _wait_terminal( + "http://x", + {"id": "j1", "status": "IN_QUEUE"}, + {}, + timeout=10, + ) + + +@pytest.fixture +def local_endpoint(monkeypatch): + """local http server standing in for the serverless data plane.""" + state = {"requests": [], "responses": {}} + + class Handler(BaseHTTPRequestHandler): + def _respond(self): + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + state["requests"].append( + { + "path": self.path, + "method": self.command, + "headers": dict(self.headers), + "body": json.loads(body) if body else None, + } + ) + reply = state["responses"].get( + self.path, {"status": "COMPLETED", "output": {"ok": True}} + ) + payload = json.dumps(reply).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + do_GET = do_POST = do_PUT = _respond + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + import runpod + + monkeypatch.setenv("RUNPOD_API_KEY", "sk-test") + monkeypatch.setattr( + runpod, + "endpoint_url_base", + f"http://127.0.0.1:{server.server_address[1]}", + ) + server.state = state + yield server + server.shutdown() + + +class TestSentinelTarget: + async def test_invoke_routes_through_sentinel(self, local_endpoint): + target = SentinelTarget("demo", "default", "chat") + result = await target.invoke({"input": {"prompt": "hi"}}, timeout=10) + assert result == {"ok": True} + + request = local_endpoint.state["requests"][0] + assert request["path"] == f"/{SENTINEL_ID}/runsync" + assert request["headers"]["X-Flash-App"] == "demo" + assert request["headers"]["X-Flash-Environment"] == "default" + assert request["headers"]["X-Flash-Endpoint"] == "chat" + + async def test_submit_and_wait(self, local_endpoint): + local_endpoint.state["responses"][f"/{SENTINEL_ID}/run"] = { + "id": "j1", + "status": "IN_QUEUE", + } + local_endpoint.state["responses"][f"/{SENTINEL_ID}/status/j1"] = { + "id": "j1", + "status": "COMPLETED", + "output": 42, + } + target = SentinelTarget("demo", "default", "chat") + job = await target.submit({"input": {}}) + assert job["id"] == "j1" + assert await target.wait(job, timeout=10) == 42 + + def test_payload_is_plain_kwargs(self): + target = SentinelTarget("demo", "default", "chat") + + def fn(prompt): + pass + + payload = target.build_payload( + fn, ResourceSpec(kind=ResourceKind.QUEUE, name="chat"), ("hi",), {} + ) + assert payload == {"input": {"prompt": "hi"}} + + +class TestLiveTarget: + def _spec(self): + return ResourceSpec(kind=ResourceKind.QUEUE, name="chat") + + def test_payload_carries_source(self): + target = LiveTarget("ep123", "chat") + + def fn(prompt): + return prompt + + payload = target.build_payload(fn, self._spec(), ("hi",), {}) + body = payload["input"] + assert body["function_name"] == "fn" + assert "def fn(" in body["function_code"] + + def test_unwrap_success_response(self): + target = LiveTarget("ep123", "chat") + output = {"success": True, "result": None, "json_result": {"x": 1}} + assert target.unwrap( + {"status": "COMPLETED", "output": output} + ) == {"x": 1} + + def test_unwrap_failure_raises(self): + target = LiveTarget("ep123", "chat") + output = {"success": False, "error": "worker exploded"} + with pytest.raises(RemoteExecutionError, match="worker exploded"): + target.unwrap({"status": "COMPLETED", "output": output}) + + def test_unwrap_passthrough(self): + target = LiveTarget("ep123", "chat") + assert target.unwrap( + {"status": "COMPLETED", "output": {"plain": 1}} + ) == {"plain": 1} + + async def test_invoke(self, local_endpoint): + target = LiveTarget("ep123", "chat") + data = await target.invoke({"input": {"x": 1}}, timeout=10) + assert data == {"ok": True} + request = local_endpoint.state["requests"][0] + assert request["path"] == "/ep123/runsync" + + async def test_sync_source_skips_unchanged(self, monkeypatch): + target = LiveTarget("ep123", "chat") + + def backing(): + return 1 + + target.attach_source(backing, "chat", self._spec()) + post = AsyncMock(return_value={}) + with patch("runpod.apps.targets._post_json", post): + await target._sync_source(timeout=10) + await target._sync_source(timeout=10) + assert post.await_count == 1 + + async def test_sync_source_noop_without_attachment(self): + target = LiveTarget("ep123", "chat") + post = AsyncMock() + with patch("runpod.apps.targets._post_json", post): + await target._sync_source(timeout=10) + post.assert_not_awaited() diff --git a/tests/test_cli/test_management_commands.py b/tests/test_cli/test_management_commands.py new file mode 100644 index 000000000..131aceff9 --- /dev/null +++ b/tests/test_cli/test_management_commands.py @@ -0,0 +1,357 @@ +"""tests for app/env/secret/registry/logs cli commands.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +from click.testing import CliRunner + +from runpod.apps.manage import UndeployResult +from runpod.rp_cli.main import cli + + +def _runner(): + return CliRunner() + + +class TestAppCommands: + def test_list_empty(self): + with patch("runpod.apps.manage.list_apps", AsyncMock(return_value=[])): + result = _runner().invoke(cli, ["app", "list"]) + assert result.exit_code == 0 + assert "no apps deployed" in result.output + + def test_list(self): + apps = [ + { + "name": "demo", + "flashEnvironments": [{"name": "default"}, {"name": "prod"}], + }, + {"name": "other", "flashEnvironments": []}, + ] + with patch( + "runpod.apps.manage.list_apps", AsyncMock(return_value=apps) + ): + result = _runner().invoke(cli, ["app", "list"]) + assert result.exit_code == 0 + assert "demo" in result.output + assert "default, prod" in result.output + + def test_get(self): + entry = { + "name": "demo", + "flashEnvironments": [ + {"name": "default", "activeBuildId": "build123456789"} + ], + } + with patch( + "runpod.apps.manage.get_app", AsyncMock(return_value=entry) + ): + result = _runner().invoke(cli, ["app", "get", "demo"]) + assert result.exit_code == 0 + assert "build1234567" in result.output + + def test_get_error(self): + with patch( + "runpod.apps.manage.get_app", + AsyncMock(side_effect=RuntimeError("no app named 'demo'")), + ): + result = _runner().invoke(cli, ["app", "get", "demo"]) + assert result.exit_code == 1 + assert "no app named" in result.output + + def test_delete_confirmed(self): + outcome = UndeployResult(endpoints_deleted=2, failures=[]) + with patch( + "runpod.apps.manage.delete_app", AsyncMock(return_value=outcome) + ): + result = _runner().invoke(cli, ["app", "delete", "demo", "--yes"]) + assert result.exit_code == 0 + + def test_delete_aborts_without_confirmation(self): + result = _runner().invoke(cli, ["app", "delete", "demo"], input="n\n") + assert result.exit_code == 1 + + def test_delete_failures_keep_app(self): + outcome = UndeployResult(endpoints_deleted=0, failures=["ep1 stuck"]) + with patch( + "runpod.apps.manage.delete_app", AsyncMock(return_value=outcome) + ): + result = _runner().invoke(cli, ["app", "delete", "demo", "--yes"]) + assert result.exit_code == 1 + assert "undeploy incomplete" in result.output + + +class TestEnvCommands: + def test_list(self): + entry = { + "flashEnvironments": [ + { + "name": "default", + "activeBuildId": "build123456789", + "createdAt": "2026-01-02T03:04:05Z", + } + ] + } + with patch( + "runpod.apps.manage.get_app", AsyncMock(return_value=entry) + ): + result = _runner().invoke( + cli, ["env", "list", "--app", "demo"] + ) + assert result.exit_code == 0 + assert "default" in result.output + + def test_list_empty(self): + with patch( + "runpod.apps.manage.get_app", + AsyncMock(return_value={"flashEnvironments": []}), + ): + result = _runner().invoke(cli, ["env", "list", "--app", "demo"]) + assert result.exit_code == 0 + assert "no environments" in result.output + + def test_get(self): + entry = { + "name": "prod", + "activeBuildId": "build123456789", + "endpoints": [{"name": "chat", "id": "ep1"}], + } + with patch( + "runpod.apps.manage.get_environment", + AsyncMock(return_value=entry), + ): + result = _runner().invoke( + cli, ["env", "get", "prod", "--app", "demo"] + ) + assert result.exit_code == 0 + assert "demo/prod" in result.output + assert "chat" in result.output + + def test_add(self): + client = MagicMock() + client.create_environment = AsyncMock(return_value={"id": "env1"}) + with ( + patch("runpod.apps.api.AppsApiClient", return_value=client), + patch( + "runpod.apps.manage.get_app", + AsyncMock(return_value={"id": "app1"}), + ), + ): + result = _runner().invoke( + cli, ["env", "add", "staging", "--app", "demo"] + ) + assert result.exit_code == 0 + client.create_environment.assert_awaited_once_with("app1", "staging") + + def test_delete(self): + outcome = UndeployResult(endpoints_deleted=1, failures=[]) + with patch( + "runpod.apps.manage.undeploy_environment", + AsyncMock(return_value=outcome), + ): + result = _runner().invoke( + cli, ["env", "delete", "prod", "--app", "demo", "--yes"] + ) + assert result.exit_code == 0 + + def test_resolve_app_name_fails_without_unique_app(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["env", "list"]) + assert result.exit_code == 1 + assert "--app" in result.output + + +class TestUndeploy: + def test_undeploy(self): + outcome = UndeployResult(endpoints_deleted=3, failures=[]) + with patch( + "runpod.apps.manage.undeploy_environment", + AsyncMock(return_value=outcome), + ): + result = _runner().invoke( + cli, ["undeploy", "--app", "demo", "--yes"] + ) + assert result.exit_code == 0 + assert "3 endpoints removed" in result.output + + def test_undeploy_failures(self): + outcome = UndeployResult(endpoints_deleted=0, failures=["boom"]) + with patch( + "runpod.apps.manage.undeploy_environment", + AsyncMock(return_value=outcome), + ): + result = _runner().invoke( + cli, ["undeploy", "--app", "demo", "--yes"] + ) + assert result.exit_code == 1 + + +class TestSecretCommands: + def _client(self): + client = MagicMock() + client.create_secret = AsyncMock(return_value={"id": "s1"}) + client.list_secrets = AsyncMock( + return_value=[ + {"id": "s1", "name": "tok", "description": "a token"} + ] + ) + client.delete_secret = AsyncMock(return_value=True) + return client + + def test_add(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["secret", "add", "tok", "--value", "v"] + ) + assert result.exit_code == 0 + client.create_secret.assert_awaited_once_with("tok", "v", "") + + def test_add_prompts(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["secret", "add"], input="tok\nv\n" + ) + assert result.exit_code == 0 + + def test_list(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke(cli, ["secret", "list"]) + assert result.exit_code == 0 + assert "tok" in result.output + + def test_list_empty(self): + client = self._client() + client.list_secrets = AsyncMock(return_value=[]) + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke(cli, ["secret", "list"]) + assert "no secrets" in result.output + + def test_delete(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["secret", "delete", "tok", "--yes"] + ) + assert result.exit_code == 0 + client.delete_secret.assert_awaited_once_with("s1") + + def test_delete_unknown(self): + client = self._client() + client.list_secrets = AsyncMock(return_value=[]) + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["secret", "delete", "tok", "--yes"] + ) + assert result.exit_code == 1 + assert "no secret named" in result.output + + +class TestRegistryCommands: + def _client(self): + client = MagicMock() + client.create_registry_auth = AsyncMock(return_value={"id": "r1"}) + client.list_registry_auths = AsyncMock( + return_value=[{"id": "r1", "name": "dockerhub"}] + ) + client.delete_registry_auth = AsyncMock(return_value=True) + return client + + def test_add(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, + [ + "registry", "add", "dockerhub", + "--username", "u", "--password", "p", + ], + ) + assert result.exit_code == 0 + client.create_registry_auth.assert_awaited_once_with( + "dockerhub", "u", "p" + ) + + def test_list(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke(cli, ["registry", "list"]) + assert result.exit_code == 0 + assert "dockerhub" in result.output + + def test_delete(self): + client = self._client() + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["registry", "delete", "dockerhub", "--yes"] + ) + assert result.exit_code == 0 + client.delete_registry_auth.assert_awaited_once_with("r1") + + def test_delete_unknown(self): + client = self._client() + client.list_registry_auths = AsyncMock(return_value=[]) + with patch("runpod.apps.api.AppsApiClient", return_value=client): + result = _runner().invoke( + cli, ["registry", "delete", "nope", "--yes"] + ) + assert result.exit_code == 1 + + +class TestLogsCommand: + def test_snapshot(self): + logs = {"system": ["booted"], "container": ["hello"]} + with patch( + "runpod.apps.logs.pod_logs", AsyncMock(return_value=logs) + ): + result = _runner().invoke(cli, ["logs", "pod1"]) + assert result.exit_code == 0 + assert "[system] booted" in result.output + assert "[container] hello" in result.output + + def test_follow(self): + async def fake_stream(pod_id, **kwargs): + yield {"source": "container", "line": "streamed"} + + with patch("runpod.apps.logs.stream_pod_logs", fake_stream): + result = _runner().invoke(cli, ["logs", "pod1", "--follow"]) + assert result.exit_code == 0 + assert "[container] streamed" in result.output + + +class TestLoginCommand: + def test_api_key_flag(self): + with patch( + "runpod.cli.groups.config.functions.set_credentials" + ) as creds: + result = _runner().invoke( + cli, ["login", "--api-key", "sk-test"] + ) + assert result.exit_code == 0 + creds.assert_called_once_with("sk-test", overwrite=True) + + def test_browser_flow(self): + with ( + patch( + "runpod.apps.auth.browser_login", + AsyncMock(return_value="sk-granted"), + ), + patch( + "runpod.cli.groups.config.functions.set_credentials" + ) as creds, + ): + result = _runner().invoke(cli, ["login", "--no-open"]) + assert result.exit_code == 0 + creds.assert_called_once_with("sk-granted", overwrite=True) + + def test_browser_flow_error(self): + from runpod.apps.auth import LoginError + + with patch( + "runpod.apps.auth.browser_login", + AsyncMock(side_effect=LoginError("expired")), + ): + result = _runner().invoke(cli, ["login", "--no-open"]) + assert result.exit_code == 1 + assert "expired" in result.output From c79afbc299df9e0c3e8b384f95be706129e3ab3f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:41:42 -0700 Subject: [PATCH 078/120] test: cover console rendering --- tests/test_cli/test_console.py | 229 +++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 tests/test_cli/test_console.py diff --git a/tests/test_cli/test_console.py b/tests/test_cli/test_console.py new file mode 100644 index 000000000..4fe4717a4 --- /dev/null +++ b/tests/test_cli/test_console.py @@ -0,0 +1,229 @@ +"""rendering tests for the rp cli console module.""" + +import io + +import pytest +from rich.console import Console + +from runpod.rp_cli import console as ui + + +@pytest.fixture(autouse=True) +def capture_console(monkeypatch): + """swap the module console for one writing to a buffer.""" + buffer = io.StringIO() + test_console = Console( + file=buffer, + highlight=False, + theme=ui.theme, + force_terminal=False, + width=120, + ) + monkeypatch.setattr(ui, "console", test_console) + yield buffer + + +def _out(buffer): + return buffer.getvalue() + + +class TestGenericLines: + def test_info(self, capture_console): + ui.info("hello") + assert "hello" in _out(capture_console) + + def test_success(self, capture_console): + ui.success("done") + assert "✓ done" in _out(capture_console) + + def test_error(self, capture_console): + ui.error("bad") + assert "✗ bad" in _out(capture_console) + + def test_warn(self, capture_console): + ui.warn("careful") + assert "! careful" in _out(capture_console) + + +class TestHelpers: + def test_endpoint_url(self): + assert ui.endpoint_url("ep1").endswith("/ep1?tab=overview") + + def test_endpoint_link_markup(self): + assert "ep1 ↗" in ui.endpoint_link("ep1") + + def test_name_width_padding(self): + ui.set_name_width(["a", "longer"]) + assert ui._padded("a") == "a " + ui.set_name_width([]) + assert ui._padded("a") == "a" + + def test_kind_badge_known_and_unknown(self): + assert "queue" in ui.kind_badge("queue") + assert "other" in ui.kind_badge("other") + + def test_fmt_elapsed(self): + assert ui._fmt_elapsed(0.5) == "500ms" + assert ui._fmt_elapsed(3.21) == "3.2s" + assert ui._fmt_elapsed(150) == "2m30s" + + def test_bar_bounds(self): + assert "━" in ui._bar(0.5) + ui._bar(-1) + ui._bar(2) + + def test_sand_spinner_registered(self): + from rich.spinner import SPINNERS + + assert ui.RUNPOD_SAND_SPINNER in SPINNERS + assert ( + SPINNERS[ui.RUNPOD_SAND_SPINNER]["frames"] + == ui.RUNPOD_SAND_SPINNER_FRAMES + ) + + +class TestBanners: + def test_dev_banner(self, capture_console): + ui.dev_banner(["demo"], "main.py") + assert "demo in main.py" in _out(capture_console) + + def test_deploy_plan(self, capture_console): + ui.deploy_plan([("demo", "main.py", 2), ("other", "", 1)]) + out = _out(capture_console) + assert "2 apps" in out + assert "main.py, 2 resources" in out + assert "1 resource" in out + + def test_deploy_banner(self, capture_console): + ui.deploy_banner("demo", "prod", [("chat", "queue")], source="main.py") + out = _out(capture_console) + assert "demo" in out + assert "prod" in out + assert "chat" in out + + def test_deploy_banner_no_resources(self, capture_console): + ui.deploy_banner("demo", "prod", []) + assert "(no resources)" in _out(capture_console) + + def test_resources_table(self, capture_console): + ui.resources_table( + [ + ("chat", "queue", "4090", "ep1"), + ("train", "task", "H100", "per-call"), + ] + ) + out = _out(capture_console) + assert "chat" in out + assert "per-call" in out + + def test_resources_table_empty(self, capture_console): + ui.resources_table([]) + assert _out(capture_console) == "" + + def test_reload_banner(self, capture_console): + ui.reload_banner("main.py") + assert "reloading" in _out(capture_console) + + def test_entrypoint_lines(self, capture_console): + ui.entrypoint_header("main") + ui.entrypoint_success(1.2) + ui.entrypoint_failure(3.4, "boom") + out = _out(capture_console) + assert "main()" in out + assert "done" in out + assert "boom" in out + + +class TestDeployEvents: + def test_phase_lifecycle(self, capture_console): + events = ui.DeployEvents() + events.phase("vendor") + events.vendor_progress(3, "torch") + events.phase("upload") + events.upload_progress(512 * 1024, 1024 * 1024) + events.phase("endpoints") + events.endpoint_ready("chat", "ep1") + events.close() + assert events.endpoints == {"chat": "ep1"} + + def test_progress_updates_ignored_outside_phase(self): + events = ui.DeployEvents() + events.vendor_progress(1, "torch") + events.upload_progress(1, 2) + events.close() + + +class TestDevEvents: + def test_request_lifecycle(self, capture_console): + events = ui.DevEvents() + events.dispatch("chat", "queued") + events.worker_ready("chat", "worker123456789") + events.worker_log("chat", "processing [1/3]") + events.worker_status("chat", {"initializing": 1}) + events.task_status("chat", "pod starting") + events.request_completed("chat", 2.5) + events.request_failed("chat", 1.0) + out = _out(capture_console) + assert "→ chat()" in out + assert "worker worker123456 ready" in out + assert "processing [1/3]" in out # markup escaped, shown verbatim + assert "waiting: 1 initializing" in out + assert "pod starting" in out + assert "✓ chat()" in out + assert "✗ chat()" in out + + def test_provision_rows_without_progress(self, capture_console): + events = ui.DevEvents() + events.provisioning("chat", "queue", "4090") + events.adopted("chat", "ep1") + events.ready("chat", "ep1") + events.deleted("chat") + out = _out(capture_console) + assert "provisioning queue on 4090" in out + assert "− chat" in out + + def test_session_progress_lifecycle(self, capture_console): + events = ui.DevEvents() + events.session_starting() + events.provisioning("chat", "queue", "4090") + events.ready("chat", "ep1") + events.session_started() + assert events._progress is None + + def test_refresh_diff(self, capture_console): + events = ui.DevEvents() + events.resource_added("new", "queue", "4090") + events.resource_changed("chat", ["gpu"]) + events.resource_changed("chat", []) + events.resource_removed("old") + events.volume_created("data", 10, "US-KS-2") + out = _out(capture_console) + assert "+ new" in out + assert "~ chat" in out + assert "− old" in out + assert "volume data" in out + + +class TestCleanupEvents: + def test_zero_total_is_silent(self, capture_console): + events = ui.CleanupEvents() + events.cleanup_started(0) + events.deleting("x") + events.deleted("x") + events.close() + + def test_delete_progress(self, capture_console): + events = ui.CleanupEvents() + events.cleanup_started(2) + events.deleting("ep1") + events.deleted("ep1") + events.delete_failed("ep2") + events.close() + assert "could not delete ep2" in _out(capture_console) + + +class TestTimer: + def test_elapsed(self): + with ui.Timer() as t: + assert t.so_far >= 0 + assert t.elapsed >= 0 From 5c423b01387401d23c15cbb3e0db53cdbd04fde5 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:44:29 -0700 Subject: [PATCH 079/120] test: cover deploy and dev command wiring --- tests/test_cli/test_deploy_command.py | 223 ++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 tests/test_cli/test_deploy_command.py diff --git a/tests/test_cli/test_deploy_command.py b/tests/test_cli/test_deploy_command.py new file mode 100644 index 000000000..8a0196566 --- /dev/null +++ b/tests/test_cli/test_deploy_command.py @@ -0,0 +1,223 @@ +"""tests for rp deploy and rp dev command wiring.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from click.testing import CliRunner + +from runpod.apps.deploy import DeployResult +from runpod.rp_cli.main import cli + + +@pytest.fixture(autouse=True) +def clean_entrypoints(): + from runpod.apps.entrypoint import _clear_entrypoints + + _clear_entrypoints() + yield + _clear_entrypoints() + +APP_SOURCE = ''' +import runpod + +app = runpod.App("demo") + + +@app.queue(name="chat", gpu="4090") +def chat(prompt: str): + return prompt +''' + +TWO_APP_SOURCE = APP_SOURCE + ''' + +other = runpod.App("other") + + +@other.queue(name="embed", gpu="4090") +def embed(text: str): + return text +''' + +ENTRYPOINT_SOURCE = APP_SOURCE + ''' + +@runpod.local_entrypoint +def main(): + pass +''' + + +def _runner(): + return CliRunner() + + +def _result(name="demo", endpoints=None): + return DeployResult( + app_name=name, + build_id="b1", + environment_id="env1", + resources=["chat"], + endpoints=endpoints or {"chat": "ep1"}, + ) + + +class TestDeploy: + def test_deploys_discovered_app(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock(return_value=_result()) + with patch("runpod.apps.deploy.deploy_app", deploy_app): + result = _runner().invoke(cli, ["deploy"]) + assert result.exit_code == 0, result.output + assert "demo/default" in result.output + assert "ep1" in result.output + deploy_app.assert_awaited_once() + + def test_env_flag(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock(return_value=_result()) + with patch("runpod.apps.deploy.deploy_app", deploy_app): + result = _runner().invoke(cli, ["deploy", "--env", "prod"]) + assert result.exit_code == 0 + assert deploy_app.call_args[1]["env_name"] == "prod" + assert "demo/prod" in result.output + + def test_multi_app_plan(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(TWO_APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock( + side_effect=[_result("demo"), _result("other", {"embed": "ep2"})] + ) + with patch("runpod.apps.deploy.deploy_app", deploy_app): + result = _runner().invoke(cli, ["deploy"]) + assert result.exit_code == 0, result.output + assert "2 apps" in result.output + assert deploy_app.await_count == 2 + + def test_missing_target(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["deploy", "missing_dir"]) + assert result.exit_code == 1 + assert "does not exist" in result.output + + def test_no_apps_found(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["deploy"]) + assert result.exit_code == 1 + assert "no runpod.App found" in result.output + + def test_engine_error_is_clean(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock(side_effect=RuntimeError("upload failed")) + with patch("runpod.apps.deploy.deploy_app", deploy_app): + result = _runner().invoke(cli, ["deploy"]) + assert result.exit_code == 1 + assert "upload failed" in result.output + assert "Traceback" not in result.output + + def test_build_only(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + artifact = tmp_path / "demo-artifact.tar.gz" + artifact.write_bytes(b"x" * 100) + build = MagicMock(return_value=artifact) + with patch("runpod.apps.deploy.build_artifact", build): + result = _runner().invoke(cli, ["deploy", "--build-only"]) + assert result.exit_code == 0, result.output + assert "demo-artifact.tar.gz" in result.output + assert build.call_args[1]["output"] == artifact + + def test_exclude_passthrough(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock(return_value=_result()) + with patch("runpod.apps.deploy.deploy_app", deploy_app): + result = _runner().invoke( + cli, ["deploy", "--exclude", "torch,numpy"] + ) + assert result.exit_code == 0 + assert deploy_app.call_args[1]["exclude"] == ["torch", "numpy"] + + def test_python_version_fallback_warns(self, tmp_path, monkeypatch): + (tmp_path / "main.py").write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + deploy_app = AsyncMock(return_value=_result()) + with ( + patch("runpod.apps.deploy.deploy_app", deploy_app), + patch( + "runpod.apps.images.local_python_version", + side_effect=RuntimeError("3.9 unsupported"), + ), + ): + result = _runner().invoke(cli, ["deploy"]) + assert result.exit_code == 0 + assert "no runtime image" in result.output + + +class TestDev: + def test_once_runs_entrypoint_and_stops(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text(ENTRYPOINT_SOURCE) + monkeypatch.chdir(tmp_path) + + session = MagicMock() + session.start = AsyncMock() + session.stop = AsyncMock() + session.refresh = AsyncMock() + session._endpoints = {"dev-demo-chat": "ep1"} + + def make_session(apps, events=None): + session.apps = apps + return session + + with patch("runpod.apps.dev.DevSession", side_effect=make_session): + result = _runner().invoke(cli, ["dev", str(module), "--once"]) + assert result.exit_code == 0, result.output + session.start.assert_awaited_once() + session.stop.assert_awaited_once() + + def test_once_entrypoint_failure_exits_nonzero(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text( + APP_SOURCE + + "\n@runpod.local_entrypoint\ndef main():\n" + + " raise RuntimeError('assertion blew up')\n" + ) + monkeypatch.chdir(tmp_path) + + session = MagicMock() + session.start = AsyncMock() + session.stop = AsyncMock() + session._endpoints = {} + + def make_session(apps, events=None): + session.apps = apps + return session + + with patch("runpod.apps.dev.DevSession", side_effect=make_session): + result = _runner().invoke(cli, ["dev", str(module), "--once"]) + assert result.exit_code == 1 + session.stop.assert_awaited_once() + + def test_module_without_entrypoint(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text(APP_SOURCE) + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["dev", str(module), "--once"]) + assert result.exit_code == 1 + assert "local_entrypoint" in result.output + + def test_module_without_app(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["dev", str(module), "--once"]) + assert result.exit_code == 1 + assert "no runpod.App" in result.output + + def test_missing_module(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = _runner().invoke(cli, ["dev", "missing.py", "--once"]) + assert result.exit_code == 2 From 6bbbe6bf40ecdb02067b6879a96b343ef460ee5a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:53:08 -0700 Subject: [PATCH 080/120] test: cover bootstrap error surface, task lifecycle, and update internals --- tests/test_apps/test_bootstrap.py | 194 ++++++++++++++++++++++++++ tests/test_apps/test_tasks.py | 154 +++++++++++++++++++- tests/test_cli/test_update_command.py | 133 +++++++++++++++++- 3 files changed, 479 insertions(+), 2 deletions(-) diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 2d46b5b53..5eadedc0c 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -264,3 +264,197 @@ def test_no_third_party_imports(self): "urllib", "io", "importlib", "http", } assert imports <= stdlib, f"non-stdlib imports: {imports - stdlib}" + + +class TestErrorSurface: + def test_error_payload_shape(self): + error = bootstrap.PhaseError("locate", "artifact missing") + payload = bootstrap._error_payload(error) + assert payload["error_type"] == "BootstrapError" + assert "locate" in payload["error_message"] + assert "artifact missing" in payload["error_message"] + + def test_queue_error_loop_exits_without_webhooks(self, monkeypatch): + monkeypatch.delenv("RUNPOD_WEBHOOK_GET_JOB", raising=False) + monkeypatch.delenv("RUNPOD_WEBHOOK_POST_OUTPUT", raising=False) + with pytest.raises(SystemExit): + bootstrap._queue_error_loop(bootstrap.PhaseError("locate", "x")) + + def test_queue_error_loop_reports_jobs(self, monkeypatch): + import contextlib + import io + + monkeypatch.setenv("RUNPOD_WEBHOOK_GET_JOB", "http://jobs/get") + monkeypatch.setenv("RUNPOD_WEBHOOK_POST_OUTPUT", "http://jobs/out/$ID") + monkeypatch.setenv("RUNPOD_AI_API_KEY", "k") + + posted = [] + + @contextlib.contextmanager + def _get_response(): + response = io.BytesIO(json.dumps({"id": "job1"}).encode()) + response.status = 200 + yield response + + def fake_urlopen(req, timeout=None): + url = req.full_url + if url.endswith("/get"): + return _get_response() + posted.append((url, req.data)) + # first report done: break out of the infinite loop + raise KeyboardInterrupt + + monkeypatch.setattr( + bootstrap.urllib.request, "urlopen", fake_urlopen + ) + error = bootstrap.PhaseError("attach", "manifest gone") + with pytest.raises(KeyboardInterrupt): + bootstrap._queue_error_loop(error) + + assert posted + url, body = posted[0] + assert url.endswith("/out/job1") + assert b"BootstrapError" in body + + def test_report_error_routes_by_kind(self, monkeypatch): + calls = [] + monkeypatch.setattr( + bootstrap, "_api_error_server", lambda e: calls.append("api") + ) + monkeypatch.setattr( + bootstrap, "_queue_error_loop", lambda e: calls.append("queue") + ) + error = bootstrap.PhaseError("locate", "x") + + monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") + bootstrap._report_error(error) + monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "queue") + bootstrap._report_error(error) + assert calls == ["api", "queue"] + + +class TestLiveRuntimeInstall: + def test_importable_runtime_no_op(self, monkeypatch): + monkeypatch.setattr( + bootstrap, "_worker_importable", lambda paths: (True, "") + ) + installs = [] + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) + ) + bootstrap._ensure_runtime_installed() + assert installs == [] + + def test_installs_package_spec(self, monkeypatch): + probes = iter([(False, "no module"), (True, "")]) + monkeypatch.setattr( + bootstrap, "_worker_importable", lambda paths: next(probes) + ) + installs = [] + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) + ) + monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==1.99.0") + bootstrap._ensure_runtime_installed() + assert installs == [["runpod==1.99.0", "cloudpickle"]] + + def test_api_kind_adds_uvicorn(self, monkeypatch): + probes = iter([(False, "no module"), (True, "")]) + monkeypatch.setattr( + bootstrap, "_worker_importable", lambda paths: next(probes) + ) + installs = [] + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) + ) + monkeypatch.delenv("RUNPOD_PACKAGE_SPEC", raising=False) + monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") + bootstrap._ensure_runtime_installed() + assert installs == [["runpod", "cloudpickle", "uvicorn>=0.30"]] + + def test_install_failure_raises_phase(self, monkeypatch): + monkeypatch.setattr( + bootstrap, "_worker_importable", lambda paths: (False, "still broken") + ) + monkeypatch.setattr( + bootstrap, "_pip_install", lambda pkgs, phase: None + ) + with pytest.raises(bootstrap.PhaseError) as exc_info: + bootstrap._ensure_runtime_installed() + assert exc_info.value.phase == "runtime" + + +class TestServe: + def test_serve_execs_worker_module(self, monkeypatch): + execs = [] + monkeypatch.setattr( + bootstrap.os, + "execve", + lambda exe, argv, env: execs.append((exe, argv, env)), + ) + monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "queue") + monkeypatch.setenv("PYTHONPATH", "/existing") + bootstrap._serve(["/env", "/app"]) + exe, argv, env = execs[0] + assert argv[-1] == bootstrap.WORKER_MODULES["queue"] + assert env["PYTHONPATH"].split(":")[:2] == ["/env", "/app"] + assert "/existing" in env["PYTHONPATH"] + assert env["RUNPOD_APP_DIR"] == "/app" + + def test_serve_live_mode_no_paths(self, monkeypatch): + execs = [] + monkeypatch.setattr( + bootstrap.os, + "execve", + lambda exe, argv, env: execs.append((exe, argv, env)), + ) + monkeypatch.delenv("PYTHONPATH", raising=False) + monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") + bootstrap._serve([]) + _, argv, env = execs[0] + assert argv[-1] == bootstrap.WORKER_MODULES["api"] + assert "RUNPOD_APP_DIR" not in env + + +class TestMain: + def test_deployed_path_serves(self, tmp_path, monkeypatch): + app_dir = str(tmp_path) + (tmp_path / bootstrap.MANIFEST_NAME).write_text( + json.dumps({"resources": [{"name": "chat"}]}) + ) + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + monkeypatch.setattr(bootstrap, "_locate", lambda: app_dir) + monkeypatch.setattr( + bootstrap, "_worker_importable", lambda paths: (True, "") + ) + served = [] + monkeypatch.setattr(bootstrap, "_serve", lambda paths: served.append(paths)) + bootstrap.main() + assert served and served[0][-1] == app_dir + + def test_phase_error_reports(self, monkeypatch): + monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") + monkeypatch.setattr( + bootstrap, + "_locate", + lambda: (_ for _ in ()).throw( + bootstrap.PhaseError("locate", "gone") + ), + ) + reported = [] + monkeypatch.setattr( + bootstrap, "_report_error", lambda e: reported.append(e) + ) + bootstrap.main() + assert reported and reported[0].phase == "locate" + + def test_live_path(self, monkeypatch): + monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) + monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) + monkeypatch.setattr( + bootstrap, "_ensure_runtime_installed", lambda: None + ) + served = [] + monkeypatch.setattr(bootstrap, "_serve", lambda paths: served.append(paths)) + bootstrap.main() + assert served == [[]] diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index d8cf9b3fd..7b05719dd 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -11,7 +11,7 @@ import base64 import json -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import cloudpickle import pytest @@ -387,3 +387,155 @@ def test_mixed_pool_and_device(self): names = _device_names(["AMPERE_48", "NVIDIA L40S"]) assert "NVIDIA A40" in names and "NVIDIA L40S" in names + + +class TestTaskExecutionLifecycle: + def _spec(self, **overrides): + from runpod.apps.spec import ResourceKind, ResourceSpec + + defaults = dict(kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"]) + defaults.update(overrides) + return ResourceSpec(**defaults) + + async def test_start_deploys_pod(self): + from runpod.apps.tasks import TaskExecution + + api = MagicMock() + api.deploy_task_pod = AsyncMock(return_value={"id": "pod-9"}) + execution = TaskExecution(self._spec(), api=api) + await execution.start() + assert execution.pod_id == "pod-9" + assert api.deploy_task_pod.call_args[1]["is_cpu"] is True + + async def test_start_resolves_registry_auth(self): + from runpod.apps.tasks import TaskExecution + + api = MagicMock() + api.deploy_task_pod = AsyncMock(return_value={"id": "pod-9"}) + execution = TaskExecution( + self._spec(image="private/img", registry_auth="dh"), api=api + ) + with patch( + "runpod.apps.registry.resolve_registry_auth", + AsyncMock(return_value="auth-1"), + ): + await execution.start() + pod = api.deploy_task_pod.call_args[0][0] + assert pod["containerRegistryAuthId"] == "auth-1" + + async def test_terminate_swallows_api_errors(self): + from runpod.apps.tasks import TaskExecution + + api = MagicMock() + api.terminate_pod = AsyncMock(side_effect=RuntimeError("api down")) + execution = TaskExecution(self._spec(), api=api) + execution.pod_id = "pod-9" + await execution.terminate() # must not raise + assert execution.pod_id is None + + async def test_terminate_noop_without_pod(self): + from runpod.apps.tasks import TaskExecution + + api = MagicMock() + api.terminate_pod = AsyncMock() + execution = TaskExecution(self._spec(), api=api) + await execution.terminate() + api.terminate_pod.assert_not_awaited() + + async def test_execute_polls_to_done(self): + from runpod.apps.tasks import TaskExecution + + execution = TaskExecution(self._spec(), api=MagicMock()) + execution.pod_id = "pod-9" + execution.submit = AsyncMock() + polls = [None, {"success": True, "json_result": 4}] + execution.poll_result = AsyncMock(side_effect=polls) + with patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()): + response = await execution.execute({"fn": "t"}, timeout=60) + assert response == {"success": True, "json_result": 4} + + async def test_execute_timeout(self): + from runpod.apps.tasks import TaskExecution + + execution = TaskExecution(self._spec(), api=MagicMock()) + execution.pod_id = "pod-9" + execution.submit = AsyncMock() + execution.poll_result = AsyncMock(return_value=None) + with ( + patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()), + patch( + "runpod.apps.tasks.time.monotonic", + side_effect=[0, 0, 100], + ), + ): + with pytest.raises(TimeoutError, match="did not finish"): + await execution.execute({"fn": "t"}, timeout=10) + + +class TestUnwrapTaskResponse: + def test_failure_raises(self): + from runpod.apps.errors import RemoteExecutionError + from runpod.apps.tasks import unwrap_task_response + + with pytest.raises(RemoteExecutionError, match="worker oom"): + unwrap_task_response({"success": False, "error": "worker oom"}) + + def test_json_result(self): + from runpod.apps.tasks import unwrap_task_response + + assert unwrap_task_response( + {"success": True, "json_result": {"x": 1}} + ) == {"x": 1} + + def test_pickled_result(self): + from runpod.apps.tasks import unwrap_task_response + + assert unwrap_task_response({"success": True, "result": _b64(7)}) == 7 + + +class TestTaskJob: + def _job(self): + from runpod.apps.tasks import TaskExecution, TaskJob + + execution = MagicMock(spec=TaskExecution) + execution.pod_id = "pod-9" + execution.terminate = AsyncMock() + return TaskJob(execution), execution + + async def test_wait_returns_result_and_terminates(self): + job, execution = self._job() + execution.poll_result = AsyncMock( + side_effect=[None, {"success": True, "json_result": 9}] + ) + with patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()): + result = await job.wait() + assert result == 9 + execution.terminate.assert_awaited_once() + + async def test_wait_timeout_keeps_pod(self): + job, execution = self._job() + execution.poll_result = AsyncMock(return_value=None) + with ( + patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()), + patch( + "runpod.apps.tasks.time.monotonic", side_effect=[0, 100] + ), + ): + with pytest.raises(TimeoutError): + await job.wait(timeout=10) + execution.terminate.assert_not_awaited() + + async def test_wait_after_done_returns_cached(self): + job, execution = self._job() + execution.poll_result = AsyncMock( + return_value={"success": True, "json_result": 9} + ) + assert await job.wait() == 9 + assert await job.wait() == 9 + assert execution.poll_result.await_count == 1 + + async def test_cancel_terminates(self): + job, execution = self._job() + await job.cancel() + execution.terminate.assert_awaited_once() + assert job._done diff --git a/tests/test_cli/test_update_command.py b/tests/test_cli/test_update_command.py index 4ed658057..7c941e654 100644 --- a/tests/test_cli/test_update_command.py +++ b/tests/test_cli/test_update_command.py @@ -1,8 +1,10 @@ """rp update and the passive update check.""" import json -from unittest.mock import patch +import threading +from unittest.mock import MagicMock, patch +import pytest from click.testing import CliRunner from runpod.rp_cli import update as upd @@ -131,3 +133,132 @@ def test_check_never_raises(self, tmp_path, monkeypatch): upd, "fetch_pypi_metadata", side_effect=RuntimeError("boom") ), patch.object(upd, "current_version", return_value="1.0.0"): upd._run_check() + + +class TestFetchPypiMetadata: + def _response(self, payload: bytes): + import contextlib + import io + + @contextlib.contextmanager + def _cm(url, timeout=None): + yield io.BytesIO(payload) + + return _cm + + def test_parses_metadata(self, monkeypatch): + import json as _json + + payload = _json.dumps( + {"info": {"version": "1.9.0"}, "releases": {"1.8.0": [], "1.9.0": []}} + ).encode() + monkeypatch.setattr( + upd.urllib.request, "urlopen", self._response(payload) + ) + latest, releases = upd.fetch_pypi_metadata() + assert latest == "1.9.0" + assert releases == {"1.8.0", "1.9.0"} + + def test_network_error(self, monkeypatch): + import urllib.error + + def _raise(url, timeout=None): + raise urllib.error.URLError("no route") + + monkeypatch.setattr(upd.urllib.request, "urlopen", _raise) + with pytest.raises(ConnectionError, match="could not reach pypi"): + upd.fetch_pypi_metadata() + + def test_http_error(self, monkeypatch): + import urllib.error + + def _raise(url, timeout=None): + raise urllib.error.HTTPError(url, 503, "unavailable", {}, None) + + monkeypatch.setattr(upd.urllib.request, "urlopen", _raise) + with pytest.raises(RuntimeError, match="HTTP 503"): + upd.fetch_pypi_metadata() + + def test_garbage_response(self, monkeypatch): + monkeypatch.setattr( + upd.urllib.request, "urlopen", self._response(b"not json") + ) + with pytest.raises(RuntimeError, match="unexpected response"): + upd.fetch_pypi_metadata() + + def test_missing_version_key(self, monkeypatch): + monkeypatch.setattr( + upd.urllib.request, "urlopen", self._response(b"{}") + ) + with pytest.raises(RuntimeError, match="missing version"): + upd.fetch_pypi_metadata() + + +class TestRunInstall: + def test_success(self, monkeypatch): + from unittest.mock import MagicMock + + result = MagicMock(returncode=0) + run = MagicMock(return_value=result) + monkeypatch.setattr(upd.subprocess, "run", run) + upd.run_install("1.9.0") + assert "runpod==1.9.0" in run.call_args[0][0] + + def test_failure_raises(self, monkeypatch): + from unittest.mock import MagicMock + + result = MagicMock(returncode=1, stderr="resolution failed") + monkeypatch.setattr( + upd.subprocess, "run", MagicMock(return_value=result) + ) + with pytest.raises(RuntimeError, match="resolution failed"): + upd.run_install("1.9.0") + + +class TestUpdateNotice: + def test_notice_printed_when_newer(self, monkeypatch, capsys): + monkeypatch.setattr(upd, "_newer_version", "9.9.9") + upd._check_done.set() + upd._print_update_notice() + err = capsys.readouterr().err + assert "9.9.9" in err + assert "rp update" in err + + def test_no_notice_when_current(self, monkeypatch, capsys): + monkeypatch.setattr(upd, "_newer_version", None) + upd._check_done.set() + upd._print_update_notice() + assert capsys.readouterr().err == "" + + def test_no_notice_when_check_incomplete(self, monkeypatch, capsys): + monkeypatch.setattr(upd, "_newer_version", "9.9.9") + monkeypatch.setattr(upd, "_check_done", threading.Event()) + upd._print_update_notice() + assert capsys.readouterr().err == "" + + +class TestStartBackgroundCheck: + def test_starts_once(self, monkeypatch): + monkeypatch.setattr(upd, "_started", False) + monkeypatch.setattr(upd, "_is_interactive", lambda: True) + threads = [] + monkeypatch.setattr( + upd.threading, + "Thread", + lambda **kw: threads.append(kw) or MagicMock(), + ) + upd.start_background_check() + upd.start_background_check() + assert len(threads) == 1 + + def test_skips_non_tty(self, monkeypatch): + monkeypatch.setattr(upd, "_started", False) + monkeypatch.setattr(upd, "_is_interactive", lambda: False) + threads = [] + monkeypatch.setattr( + upd.threading, + "Thread", + lambda **kw: threads.append(kw) or MagicMock(), + ) + upd.start_background_check() + assert threads == [] From e7956358a8f1571414894ce3bf048f19764644b7 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:55:18 -0700 Subject: [PATCH 081/120] test: cover task runner self-termination and install helpers --- tests/test_apps/test_task_runner_http.py | 100 +++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index fea641ad3..ded81bf2f 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -243,3 +243,103 @@ def test_authed_requests_touch_contact(self, server, monkeypatch): monkeypatch.setattr(task_runner, "_last_contact", {"ts": None}) _request(f"{server}/result") assert task_runner._last_contact["ts"] is not None + + +class TestSelfTermination: + def test_terminate_self_calls_graphql_then_exits(self, monkeypatch): + import contextlib + import io as _io + + calls = [] + + @contextlib.contextmanager + def _response(): + yield _io.BytesIO(b"{}") + + def fake_urlopen(req, timeout=None): + calls.append((req.full_url, req.data)) + return _response() + + exits = [] + monkeypatch.setenv("RUNPOD_POD_ID", "pod-1") + monkeypatch.setenv("RUNPOD_API_KEY", "pod-scoped-key") + monkeypatch.setattr( + "urllib.request.urlopen", fake_urlopen + ) + monkeypatch.setattr( + task_runner.os, "_exit", lambda code: exits.append(code) + ) + task_runner._terminate_self() + + assert exits == [0] + url, body = calls[0] + assert url.endswith("/graphql") + assert b"podTerminate" in body + assert b"pod-1" in body + + def test_terminate_self_exits_without_credentials(self, monkeypatch): + exits = [] + monkeypatch.delenv("RUNPOD_POD_ID", raising=False) + monkeypatch.delenv("RUNPOD_API_KEY", raising=False) + monkeypatch.setattr( + task_runner.os, "_exit", lambda code: exits.append(code) + ) + task_runner._terminate_self() + assert exits == [0] + + def test_terminate_self_survives_api_failure(self, monkeypatch): + def fail(req, timeout=None): + raise OSError("network down") + + exits = [] + monkeypatch.setenv("RUNPOD_POD_ID", "pod-1") + monkeypatch.setenv("RUNPOD_API_KEY", "k") + monkeypatch.setattr("urllib.request.urlopen", fail) + monkeypatch.setattr( + task_runner.os, "_exit", lambda code: exits.append(code) + ) + task_runner._terminate_self() + assert exits == [0] + + +class TestCloudpickleLoading: + def test_available_returns_module(self): + module = task_runner._load_cloudpickle() + assert module is not None + assert hasattr(module, "dumps") + + def test_missing_without_install_returns_none(self, monkeypatch): + import builtins + + real_import = builtins.__import__ + + def no_cloudpickle(name, *args, **kwargs): + if name == "cloudpickle": + raise ImportError("not installed") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_cloudpickle) + assert task_runner._load_cloudpickle(install=False) is None + + +class TestInstallHelpers: + def test_install_empty_is_noop(self): + assert task_runner._install([], "nothing") is None + + def test_install_failure_returns_message(self, monkeypatch): + from unittest.mock import MagicMock + + result = MagicMock(returncode=1, stderr="resolver exploded") + monkeypatch.setattr( + task_runner.subprocess, "run", MagicMock(return_value=result) + ) + message = task_runner._install(["ghost-package"], "deps") + assert "resolver exploded" in message + + def test_install_system_requires_apt(self, monkeypatch): + monkeypatch.setattr(task_runner.shutil, "which", lambda _: None) + message = task_runner._install_system(["ffmpeg"]) + assert "apt-get" in message + + def test_install_system_empty_is_noop(self): + assert task_runner._install_system([]) is None From 3397a3e07274167a05a7307c0290269ccf691dec Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:36:03 -0700 Subject: [PATCH 082/120] fix: address codeql alerts on raises, tar links, and duplicate tests --- runpod/apps/api.py | 3 ++- runpod/apps/targets.py | 3 ++- runpod/apps/tasks.py | 1 + runpod/runtimes/bootstrap.py | 17 +++++++++++++++-- tests/test_apps/test_model.py | 5 ++++- tests/test_apps/test_tasks.py | 21 --------------------- 6 files changed, 24 insertions(+), 26 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index e973f30e9..f9fac098b 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -41,7 +41,8 @@ async def _execute( # transport-level failures (ssl hiccups, resets, dns) # are transient; graphql/auth errors propagate untouched last_exc = exc - raise last_exc # type: ignore[misc] + assert last_exc is not None # loop always runs at least once + raise last_exc # -- endpoints (dev session provisioning) -- diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index a9ddfd317..434b0feee 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -184,7 +184,8 @@ async def _request_json( # decided retryability last_exc = exc continue - raise last_exc # type: ignore[misc] + assert last_exc is not None # loop always runs at least once + raise last_exc async def _post_json( diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index db7fe8944..8b6467a40 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -281,6 +281,7 @@ async def poll_result(self) -> Optional[Dict[str, Any]]: """ url = f"{_proxy_url(self.pod_id)}/result" attempts = 6 + data: Dict[str, Any] = {} async with aiohttp.ClientSession() as session: for attempt in range(attempts): async with session.get( diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py index 8d41d1452..1ba19a774 100644 --- a/runpod/runtimes/bootstrap.py +++ b/runpod/runtimes/bootstrap.py @@ -117,7 +117,8 @@ def _locate(): # and absolute paths tar.extractall(path=APP_DIR, filter="data") else: - for member in tar.getmembers(): + members = tar.getmembers() + for member in members: dest = os.path.realpath( os.path.join(APP_DIR, member.name) ) @@ -125,7 +126,19 @@ def _locate(): raise PhaseError( "locate", f"unsafe tar member path: {member.name}" ) - tar.extractall(path=APP_DIR) + # links can point outside and later members write + # through them; artifacts never contain links + if member.issym() or member.islnk(): + raise PhaseError( + "locate", + f"unsafe tar link member: {member.name}", + ) + if not (member.isfile() or member.isdir()): + raise PhaseError( + "locate", + f"unsupported tar member type: {member.name}", + ) + tar.extractall(path=APP_DIR, members=members) # noqa: S202 - members validated above except (OSError, tarfile.TarError) as exc: raise PhaseError("locate", f"failed to extract artifact: {exc}") diff --git a/tests/test_apps/test_model.py b/tests/test_apps/test_model.py index 5a762d4e3..2aaaaf232 100644 --- a/tests/test_apps/test_model.py +++ b/tests/test_apps/test_model.py @@ -72,9 +72,12 @@ def test_task_decorator_rejects_model(self): app = App("modeltest") + # model= is not a parameter of @app.task; passed via a kwargs + # dict so the intentional misuse is constructed at runtime + kwargs = {"name": "train", "gpu": "4090", "model": Model("org/name")} with pytest.raises(TypeError, match="model"): - @app.task(name="train", gpu="4090", model=Model("org/name")) + @app.task(**kwargs) def train(): pass diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 7b05719dd..beacf36b7 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -472,27 +472,6 @@ async def test_execute_timeout(self): await execution.execute({"fn": "t"}, timeout=10) -class TestUnwrapTaskResponse: - def test_failure_raises(self): - from runpod.apps.errors import RemoteExecutionError - from runpod.apps.tasks import unwrap_task_response - - with pytest.raises(RemoteExecutionError, match="worker oom"): - unwrap_task_response({"success": False, "error": "worker oom"}) - - def test_json_result(self): - from runpod.apps.tasks import unwrap_task_response - - assert unwrap_task_response( - {"success": True, "json_result": {"x": 1}} - ) == {"x": 1} - - def test_pickled_result(self): - from runpod.apps.tasks import unwrap_task_response - - assert unwrap_task_response({"success": True, "result": _b64(7)}) == 7 - - class TestTaskJob: def _job(self): from runpod.apps.tasks import TaskExecution, TaskJob From 1e145e8ac285b410730b3bd47090441e1202db5a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 21:05:24 -0700 Subject: [PATCH 083/120] ci: skip duplicate runtime image builds on same-repo prs --- .github/workflows/runtimes.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index 7ef5e4c89..aa6147b10 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -12,6 +12,12 @@ on: - ".github/workflows/runtimes.yml" workflow_dispatch: +# a branch with an open pr triggers both push and pull_request for the +# same commit; superseded runs of the same flavor are cancelled +concurrency: + group: runtimes-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + env: REGISTRY: docker.io # the apps sdk branch publishes latest while the feature is prerelease: @@ -55,9 +61,15 @@ jobs: runs-on: ubuntu-latest # the torch layer is multi-GB; only rebuild when the gpu-base # definition itself changes. runtime images build FROM the - # registry tag, so the previously published base keeps working + # registry tag, so the previously published base keeps working. + # same-repo prs skip builds entirely: the push event on the branch + # already builds these commits (pr path filters diff against the + # base branch, which would force full rebuilds every time) needs: [changes, test-runner] - if: needs.changes.outputs.gpu-base == 'true' + if: >- + needs.changes.outputs.gpu-base == 'true' && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name != github.repository) strategy: # a transient registry/mirror failure on one leg must not cancel # the other python versions @@ -102,7 +114,10 @@ jobs: # failed gpu-base rebuild (e.g. torch mirror outage) must not block # runtime images: the previously published base keeps working needs: [test-runner, build-gpu-base] - if: always() && needs.test-runner.result == 'success' + if: >- + always() && needs.test-runner.result == 'success' && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name != github.repository) strategy: fail-fast: false matrix: From 43b5956db17d55c20e33ab7c120a9ead274fd4c4 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 21:08:33 -0700 Subject: [PATCH 084/120] ci: publish runtime images on release instead of per pr --- .github/workflows/runtimes.yml | 61 +++++++++++++++------------------- 1 file changed, 27 insertions(+), 34 deletions(-) diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml index aa6147b10..64deccd5a 100644 --- a/.github/workflows/runtimes.yml +++ b/.github/workflows/runtimes.yml @@ -1,38 +1,40 @@ name: Runtimes +# runtime images embed the runpod package, so they are published when a +# release ships (release-please merge -> github release), not per pr. +# the feature branch is a prerelease channel while the apps sdk is in +# development; runtime tests for prs run in ci via the full test suite. on: + release: + types: [published] push: - branches: [main, feat/apps-sdk] - paths: - - "runpod/runtimes/**" - - ".github/workflows/runtimes.yml" - pull_request: + branches: [feat/apps-sdk] paths: - "runpod/runtimes/**" - ".github/workflows/runtimes.yml" workflow_dispatch: -# a branch with an open pr triggers both push and pull_request for the -# same commit; superseded runs of the same flavor are cancelled concurrency: - group: runtimes-${{ github.event_name }}-${{ github.ref }} + group: runtimes-${{ github.ref }} cancel-in-progress: true env: REGISTRY: docker.io - # the apps sdk branch publishes latest while the feature is prerelease: - # the runtime image names are new, so nothing released depends on them - TAG_SUFFIX: ${{ (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/apps-sdk') && 'latest' || 'dev' }} + TAG_SUFFIX: latest jobs: changes: runs-on: ubuntu-latest outputs: - gpu-base: ${{ steps.filter.outputs.gpu-base }} + gpu-base: ${{ steps.filter.outputs.gpu-base || 'true' }} steps: - uses: actions/checkout@v7 + # branch pushes rebuild the multi-gb torch base only when its + # definition changed; releases and manual runs always rebuild so + # published images never lag the release content - uses: dorny/paths-filter@v3 + if: github.event_name == 'push' id: filter with: filters: | @@ -59,17 +61,8 @@ jobs: build-gpu-base: runs-on: ubuntu-latest - # the torch layer is multi-GB; only rebuild when the gpu-base - # definition itself changes. runtime images build FROM the - # registry tag, so the previously published base keeps working. - # same-repo prs skip builds entirely: the push event on the branch - # already builds these commits (pr path filters diff against the - # base branch, which would force full rebuilds every time) needs: [changes, test-runner] - if: >- - needs.changes.outputs.gpu-base == 'true' && - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository) + if: needs.changes.outputs.gpu-base == 'true' strategy: # a transient registry/mirror failure on one leg must not cancel # the other python versions @@ -87,7 +80,6 @@ jobs: - uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} @@ -102,7 +94,7 @@ jobs: platforms: linux/amd64 build-args: | PYTHON_VERSION=${{ matrix.python-version }} - push: ${{ github.event_name != 'pull_request' }} + push: true tags: | runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} cache-from: type=gha,scope=gpu-base-py${{ matrix.python-version }} @@ -114,10 +106,7 @@ jobs: # failed gpu-base rebuild (e.g. torch mirror outage) must not block # runtime images: the previously published base keeps working needs: [test-runner, build-gpu-base] - if: >- - always() && needs.test-runner.result == 'success' && - (github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository) + if: always() && needs.test-runner.result == 'success' strategy: fail-fast: false matrix: @@ -136,7 +125,6 @@ jobs: - uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} @@ -148,11 +136,17 @@ jobs: run: | if [ "${{ matrix.accelerator }}" = "gpu" ]; then echo "base=runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }}" >> "$GITHUB_OUTPUT" - echo "repo=runpod/${{ matrix.kind }}-gpu" >> "$GITHUB_OUTPUT" + repo="runpod/${{ matrix.kind }}-gpu" else echo "base=python:${{ matrix.python-version }}-slim" >> "$GITHUB_OUTPUT" - echo "repo=runpod/${{ matrix.kind }}" >> "$GITHUB_OUTPUT" + repo="runpod/${{ matrix.kind }}" fi + tags="$repo:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }}" + # releases also pin a version tag for rollback + if [ "${{ github.event_name }}" = "release" ]; then + tags="$tags,$repo:py${{ matrix.python-version }}-${{ github.event.release.tag_name }}" + fi + echo "tags=$tags" >> "$GITHUB_OUTPUT" - name: Build and push runtime image uses: docker/build-push-action@v6 @@ -162,8 +156,7 @@ jobs: platforms: linux/amd64 build-args: | BASE_IMAGE=${{ steps.names.outputs.base }} - push: ${{ github.event_name != 'pull_request' }} - tags: | - ${{ steps.names.outputs.repo }}:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} + push: true + tags: ${{ steps.names.outputs.tags }} cache-from: type=gha,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} cache-to: type=gha,mode=max,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} From 7c3cc21f331509fe2f7a49ddb1356d6ed64b2ff6 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:27:40 -0700 Subject: [PATCH 085/120] feat: job parity, streaming, and unified execution engine --- examples/apps/README.md | 1 + examples/apps/streaming.py | 42 ++++ runpod/__init__.py | 27 +- runpod/apps/__init__.py | 15 +- runpod/apps/handles.py | 120 +++++---- runpod/apps/invoker.py | 24 +- runpod/apps/job.py | 80 ++++++ runpod/apps/serialization.py | 19 ++ runpod/apps/stubs.py | 29 ++- runpod/apps/targets.py | 308 +++++++++++++++++------ runpod/apps/tasks.py | 13 +- runpod/runtimes/api/server.py | 6 +- runpod/runtimes/executor.py | 288 +++++++++++++++++++++ runpod/runtimes/queue/worker.py | 71 +++++- runpod/runtimes/task/runner.py | 240 +----------------- tests/e2e/examples/13_streaming.py | 56 +++++ tests/e2e/matrix/main.py | 14 ++ tests/e2e/matrix/run.py | 57 ++++- tests/test_apps/test_api_server.py | 4 +- tests/test_apps/test_dispatch.py | 84 ++++++- tests/test_apps/test_executor.py | 195 ++++++++++++++ tests/test_apps/test_job.py | 161 ++++++++++++ tests/test_apps/test_queue_worker.py | 91 ++++++- tests/test_apps/test_targets.py | 122 +++++++++ tests/test_apps/test_task_runner_http.py | 147 ++--------- tests/test_apps/test_tasks.py | 2 +- tests/test_init.py | 2 +- tests/test_shared/test_auth.py | 6 +- 28 files changed, 1686 insertions(+), 538 deletions(-) create mode 100644 examples/apps/streaming.py create mode 100644 runpod/apps/job.py create mode 100644 runpod/runtimes/executor.py create mode 100644 tests/e2e/examples/13_streaming.py create mode 100644 tests/test_apps/test_executor.py create mode 100644 tests/test_apps/test_job.py diff --git a/examples/apps/README.md b/examples/apps/README.md index 9abdbffa0..b8a381300 100644 --- a/examples/apps/README.md +++ b/examples/apps/README.md @@ -10,6 +10,7 @@ rp dev examples/apps/hello_world.py | example | shows | |---|---| | `hello_world` | one queue function, one `.remote()` call | +| `streaming` | generator functions, `.stream()` chunks, `job.stream()` | | `gpu_inference` | gpu selection, dependencies, a cuda benchmark | | `web_service` | `@app.api` class with routes and per-worker state | | `train_and_eval` | gpu tasks sharing checkpoints through a volume | diff --git a/examples/apps/streaming.py b/examples/apps/streaming.py new file mode 100644 index 000000000..35267bcff --- /dev/null +++ b/examples/apps/streaming.py @@ -0,0 +1,42 @@ +"""streaming: yield partial results from a queue function. + +A generator function decorated with @app.queue streams its output. +Calling .stream() yields each chunk as the worker produces it; +calling .remote() returns the full list of chunks at once. A spawned +job can be streamed later with job.stream(). + +Try it: + + rp dev examples/apps/streaming.py +""" + +import time + +import runpod +from runpod import App + +app = App("streaming") + + +@app.queue(cpu="cpu3c-1-2") +def tokens(prompt: str): + # stand-in for token-by-token llm output + for word in f"you said: {prompt}".split(): + time.sleep(0.2) + yield word + + +@runpod.local_entrypoint +def main(): + # chunks arrive as the worker yields them + for chunk in tokens.stream("hello streaming world"): + print(chunk, end=" ", flush=True) + print() + + # .remote() on a generator returns every chunk at once + all_chunks = tokens.remote("all at once") + print(all_chunks) + + # spawn now, stream later + job = tokens.spawn("stream me later") + print(list(job.stream())) diff --git a/runpod/__init__.py b/runpod/__init__.py index 296f0c659..5303284fa 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -1,4 +1,4 @@ -""" Allows runpod to be imported as a module. """ +"""Allows runpod to be imported as a module.""" import logging import os @@ -34,6 +34,7 @@ App, DataCenter, EndpointNotFound, + Job, Model, Queue, Secret, @@ -50,13 +51,13 @@ __all__ = [ # API functions "create_container_registry_auth", - "create_endpoint", + "create_endpoint", "create_pod", "create_template", "delete_container_registry_auth", "get_endpoints", "get_gpu", - "get_gpus", + "get_gpus", "get_pod", "get_pods", "get_user", @@ -68,7 +69,7 @@ "update_user_settings", # Config functions "check_credentials", - "get_credentials", + "get_credentials", "set_credentials", # Endpoint classes "AsyncioEndpoint", @@ -82,6 +83,7 @@ "EndpointNotFound", "GpuGroup", "GpuType", + "Job", "Queue", "delete", "get", @@ -101,8 +103,8 @@ # Module variables "SSH_KEY_PATH", "profile", - "api_key", - "endpoint_url_base" + "api_key", + "endpoint_url_base", ] # ------------------------------- Config Paths ------------------------------- # @@ -111,13 +113,14 @@ profile = "default" # pylint: disable=invalid-name -_credentials = get_credentials(profile) -if _credentials is not None: - api_key = _credentials["api_key"] # pylint: disable=invalid-name +if os.environ.get("RUNPOD_API_KEY"): + api_key = os.environ.get("RUNPOD_API_KEY") else: - # workers and ci have no config file; the env var is the standard - # way credentials reach them - api_key = os.environ.get("RUNPOD_API_KEY") # pylint: disable=invalid-name + _credentials = get_credentials(profile) + if _credentials is not None: + api_key = _credentials["api_key"] # pylint: disable=invalid-name + else: + api_key = None # pylint: disable=invalid-name endpoint_url_base = os.environ.get( "RUNPOD_ENDPOINT_BASE_URL", "https://api.runpod.ai/v2" diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index c179e482b..3651f574f 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -1,15 +1,4 @@ -"""app-centric sdk surface: App, decorated handles, and execution primitives. - -the public entry points here are: - -- App: the registry a project defines its resources on -- @app.queue: queue-based serverless endpoint from a function -- @app.api: load-balanced serverless endpoint from a class (or asgi factory) -- @app.task: ephemeral pod compute from a function -- runpod.local_entrypoint: dev-session entry hook -- runpod.is_local: context check -- runpod.Queue / runpod.Api: stubs for resources deployed from elsewhere -""" +"""app-centric sdk surface: App, decorated handles, and execution primitives.""" from .app import App, get_registered_apps from .context import Context, current_context, is_local @@ -22,6 +11,7 @@ ScheduleNotSupported, ) from .handles import ApiHandle, FunctionHandle +from .job import Job from .markers import delete, get, init, patch, post, put from .schedule import schedule from .spec import ResourceKind, ResourceSpec, RouteSpec @@ -39,6 +29,7 @@ "DataCenter", "EndpointNotFound", "FunctionHandle", + "Job", "Model", "Queue", "Secret", diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 768fb185e..bec331411 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -11,54 +11,37 @@ async def transcribe(audio_url: str): ... transcribe.spawn("https://...") # fire and forget -> job handle transcribe.local("https://...") # run the body here + for chunk in generate.stream(prompt="hi"): # generator functions + ... + @transcribe.init def load_model(): ... # worker startup hook """ import inspect -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Callable, + Dict, + List, + Optional, + Type, +) from .context import Context, current_context -from .invoker import Invoker -from .markers import INIT_ATTR, RESERVED_PATHS, is_init, route_of +from .errors import InvalidResourceError +from .invoker import Invoker, StreamInvoker +from .job import Job +from .markers import INIT_ATTR, is_init, route_of from .schedule import SCHEDULE_ATTR from .spec import ResourceKind, ResourceSpec, RouteSpec -from .errors import InvalidResourceError, RemoteExecutionError if TYPE_CHECKING: from .app import App -class Job: - """a submitted queue job, returned by .spawn(). - - `job.result()` blocks until the job finishes and returns its - output; `await job.result.aio()` is the async form. - """ - - def __init__(self, data: Dict[str, Any], handle: "FunctionHandle"): - self._data = data - self._handle = handle - from .invoker import Invoker - - self.result = Invoker(self._result_async) - - @property - def id(self) -> str: - return self._data.get("id", "") - - @property - def status(self) -> str: - return self._data.get("status", "UNKNOWN") - - async def _result_async(self, timeout: float = 300.0) -> Any: - target = await self._handle._app._resolve(self._handle.spec) - return await target.wait(self._data, timeout=timeout) - - def __repr__(self) -> str: - return f"Job(id={self.id!r}, status={self.status!r})" - - class FunctionHandle: """handle for @app.queue and @app.task functions.""" @@ -79,14 +62,14 @@ def __init__( spec.schedule = stamped self.remote = Invoker(self._remote_async) + self.stream = StreamInvoker(self._stream_async) self.spawn = Invoker(self._spawn_async) + self.job = Invoker(self._job_async) self.__name__ = getattr(fn, "__name__", spec.name) self.__doc__ = getattr(fn, "__doc__", None) self.__wrapped__ = fn - # -- lifecycle -- - def init(self, fn: Callable) -> Callable: """register a worker-startup hook; runs before the worker is ready, never locally.""" @@ -94,15 +77,11 @@ def init(self, fn: Callable) -> Callable: self._init_fn = fn return fn - # -- local execution -- - def local(self, *args: Any, **kwargs: Any) -> Any: """run the function body here. returns a coroutine iff the function is async.""" return self._fn(*args, **kwargs) - # -- remote execution -- - async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: self._guard_discovery() ctx = current_context() @@ -111,12 +90,55 @@ async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: result = self._fn(*args, **kwargs) if inspect.isawaitable(result): result = await result + # generators aggregate, matching the deployed worker's + # return_aggregate_stream output for .remote() + if inspect.isasyncgen(result): + return [chunk async for chunk in result] + if inspect.isgenerator(result): + return list(result) return result target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) return await target.invoke(payload) + async def _stream_async(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + """invoke a generator function remotely, yielding partial outputs + as the worker produces them.""" + self._guard_discovery() + self._guard_generator() + ctx = current_context() + + if ctx is Context.WORKER and self._is_current_worker(): + gen = self._fn(*args, **kwargs) + if inspect.isasyncgen(gen): + async for chunk in gen: + yield chunk + else: + for chunk in gen: + yield chunk + return + + target = await self._app._resolve(self.spec) + payload = target.build_payload(self._fn, self.spec, args, kwargs) + data = await target.submit(payload) + async for chunk in target.stream_job(data["id"]): + yield chunk + + def _guard_generator(self) -> None: + if self.spec.kind is ResourceKind.TASK: + raise InvalidResourceError( + "tasks do not stream; use @app.queue for generator functions" + ) + if not ( + inspect.isgeneratorfunction(self._fn) + or inspect.isasyncgenfunction(self._fn) + ): + raise InvalidResourceError( + f"'{self.__name__}' is not a generator function; " + f"use .remote(...) instead" + ) + async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: self._guard_discovery() target = await self._app._resolve(self.spec) @@ -124,9 +146,19 @@ async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: data = await target.submit(payload) # queue targets return raw job data; task targets return a TaskJob if isinstance(data, dict): - return Job(data, self) + return Job(data, target) return data + async def _job_async(self, job_id: str) -> Job: + """reconnect to a submitted queue job by id.""" + if self.spec.kind is ResourceKind.TASK: + raise InvalidResourceError( + "task jobs cannot be reconnected by id; retain the job " + "returned by .spawn()" + ) + target = await self._app._resolve(self.spec) + return Job({"id": job_id, "status": "UNKNOWN"}, target) + def _guard_discovery(self) -> None: from .discovery import DiscoveryInvocationError, in_discovery @@ -136,13 +168,9 @@ def _guard_discovery(self) -> None: def _is_current_worker(self) -> bool: import os - current = os.getenv("FLASH_RESOURCE_NAME") or os.getenv( - "RUNPOD_RESOURCE_NAME" - ) + current = os.getenv("FLASH_RESOURCE_NAME") or os.getenv("RUNPOD_RESOURCE_NAME") return current is not None and current == self.spec.name - # calling the handle directly is a common mistake; be explicit - def __call__(self, *args: Any, **kwargs: Any) -> Any: raise TypeError( f"'{self.__name__}' is a runpod {self.spec.kind.value} handle. " diff --git a/runpod/apps/invoker.py b/runpod/apps/invoker.py index 15251db9e..f44c6585d 100644 --- a/runpod/apps/invoker.py +++ b/runpod/apps/invoker.py @@ -5,13 +5,17 @@ handle.remote(x) # sync, blocks await handle.remote.aio(x) # async + +a StreamInvoker is the same idea for async generators: calling it +returns a sync iterator, `.aio(...)` returns the async iterator. """ -from typing import Any, Callable, Coroutine +from typing import Any, AsyncIterator, Callable, Coroutine, Iterator from .context import block CoroFactory = Callable[..., Coroutine[Any, Any, Any]] +AsyncGenFactory = Callable[..., AsyncIterator[Any]] class Invoker: @@ -25,3 +29,21 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any: def aio(self, *args: Any, **kwargs: Any) -> Coroutine[Any, Any, Any]: return self._factory(*args, **kwargs) + + +class StreamInvoker: + __slots__ = ("_factory",) + + def __init__(self, factory: AsyncGenFactory): + self._factory = factory + + def __call__(self, *args: Any, **kwargs: Any) -> Iterator[Any]: + agen = self._factory(*args, **kwargs) + while True: + try: + yield block(agen.__anext__()) + except StopAsyncIteration: + return + + def aio(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + return self._factory(*args, **kwargs) diff --git a/runpod/apps/job.py b/runpod/apps/job.py new file mode 100644 index 000000000..0d0dc24ca --- /dev/null +++ b/runpod/apps/job.py @@ -0,0 +1,80 @@ +"""submitted queue jobs shared by decorated handles and Queue stubs.""" + +from typing import Any, AsyncIterator, Dict, TYPE_CHECKING + +from .invoker import Invoker, StreamInvoker +from .targets import FINAL_STATUSES, DEFAULT_TIMEOUT_SECONDS + +if TYPE_CHECKING: + from .targets import InvocationTarget + + +class Job: + """a submitted queue job. + + job operations are sync by default and expose async forms through + ``.aio``: + + job.status() + await job.status.aio() + job.result() + job.cancel() + job.retry() + + generator jobs stream partial outputs: + + for chunk in job.stream(): ... + async for chunk in job.stream.aio(): ... + """ + + def __init__(self, data: Dict[str, Any], target: "InvocationTarget"): + self._data = dict(data) + self._target = target + self.status = Invoker(self._status_async) + self.result = Invoker(self._result_async) + self.output = self.result + self.cancel = Invoker(self._cancel_async) + self.retry = Invoker(self._retry_async) + self.stream = StreamInvoker(self._stream_async) + + @property + def id(self) -> str: + return self._data.get("id", "") + + @property + def done(self) -> bool: + return self._data.get("status", "UNKNOWN") in FINAL_STATUSES + + def _update(self, data: Dict[str, Any]) -> None: + self._data.update(data) + + async def _status_async(self) -> str: + # terminal statuses never change; skip the network round trip + if not self.done: + self._update(await self._target.job_status(self.id)) + return self._data.get("status", "UNKNOWN") + + async def _result_async(self, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> Any: + return await self._target.wait( + self._data, + timeout=timeout, + on_status=self._update, + ) + + async def _cancel_async(self) -> "Job": + self._update(await self._target.cancel_job(self.id)) + return self + + async def _retry_async(self) -> "Job": + self._update(await self._target.retry_job(self.id)) + return self + + async def _stream_async( + self, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> AsyncIterator[Any]: + async for chunk in self._target.stream_job(self.id, timeout=timeout): + yield chunk + + def __repr__(self) -> str: + status = self._data.get("status", "UNKNOWN") + return f"Job(id={self.id!r}, status={status!r})" diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py index 40bcae55e..684ccd719 100644 --- a/runpod/apps/serialization.py +++ b/runpod/apps/serialization.py @@ -41,6 +41,25 @@ def deserialize_result(result_b64: str) -> Any: ) from exc +def require_json_args(fn: Callable, args: tuple, kwargs: dict) -> None: + """reject arguments that cannot cross the queue data plane. + + queue jobs are json on the wire in every mode, so non-json + arguments fail at call time with a pointed error instead of a + worker-side decode failure. + """ + import json + + try: + json.dumps(list(args)) + json.dumps(kwargs) + except (TypeError, ValueError) as exc: + raise TypeError( + f"{fn.__name__}() arguments must be json-serializable " + f"(queue jobs are plain json): {exc}" + ) from exc + + def get_function_source(fn: Callable) -> str: """the full source of the function's module. diff --git a/runpod/apps/stubs.py b/runpod/apps/stubs.py index d2b9fe08b..c81b66923 100644 --- a/runpod/apps/stubs.py +++ b/runpod/apps/stubs.py @@ -14,11 +14,11 @@ """ import os -from typing import Any, Dict, Optional +from typing import Any, AsyncIterator, Optional -from .errors import EndpointNotFound, RemoteExecutionError -from .invoker import Invoker -from .targets import DEFAULT_TIMEOUT_SECONDS, SentinelTarget +from .invoker import Invoker, StreamInvoker +from .job import Job +from .targets import SentinelTarget DEFAULT_ENV = "default" @@ -52,15 +52,32 @@ class Queue(_StubBase): def __init__(self, **kwargs: Any): super().__init__(**kwargs) self.remote = Invoker(self._remote_async) + self.stream = StreamInvoker(self._stream_async) self.spawn = Invoker(self._spawn_async) + self.job = Invoker(self._job_async) async def _remote_async(self, **kwargs: Any) -> Any: payload = {"input": kwargs or {"__empty": True}} return await self._target().invoke(payload) - async def _spawn_async(self, **kwargs: Any) -> Dict[str, Any]: + async def _spawn_async(self, **kwargs: Any) -> Job: payload = {"input": kwargs or {"__empty": True}} - return await self._target().submit(payload) + target = self._target() + data = await target.submit(payload) + return Job(data, target) + + async def _stream_async(self, **kwargs: Any) -> AsyncIterator[Any]: + payload = {"input": kwargs or {"__empty": True}} + target = self._target() + data = await target.submit(payload) + async for chunk in target.stream_job(data["id"]): + yield chunk + + async def _job_async(self, job_id: str) -> Job: + return Job( + {"id": job_id, "status": "UNKNOWN"}, + self._target(), + ) def __repr__(self) -> str: ref = self.resource_name or self.resource_id diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 434b0feee..067201ba9 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -9,13 +9,13 @@ import json import os from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Optional +from typing import Any, AsyncIterator, Callable, Dict, Optional import aiohttp from ..user_agent import USER_AGENT from .errors import EndpointNotFound, RemoteExecutionError -from .protocol import FunctionRequest, FunctionResponse +from .protocol import FORMAT_JSON, FunctionRequest, FunctionResponse from .spec import ResourceSpec # the sentinel pseudo-endpoint id; ai-api resolves the real endpoint from @@ -120,8 +120,38 @@ async def invoke(self, payload: Dict[str, Any], *, timeout: float) -> Any: async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: """submit without waiting; returns the raw job data.""" + async def job_status(self, job_id: str) -> Dict[str, Any]: + """fetch the current state of a submitted job.""" + raise NotImplementedError( + f"{type(self).__name__} does not support job status" + ) + + async def cancel_job(self, job_id: str) -> Dict[str, Any]: + """cancel a submitted job.""" + raise NotImplementedError( + f"{type(self).__name__} does not support job cancellation" + ) + + async def retry_job(self, job_id: str) -> Dict[str, Any]: + """retry a failed or timed-out job.""" + raise NotImplementedError( + f"{type(self).__name__} does not support job retries" + ) + + def stream_job( + self, job_id: str, *, timeout: float + ) -> AsyncIterator[Any]: + """yield partial outputs of a generator job as they arrive.""" + raise NotImplementedError( + f"{type(self).__name__} does not support job streaming" + ) + async def wait( - self, job_data: Dict[str, Any], *, timeout: float + self, + job_data: Dict[str, Any], + *, + timeout: float, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: """wait for a submitted job to finish and return its output.""" raise NotImplementedError( @@ -254,6 +284,126 @@ async def _wait_terminal( return data +class QueueClient: + """queue data-plane client for a single endpoint id. + + owns the run/runsync/status/cancel/retry routes and the lb subdomain + for http resources. targets compose it with their own routing + headers: the sentinel client carries flash headers, a live client + carries plain auth headers. + """ + + def __init__( + self, + endpoint_id: str, + headers: Callable[[], Dict[str, str]], + *, + app_name: str = "", + resource_name: str = "", + ): + self.endpoint_id = endpoint_id + self._headers = headers + self._app_name = app_name + self._resource_name = resource_name + + @property + def base_url(self) -> str: + return f"{_endpoint_url_base()}/{self.endpoint_id}" + + async def _call( + self, + method: str, + path: str, + payload: Any = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Dict[str, Any]: + return await _request_json( + method, + f"{self.base_url}/{path}", + self._headers(), + timeout, + payload=payload, + app_name=self._app_name, + resource_name=self._resource_name, + ) + + async def run(self, payload: Dict[str, Any]) -> Dict[str, Any]: + return await self._call("POST", "run", payload) + + async def runsync( + self, payload: Dict[str, Any], *, timeout: float + ) -> Dict[str, Any]: + return await self._call("POST", "runsync", payload, timeout) + + async def status(self, job_id: str) -> Dict[str, Any]: + return await self._call("GET", f"status/{job_id}") + + async def cancel(self, job_id: str) -> Dict[str, Any]: + return await self._call("POST", f"cancel/{job_id}") + + async def retry(self, job_id: str) -> Dict[str, Any]: + return await self._call("POST", f"retry/{job_id}") + + async def stream( + self, job_id: str, *, timeout: float + ) -> AsyncIterator[Any]: + """yield partial outputs from /stream until the job is terminal. + + the route long-polls (wait=) and drains buffered chunks, so each + call returns quickly with whatever is available. + """ + import time + + deadline = time.monotonic() + timeout + while True: + if time.monotonic() >= deadline: + raise TimeoutError( + f"job {job_id} stream did not complete within {timeout}s" + ) + data = await self._call("GET", f"stream/{job_id}") + if data.get("status") == "FAILED" or data.get("error"): + unwrap_job_output(data) + for chunk in data.get("stream") or []: + yield chunk.get("output") + if data.get("status") in FINAL_STATUSES: + return + + async def wait( + self, + data: Dict[str, Any], + *, + timeout: float, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + return await _wait_terminal( + self.base_url, data, self._headers(), timeout, on_status + ) + + async def http( + self, + method: str, + path: str, + body: Any = None, + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + ) -> Any: + """plain http call through the endpoint's lb subdomain.""" + url = f"https://{self.endpoint_id}.{_lb_domain()}{path}" + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.request( + method, url, json=body, headers=self._headers() + ) as resp: + if resp.status == 404 and self._app_name: + raise EndpointNotFound(self._app_name, self._resource_name) + resp.raise_for_status() + text = await resp.text() + try: + return json.loads(text) + except json.JSONDecodeError: + return text + + class SentinelTarget(InvocationTarget): """routes through the sentinel endpoint; ai-api resolves the real endpoint server-side from app/env/resource headers. @@ -266,6 +416,12 @@ def __init__(self, app_name: str, env_name: str, resource_name: str): self.app_name = app_name self.env_name = env_name self.resource_name = resource_name + self._client = QueueClient( + SENTINEL_ID, + self._sentinel_headers, + app_name=app_name, + resource_name=resource_name, + ) def _sentinel_headers(self) -> Dict[str, str]: return _headers( @@ -279,41 +435,38 @@ def _sentinel_headers(self) -> Dict[str, str]: async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: - base = f"{_endpoint_url_base()}/{SENTINEL_ID}" - headers = self._sentinel_headers() - data = await _post_json( - f"{base}/runsync", - payload, - headers, - timeout, - app_name=self.app_name, - resource_name=self.resource_name, - ) - data = await _wait_terminal(base, data, headers, timeout) - return unwrap_job_output(data) + data = await self._client.runsync(payload, timeout=timeout) + data = await self._client.wait(data, timeout=timeout) + return self.unwrap(data) async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: - url = f"{_endpoint_url_base()}/{SENTINEL_ID}/run" - return await _post_json( - url, - payload, - self._sentinel_headers(), - DEFAULT_TIMEOUT_SECONDS, - app_name=self.app_name, - resource_name=self.resource_name, - ) + return await self._client.run(payload) + + async def job_status(self, job_id: str) -> Dict[str, Any]: + return await self._client.status(job_id) + + async def cancel_job(self, job_id: str) -> Dict[str, Any]: + return await self._client.cancel(job_id) + + async def retry_job(self, job_id: str) -> Dict[str, Any]: + return await self._client.retry(job_id) + + def stream_job( + self, job_id: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> AsyncIterator[Any]: + return self._client.stream(job_id, timeout=timeout) async def wait( self, job_data: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: - base = f"{_endpoint_url_base()}/{SENTINEL_ID}" - data = await _wait_terminal( - base, job_data, self._sentinel_headers(), timeout + data = await self._client.wait( + job_data, timeout=timeout, on_status=on_status ) - return unwrap_job_output(data) + return self.unwrap(data) async def request( self, @@ -323,20 +476,7 @@ async def request( *, timeout: float = DEFAULT_TIMEOUT_SECONDS, ) -> Any: - url = f"https://{SENTINEL_ID}.{_lb_domain()}{path}" - client_timeout = aiohttp.ClientTimeout(total=timeout) - async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.request( - method, url, json=body, headers=self._sentinel_headers() - ) as resp: - if resp.status == 404: - raise EndpointNotFound(self.app_name, self.resource_name) - resp.raise_for_status() - text = await resp.text() - try: - return json.loads(text) - except json.JSONDecodeError: - return text + return await self._client.http(method, path, body, timeout=timeout) class LiveTarget(InvocationTarget): @@ -360,6 +500,7 @@ def __init__( self.resource_name = resource_name self.events = events self.metrics_key = metrics_key + self._client = QueueClient(endpoint_id, _headers) self._source_target: Optional[Any] = None self._source_resource: str = "" self._source_spec: Optional[ResourceSpec] = None @@ -408,25 +549,40 @@ async def _sync_source(self, timeout: float) -> None: def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict ) -> Dict[str, Any]: - from .serialization import ( - get_function_source, - serialize_args, - serialize_kwargs, - ) + from .serialization import get_function_source, require_json_args + # queue arguments are plain json, the same contract deployed + # workers serve; non-json arguments fail here in dev exactly as + # they would against a deployed endpoint + require_json_args(fn, args, kwargs) request = FunctionRequest( function_name=fn.__name__, function_code=get_function_source(fn), - args=serialize_args(args), - kwargs=serialize_kwargs(kwargs), + args=list(args), + kwargs=dict(kwargs), dependencies=spec.dependencies, system_dependencies=spec.system_dependencies, accelerate_downloads=spec.accelerate_downloads, + serialization_format=FORMAT_JSON, ) return {"input": request.to_input()} def unwrap(self, data: Dict[str, Any]) -> Any: output = unwrap_job_output(data) + # the live handler is a generator handler, so job output is an + # aggregated list: one plain response for normal functions, + # __stream__-marked chunk envelopes for generator functions + if isinstance(output, list): + if ( + len(output) == 1 + and isinstance(output[0], dict) + and not output[0].get("__stream__") + ): + return self._unwrap_response(output[0]) + return [self._unwrap_response(o) for o in output] + return self._unwrap_response(output) + + def _unwrap_response(self, output: Any) -> Any: if isinstance(output, dict) and "success" in output: response = FunctionResponse.from_output(output) if not response.success: @@ -459,8 +615,6 @@ async def invoke( from .monitor import emit - base = f"{_endpoint_url_base()}/{self.endpoint_id}" - headers = _headers() monitor = self._monitor() emit(self.events, "dispatch", self.resource_name) start = time.monotonic() @@ -471,16 +625,12 @@ async def invoke( # async submit + status polling: runsync would hold the # connection, hiding the workerId until completion and # starving the live worker feed - data = await _post_json(f"{base}/run", payload, headers, timeout) + data = await self._client.run(payload) else: - data = await _post_json( - f"{base}/runsync", payload, headers, timeout - ) - data = await _wait_terminal( - base, + data = await self._client.runsync(payload, timeout=timeout) + data = await self._client.wait( data, - headers, - timeout, + timeout=timeout, on_status=monitor.on_status if monitor else None, ) result = self.unwrap(data) @@ -504,17 +654,35 @@ async def invoke( return result async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: - url = f"{_endpoint_url_base()}/{self.endpoint_id}/run" - return await _post_json(url, payload, _headers(), DEFAULT_TIMEOUT_SECONDS) + return await self._client.run(payload) + + async def job_status(self, job_id: str) -> Dict[str, Any]: + return await self._client.status(job_id) + + async def cancel_job(self, job_id: str) -> Dict[str, Any]: + return await self._client.cancel(job_id) + + async def retry_job(self, job_id: str) -> Dict[str, Any]: + return await self._client.retry(job_id) + + async def stream_job( + self, job_id: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS + ) -> AsyncIterator[Any]: + # live workers stream serialized chunk envelopes through the + # same /stream route deployed workers use + async for chunk in self._client.stream(job_id, timeout=timeout): + yield self._unwrap_response(chunk) async def wait( self, job_data: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: - base = f"{_endpoint_url_base()}/{self.endpoint_id}" - data = await _wait_terminal(base, job_data, _headers(), timeout) + data = await self._client.wait( + job_data, timeout=timeout, on_status=on_status + ) return self.unwrap(data) async def request( @@ -531,16 +699,9 @@ async def request( emit(self.events, "dispatch", self.resource_name, f"{method} {path}") start = time.monotonic() - url = f"https://{self.endpoint_id}.{_lb_domain()}{path}" - client_timeout = aiohttp.ClientTimeout(total=timeout) try: await self._sync_source(timeout) - async with aiohttp.ClientSession(timeout=client_timeout) as session: - async with session.request( - method, url, json=body, headers=_headers() - ) as resp: - resp.raise_for_status() - text = await resp.text() + result = await self._client.http(method, path, body, timeout=timeout) except Exception: emit( self.events, @@ -555,10 +716,7 @@ async def request( self.resource_name, time.monotonic() - start, ) - try: - return json.loads(text) - except json.JSONDecodeError: - return text + return result class PodTarget(InvocationTarget): diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 8b6467a40..7ddcd1cde 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -43,9 +43,16 @@ def _runner_source() -> str: - return ( - Path(__file__).parent.parent / "runtimes" / "task" / "runner.py" - ).read_text() + """single-file runner for bare-image bootstrap. + + the shared executor is concatenated ahead of the task server so the + shipped file has no runpod imports; the server's executor import + fails on the pod and falls through to the names already in scope. + """ + runtimes = Path(__file__).parent.parent / "runtimes" + executor = (runtimes / "executor.py").read_text() + server = (runtimes / "task" / "runner.py").read_text() + return f"{executor}\n\n{server}" def _bootstrap_command() -> str: diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py index 8637c06d5..534648d89 100644 --- a/runpod/runtimes/api/server.py +++ b/runpod/runtimes/api/server.py @@ -171,7 +171,7 @@ def _install_live_dependencies(request: dict) -> None: dependencies baked into the build artifact, so live mode installs them at sync time to match. """ - from runpod.runtimes.task.runner import _install, _install_system + from runpod.runtimes.executor import _install, _install_system error = _install_system(request.get("system_dependencies")) if error: @@ -190,7 +190,7 @@ async def _materialize_live_api(source: str, resource: str) -> Any: from fastapi import FastAPI from runpod.apps.handles import ApiHandle - from runpod.runtimes.task.runner import _materialize_source + from runpod.runtimes.executor import _materialize_source path = _materialize_source(source) namespace: dict = {"__name__": "__runpod_live__", "__file__": path} @@ -242,7 +242,7 @@ async def ping(): @app.post("/execute") async def execute(request: dict): - from runpod.runtimes.task.runner import execute_request + from runpod.runtimes.executor import execute_request return execute_request(request.get("input", request)) diff --git a/runpod/runtimes/executor.py b/runpod/runtimes/executor.py new file mode 100644 index 000000000..1e2b50a1a --- /dev/null +++ b/runpod/runtimes/executor.py @@ -0,0 +1,288 @@ +"""shared FunctionRequest execution engine. + +runs one function from shipped source: installs dependencies, execs the +module, resolves the function, and serializes the result per the +request's serialization_format. every runtime that executes live +requests (task pods, queue workers in dev mode, api servers) drives +this same engine. + +stdlib-only (cloudpickle optional, required only for the cloudpickle +serialization format) so it can ship as part of a single-file bootstrap +onto any python image. +""" + +import base64 +import inspect +import io +import json +import os +import shutil +import subprocess +import sys +import traceback +from contextlib import redirect_stdout + + +def _load_cloudpickle(install: bool = False): + try: + import cloudpickle + + return cloudpickle + except ImportError: + if not install: + return None + # bare images (custom image= without the runtime baked in) may lack + # cloudpickle; install it on demand the same way dependencies are + if _install(["cloudpickle"], "cloudpickle"): + return None + import cloudpickle + + return cloudpickle + + +def _install(packages, label): + if not packages: + return None + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", *packages], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return f"failed to install {label}: {result.stderr[-2000:]}" + return None + + +_apt_updated = False + + +def _install_system(packages): + """install apt packages; requires a debian-family image with root.""" + global _apt_updated + if not packages: + return None + if shutil.which("apt-get") is None: + return ( + f"system dependencies {packages} requested but apt-get is not " + f"available in this image; use a debian-based image or bake " + f"them in" + ) + env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") + if not _apt_updated: + update = subprocess.run( + ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env + ) + if update.returncode != 0: + return f"apt-get update failed: {update.stderr[-2000:]}" + _apt_updated = True + result = subprocess.run( + ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], + capture_output=True, + text=True, + env=env, + ) + if result.returncode != 0: + return ( + f"failed to install system dependencies {packages}: " + f"{result.stderr[-2000:]}" + ) + return None + + +def _deserialize_args(request): + fmt = request.get("serialization_format", "cloudpickle") + args = request.get("args") or [] + kwargs = request.get("kwargs") or {} + if fmt == "json": + return list(args), dict(kwargs) + cloudpickle = _load_cloudpickle(install=True) + if cloudpickle is None: + raise RuntimeError("cloudpickle not available for argument deserialization") + args = [cloudpickle.loads(base64.b64decode(a)) for a in args] + kwargs = {k: cloudpickle.loads(base64.b64decode(v)) for k, v in kwargs.items()} + return args, kwargs + + +def _serialize_result(result, fmt): + if fmt == "json": + # json-format requests promise json on the wire; failing here + # keeps dev behavior identical to a deployed endpoint + try: + json.dumps(result) + except (TypeError, ValueError) as exc: + raise TypeError( + f"return value must be json-serializable " + f"(this request uses the json wire format): {exc}" + ) from exc + return {"json_result": result} + cloudpickle = _load_cloudpickle(install=True) + if cloudpickle is None: + return {"json_result": result} + return {"result": base64.b64encode(cloudpickle.dumps(result)).decode("utf-8")} + + +def resolve_request(request): + """prepare one FunctionRequest for execution. + + installs dependencies, execs the shipped source, and resolves the + target function. returns ((fn, args, kwargs), None) on success or + (None, response_dict) on failure. + """ + error = _install_system(request.get("system_dependencies")) + if error: + return None, {"success": False, "error": error} + error = _install(request.get("dependencies"), "dependencies") + if error: + return None, {"success": False, "error": error} + + function_name = request.get("function_name") + function_code = request.get("function_code") + if not function_name or not function_code: + return None, { + "success": False, + "error": "function_name and function_code are required", + } + + # exec mirrors deployed-mode module import: the code is the + # user's full module, so __name__ is set like an import would + # (main guards stay inert) and decorated handles resolve to + # their wrapped functions. the source is written to a real + # file first so inspect.getsource works inside the function + # (nested .remote() calls re-extract sibling source) + source_path = _materialize_source(function_code) + namespace = {"__name__": "__runpod_live__", "__file__": source_path} + code_obj = compile(function_code, source_path, "exec") + exec(code_obj, namespace) # noqa: S102 - that is the job + if function_name not in namespace: + return None, { + "success": False, + "error": f"function '{function_name}' not found in provided code", + } + fn = namespace[function_name] + fn = getattr(fn, "_fn", fn) + + args, kwargs = _deserialize_args(request) + return (fn, args, kwargs), None + + +def serialize_chunk(chunk, request): + """wrap one generator chunk in the response envelope. + + the __stream__ marker lets clients tell a stream of chunks apart + from a single aggregated function response. + """ + response = {"success": True, "__stream__": True} + response.update( + _serialize_result( + chunk, request.get("serialization_format", "cloudpickle") + ) + ) + return response + + +def execute_request(request): + """run one FunctionRequest to completion, returning a response dict.""" + stdout_io = io.StringIO() + try: + prepared, error_response = resolve_request(request) + if error_response is not None: + return error_response + fn, args, kwargs = prepared + + # stdout is teed: captured for the job response and written + # through to the real stdout so container logs (and live log + # streaming in dev sessions) carry the function's prints. + # sys.stderr carries the runner's own request logs from other + # threads, which must not leak into job output + with redirect_stdout(_Tee(stdout_io, sys.__stdout__)): + result = fn(*args, **kwargs) + if hasattr(result, "__await__"): + result = _run_awaitable(result) + # generators aggregate so live .remote() matches the + # deployed worker's return_aggregate_stream output + elif inspect.isgenerator(result): + result = list(result) + elif inspect.isasyncgen(result): + result = _run_awaitable(_drain_async_gen(result)) + + response = {"success": True, "stdout": stdout_io.getvalue()} + response.update( + _serialize_result( + result, request.get("serialization_format", "cloudpickle") + ) + ) + return response + except Exception: # noqa: BLE001 - all failures go over the wire + return { + "success": False, + "error": traceback.format_exc(), + "stdout": stdout_io.getvalue(), + } + + +def _materialize_source(function_code): + """persist request source to a stable path for inspect/linecache.""" + import hashlib + import tempfile + + digest = hashlib.sha256(function_code.encode()).hexdigest()[:16] + path = os.path.join( + tempfile.gettempdir(), f"runpod_live_{digest}.py" + ) + if not os.path.exists(path): + with open(path, "w") as f: + f.write(function_code) + return path + + +class _Tee(io.TextIOBase): + """write-through to multiple streams, flushing eagerly so log + followers see lines as they happen.""" + + def __init__(self, *streams): + self._streams = [s for s in streams if s is not None] + + def write(self, s): + for stream in self._streams: + try: + stream.write(s) + stream.flush() + except (ValueError, OSError): + pass + return len(s) + + def flush(self): + for stream in self._streams: + try: + stream.flush() + except (ValueError, OSError): + pass + + +def _run_awaitable(awaitable): + """drive an awaitable to completion from sync code. + + task pods run this on plain threads, but queue workers in live mode + call execute_request from inside the serverless event loop, where + asyncio.run would raise; a private loop on a helper thread covers + both. + """ + import asyncio + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(_await(awaitable)) + + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, _await(awaitable)).result() + + +async def _drain_async_gen(agen): + return [chunk async for chunk in agen] + + +async def _await(awaitable): + return await awaitable diff --git a/runpod/runtimes/queue/worker.py b/runpod/runtimes/queue/worker.py index 512b859d6..72f2857be 100644 --- a/runpod/runtimes/queue/worker.py +++ b/runpod/runtimes/queue/worker.py @@ -12,8 +12,8 @@ live mode (rp dev): no artifact. each job carries a FunctionRequest (source + - serialized args) which is executed via the task runner's - execute_request. + serialized args) which is executed via the shared engine in + runpod.runtimes.executor. """ import importlib @@ -75,14 +75,37 @@ def _load_deployed_handle(): ) +def _job_kwargs(job: dict) -> dict: + body = dict(job.get("input") or {}) + body.pop("__empty", None) + return body + + def _make_deployed_handler(handle): - """serverless handler that calls the user function directly.""" + """serverless handler that calls the user function directly. + + generator functions become generator handlers so the serverless + core streams partial outputs to /stream as they are yielded. + """ fn = handle._fn + if inspect.isasyncgenfunction(fn): + + async def async_gen_handler(job: dict): + async for chunk in fn(**_job_kwargs(job)): + yield chunk + + return async_gen_handler + + if inspect.isgeneratorfunction(fn): + + def gen_handler(job: dict): + yield from fn(**_job_kwargs(job)) + + return gen_handler + async def handler(job: dict) -> dict: - body = dict(job.get("input") or {}) - body.pop("__empty", None) - result = fn(**body) + result = fn(**_job_kwargs(job)) if inspect.isawaitable(result): result = await result return result @@ -102,10 +125,32 @@ def _run_init(handle) -> None: asyncio.run(result) -def _live_handler(job: dict) -> dict: - from runpod.runtimes.task.runner import execute_request +async def _live_handler(job: dict): + """execute a FunctionRequest, streaming when the function is a + generator so dev sessions behave exactly like deployed workers.""" + from runpod.runtimes.executor import ( + execute_request, + resolve_request, + serialize_chunk, + ) + + request = job.get("input") or {} + prepared, error_response = resolve_request(request) + if error_response is not None: + yield error_response + return + fn, args, kwargs = prepared + + if inspect.isasyncgenfunction(fn): + async for chunk in fn(*args, **kwargs): + yield serialize_chunk(chunk, request) + return + if inspect.isgeneratorfunction(fn): + for chunk in fn(*args, **kwargs): + yield serialize_chunk(chunk, request) + return - return execute_request(job.get("input") or {}) + yield execute_request(request) def _max_concurrency() -> int: @@ -118,6 +163,12 @@ def _max_concurrency() -> int: def _worker_config(handler) -> dict: config: dict = {"handler": handler} + from runpod.serverless.modules.rp_handler import is_generator + + if is_generator(handler): + # generator jobs stream partials and still finish with a full + # output, so .remote()/result() work alongside .stream() + config["return_aggregate_stream"] = True concurrency = _max_concurrency() if concurrency > 1: config["concurrency_modifier"] = lambda _current: concurrency @@ -132,6 +183,8 @@ def main() -> None: _worker_config(_make_deployed_handler(handle)) ) else: + # the live handler is a generator so it can stream; aggregation + # keeps .remote() working for plain functions and generators alike runpod.serverless.start(_worker_config(_live_handler)) diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index 4e70388dd..b0694890b 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -1,10 +1,8 @@ """single-shot task runner: the process a task pod boots into. a minimal http server speaking the FunctionRequest/FunctionResponse -protocol. stdlib-only (cloudpickle optional, required only for the -cloudpickle serialization format) so it can bootstrap onto any python -image via an env-var payload, and be pre-baked into the task runtime -image later. +protocol, driving the shared execution engine in +runpod.runtimes.executor. endpoints: GET /ping readiness probe (unauthenticated) @@ -17,20 +15,25 @@ one pod runs one function; the client terminates the pod after collecting the result. + +ships to bare images as a single file: the sdk concatenates the +executor source above this module, so the import below is satisfied +either way. """ -import base64 -import io import json import os -import shutil -import subprocess import sys import threading -import traceback -from contextlib import redirect_stdout from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +try: + from runpod.runtimes.executor import execute_request +except ImportError: + # single-file bootstrap: the executor source is concatenated above + # this module and its names are already in scope + pass + PORT = int(os.environ.get("RUNPOD_TASK_PORT", "8080")) TOKEN = os.environ.get("RUNPOD_TASK_TOKEN", "") @@ -118,223 +121,6 @@ def _watchdog(): _terminate_self() -def _load_cloudpickle(install: bool = False): - try: - import cloudpickle - - return cloudpickle - except ImportError: - if not install: - return None - # bare images (custom image= without the runtime baked in) may lack - # cloudpickle; install it on demand the same way dependencies are - if _install(["cloudpickle"], "cloudpickle"): - return None - import cloudpickle - - return cloudpickle - - -def _install(packages, label): - if not packages: - return None - result = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", *packages], - capture_output=True, - text=True, - ) - if result.returncode != 0: - return f"failed to install {label}: {result.stderr[-2000:]}" - return None - - -_apt_updated = False - - -def _install_system(packages): - """install apt packages; requires a debian-family image with root.""" - global _apt_updated - if not packages: - return None - if shutil.which("apt-get") is None: - return ( - f"system dependencies {packages} requested but apt-get is not " - f"available in this image; use a debian-based image or bake " - f"them in" - ) - env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") - if not _apt_updated: - update = subprocess.run( - ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env - ) - if update.returncode != 0: - return f"apt-get update failed: {update.stderr[-2000:]}" - _apt_updated = True - result = subprocess.run( - ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], - capture_output=True, - text=True, - env=env, - ) - if result.returncode != 0: - return ( - f"failed to install system dependencies {packages}: " - f"{result.stderr[-2000:]}" - ) - return None - - -def _deserialize_args(request): - fmt = request.get("serialization_format", "cloudpickle") - args = request.get("args") or [] - kwargs = request.get("kwargs") or {} - if fmt == "json": - return list(args), dict(kwargs) - cloudpickle = _load_cloudpickle(install=True) - if cloudpickle is None: - raise RuntimeError("cloudpickle not available for argument deserialization") - args = [cloudpickle.loads(base64.b64decode(a)) for a in args] - kwargs = {k: cloudpickle.loads(base64.b64decode(v)) for k, v in kwargs.items()} - return args, kwargs - - -def _serialize_result(result, fmt): - if fmt == "json": - return {"json_result": result} - cloudpickle = _load_cloudpickle(install=True) - if cloudpickle is None: - return {"json_result": result} - return {"result": base64.b64encode(cloudpickle.dumps(result)).decode("utf-8")} - - -def execute_request(request): - """run one FunctionRequest to completion, returning a response dict.""" - stdout_io = io.StringIO() - try: - error = _install_system(request.get("system_dependencies")) - if error: - return {"success": False, "error": error} - error = _install(request.get("dependencies"), "dependencies") - if error: - return {"success": False, "error": error} - - function_name = request.get("function_name") - function_code = request.get("function_code") - if not function_name or not function_code: - return { - "success": False, - "error": "function_name and function_code are required", - } - - # exec mirrors deployed-mode module import: the code is the - # user's full module, so __name__ is set like an import would - # (main guards stay inert) and decorated handles resolve to - # their wrapped functions. the source is written to a real - # file first so inspect.getsource works inside the function - # (nested .remote() calls re-extract sibling source) - source_path = _materialize_source(function_code) - namespace = {"__name__": "__runpod_live__", "__file__": source_path} - code_obj = compile(function_code, source_path, "exec") - exec(code_obj, namespace) # noqa: S102 - that is the job - if function_name not in namespace: - return { - "success": False, - "error": f"function '{function_name}' not found in provided code", - } - fn = namespace[function_name] - fn = getattr(fn, "_fn", fn) - - args, kwargs = _deserialize_args(request) - - # stdout is teed: captured for the job response and written - # through to the real stdout so container logs (and live log - # streaming in dev sessions) carry the function's prints. - # sys.stderr carries the runner's own request logs from other - # threads, which must not leak into job output - with redirect_stdout(_Tee(stdout_io, sys.__stdout__)): - result = fn(*args, **kwargs) - if hasattr(result, "__await__"): - result = _run_awaitable(result) - - response = {"success": True, "stdout": stdout_io.getvalue()} - response.update( - _serialize_result( - result, request.get("serialization_format", "cloudpickle") - ) - ) - return response - except Exception: # noqa: BLE001 - all failures go over the wire - return { - "success": False, - "error": traceback.format_exc(), - "stdout": stdout_io.getvalue(), - } - - -def _materialize_source(function_code): - """persist request source to a stable path for inspect/linecache.""" - import hashlib - import tempfile - - digest = hashlib.sha256(function_code.encode()).hexdigest()[:16] - path = os.path.join( - tempfile.gettempdir(), f"runpod_live_{digest}.py" - ) - if not os.path.exists(path): - with open(path, "w") as f: - f.write(function_code) - return path - - -class _Tee(io.TextIOBase): - """write-through to multiple streams, flushing eagerly so log - followers see lines as they happen.""" - - def __init__(self, *streams): - self._streams = [s for s in streams if s is not None] - - def write(self, s): - for stream in self._streams: - try: - stream.write(s) - stream.flush() - except (ValueError, OSError): - pass - return len(s) - - def flush(self): - for stream in self._streams: - try: - stream.flush() - except (ValueError, OSError): - pass - - -def _run_awaitable(awaitable): - """drive an awaitable to completion from sync code. - - the task runner runs this on plain threads, but the queue worker's - live mode calls execute_request from inside the serverless event - loop, where asyncio.run would raise; a private loop on a helper - thread covers both. - """ - import asyncio - - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(_await(awaitable)) - - import concurrent.futures - - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - return pool.submit(asyncio.run, _await(awaitable)).result() - - -async def _await(awaitable): - return await awaitable - - class Handler(BaseHTTPRequestHandler): def _send(self, code, payload): body = json.dumps(payload).encode("utf-8") diff --git a/tests/e2e/examples/13_streaming.py b/tests/e2e/examples/13_streaming.py new file mode 100644 index 000000000..a2cb52157 --- /dev/null +++ b/tests/e2e/examples/13_streaming.py @@ -0,0 +1,56 @@ +"""streaming permutations: sync/async generators, every consumption style. + + rp dev tests/e2e/examples/13_streaming.py --once +""" + +import asyncio + +import runpod +from runpod import App + +app = App("ex-stream") + + +@app.queue(cpu="cpu3c-1-2") +def count_up(n: int): + for i in range(n): + yield {"i": i} + + +@app.queue(cpu="cpu3c-1-2") +async def count_down(n: int): + for i in range(n, 0, -1): + yield i + + +@runpod.local_entrypoint +def main(): + # sync iteration over a sync generator + chunks = list(count_up.stream(3)) + assert chunks == [{"i": 0}, {"i": 1}, {"i": 2}], chunks + + # async iteration over an async generator + async def consume(): + return [c async for c in count_down.stream.aio(3)] + + chunks = asyncio.run(consume()) + assert chunks == [3, 2, 1], chunks + + # .remote() on a generator aggregates every chunk + all_chunks = count_up.remote(2) + assert all_chunks == [{"i": 0}, {"i": 1}], all_chunks + + # spawn -> reconnect-style streaming from the job handle + job = count_up.spawn(2) + chunks = list(job.stream()) + assert chunks == [{"i": 0}, {"i": 1}], chunks + assert job.result() == [{"i": 0}, {"i": 1}] + + # .local() still yields directly, no cloud involved + assert list(count_up.local(2)) == [{"i": 0}, {"i": 1}] + + print("all streaming styles ok") + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/matrix/main.py b/tests/e2e/matrix/main.py index aa4463297..a82298026 100644 --- a/tests/e2e/matrix/main.py +++ b/tests/e2e/matrix/main.py @@ -59,6 +59,20 @@ def q_gpu(): } +# -- streaming --------------------------------------------------------- + +@app.queue(name="q-stream", cpu="cpu3c-1-2", workers=(0, 1)) +def q_stream(n: int): + for i in range(n): + yield {"i": i} + + +@app.queue(name="q-stream-async", cpu="cpu3c-1-2", workers=(0, 1)) +async def q_stream_async(n: int): + for i in range(n, 0, -1): + yield i + + # -- cross-calls ------------------------------------------------------- @app.queue(name="q-caller", cpu="cpu3c-2-4", workers=(0, 1)) diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py index fc4ce8356..71e8bb795 100644 --- a/tests/e2e/matrix/run.py +++ b/tests/e2e/matrix/run.py @@ -12,7 +12,19 @@ import time import traceback -from main import Svc, app, q_basic, q_caller, q_custom, q_deps, q_gpu, t_gpu, t_mul # noqa: E402 +from main import ( # noqa: E402 + Svc, + app, + q_basic, + q_caller, + q_custom, + q_deps, + q_gpu, + q_stream, + q_stream_async, + t_gpu, + t_mul, +) async def check_q_basic(): @@ -74,6 +86,44 @@ async def check_spawn(): return r +async def check_job_ops(): + job = await q_basic.spawn.aio(5) + status = await job.status.aio() + assert status, status + r = await job.result.aio() + assert r == {"doubled": 10}, r + # reconnect by id and read the terminal status + reconnected = await q_basic.job.aio(job.id) + status = await reconnected.status.aio() + assert status == "COMPLETED", status + return {"id": job.id, "status": status} + + +async def check_stream(): + chunks = [c async for c in q_stream.stream.aio(3)] + assert chunks == [{"i": 0}, {"i": 1}, {"i": 2}], chunks + return chunks + + +async def check_stream_async_gen(): + chunks = [c async for c in q_stream_async.stream.aio(3)] + assert chunks == [3, 2, 1], chunks + return chunks + + +async def check_stream_aggregate(): + r = await q_stream.remote.aio(2) + assert r == [{"i": 0}, {"i": 1}], r + return r + + +async def check_stream_from_job(): + job = await q_stream.spawn.aio(2) + chunks = [c async for c in job.stream.aio()] + assert chunks == [{"i": 0}, {"i": 1}], chunks + return chunks + + CHECKS = { "q-basic": check_q_basic, "q-deps": check_q_deps, @@ -84,6 +134,11 @@ async def check_spawn(): "t-gpu": check_t_gpu, "api": check_api, "spawn": check_spawn, + "job-ops": check_job_ops, + "stream": check_stream, + "stream-async-gen": check_stream_async_gen, + "stream-aggregate": check_stream_aggregate, + "stream-from-job": check_stream_from_job, } diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py index 054c2359d..10804879c 100644 --- a/tests/test_apps/test_api_server.py +++ b/tests/test_apps/test_api_server.py @@ -218,10 +218,10 @@ def test_sync_installs_dependencies_before_materializing(self): client = TestClient(_build_live_app()) with ( patch( - "runpod.runtimes.task.runner._install", return_value=None + "runpod.runtimes.executor._install", return_value=None ) as install, patch( - "runpod.runtimes.task.runner._install_system", + "runpod.runtimes.executor._install_system", return_value=None, ) as install_system, ): diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index 2ed21b6a5..19c0f82af 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -7,7 +7,11 @@ import runpod from runpod.apps import App, Context, current_context, is_local from runpod.apps.app import _clear_registry -from runpod.apps.errors import EndpointNotFound, RemoteExecutionError +from runpod.apps.errors import ( + EndpointNotFound, + InvalidResourceError, + RemoteExecutionError, +) from runpod.apps.targets import ( SentinelTarget, args_to_input, @@ -181,8 +185,44 @@ def q(x): mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} job = q.spawn(1) + assert isinstance(job, runpod.Job) assert job.id == "job-1" - assert job.status == "IN_QUEUE" + + def test_stream_generator_function(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="gen") + def gen(n): + yield from range(n) + + async def fake_stream(self, job_id, *, timeout=300.0): + for chunk in ("x", "y"): + yield chunk + + with ( + patch.object( + SentinelTarget, "submit", new_callable=AsyncMock + ) as mock_submit, + patch.object(SentinelTarget, "stream_job", fake_stream), + ): + mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} + chunks = list(gen.stream(2)) + + assert chunks == ["x", "y"] + + def test_stream_rejects_non_generator(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="plain") + def plain(x): + return x + + with pytest.raises(InvalidResourceError, match="not a generator"): + list(plain.stream(1)) class TestStubs: @@ -203,6 +243,42 @@ def test_queue_stub_remote(self): assert result == {"ok": 1} mock_invoke.assert_awaited_once_with({"input": {"prompt": "hi"}}) + def test_queue_stub_spawn_returns_job(self): + stub = runpod.Queue(app="other-app", name="q") + with patch.object( + SentinelTarget, "submit", new_callable=AsyncMock + ) as mock_submit: + mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} + job = stub.spawn(prompt="hi") + + assert isinstance(job, runpod.Job) + assert job.id == "job-1" + + def test_queue_stub_stream(self): + stub = runpod.Queue(app="other-app", name="q") + + async def fake_stream(self, job_id, *, timeout=300.0): + for chunk in ("x", "y"): + yield chunk + + with ( + patch.object( + SentinelTarget, "submit", new_callable=AsyncMock + ) as mock_submit, + patch.object(SentinelTarget, "stream_job", fake_stream), + ): + mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} + chunks = list(stub.stream(prompt="hi")) + + assert chunks == ["x", "y"] + + def test_queue_stub_reconnects_to_job(self): + stub = runpod.Queue(app="other-app", name="q") + job = stub.job("job-1") + + assert isinstance(job, runpod.Job) + assert job.id == "job-1" + def test_api_stub_http(self): stub = runpod.Api(app="other-app", name="api") with patch.object( @@ -284,7 +360,7 @@ def test_whole_module_ships(self, tmp_path): def test_execute_request_unwraps_handles(self): # a shipped module defines decorated handles; the runner must # call the wrapped function, not the handle - from runpod.runtimes.task.runner import execute_request + from runpod.runtimes.executor import execute_request code = ( "import runpod\n" @@ -309,7 +385,7 @@ def test_nested_source_extraction_from_shipped_module(self): # inside a live worker, a function calling sibling.remote() # re-extracts that sibling's source via inspect; the shipped # module must be inspectable after exec - from runpod.runtimes.task.runner import execute_request + from runpod.runtimes.executor import execute_request code = ( "import runpod\n" diff --git a/tests/test_apps/test_executor.py b/tests/test_apps/test_executor.py new file mode 100644 index 000000000..01205e702 --- /dev/null +++ b/tests/test_apps/test_executor.py @@ -0,0 +1,195 @@ +"""unit tests for the shared FunctionRequest execution engine.""" + +import pytest + +from runpod.runtimes import executor + + +def test_execute_roundtrip_json(): + response = executor.execute_request( + { + "function_name": "mul", + "function_code": "def mul(a, b):\n return a * b", + "args": [6, 7], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"] is True + assert response["json_result"] == 42 + + +def test_json_result_rejects_non_json_return(): + with pytest.raises(TypeError, match="json-serializable"): + executor._serialize_result(object(), "json") + + +def test_serialize_chunk_marks_stream(): + chunk = executor.serialize_chunk("tok", {"serialization_format": "json"}) + assert chunk == {"success": True, "__stream__": True, "json_result": "tok"} + + +def test_resolve_request_returns_callable(): + prepared, error = executor.resolve_request( + { + "function_name": "double", + "function_code": "def double(x):\n return x * 2", + "kwargs": {}, + "args": [], + "serialization_format": "json", + } + ) + assert error is None + fn, args, kwargs = prepared + assert fn(4) == 8 + + +def test_resolve_request_missing_function(): + prepared, error = executor.resolve_request( + {"function_name": "nope", "function_code": "x = 1"} + ) + assert prepared is None + assert error["success"] is False + assert "not found" in error["error"] + + +def test_generator_results_aggregate(): + response = executor.execute_request( + { + "function_name": "gen", + "function_code": "def gen(n):\n yield from range(n)", + "args": [], + "kwargs": {"n": 3}, + "serialization_format": "json", + } + ) + assert response["success"] is True + assert response["json_result"] == [0, 1, 2] + + +class TestSystemDependencies: + def test_missing_apt_reports_error(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: None) + response = executor.execute_request( + { + "function_name": "f", + "function_code": "def f():\n return 1", + "system_dependencies": ["ffmpeg"], + "serialization_format": "json", + } + ) + assert response["success"] is False + assert "apt-get is not available" in response["error"] + + def test_system_deps_installed_before_execution(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") + monkeypatch.setattr(executor, "_apt_updated", False) + calls = [] + + class R: + returncode = 0 + stderr = "" + + monkeypatch.setattr( + executor.subprocess, + "run", + lambda cmd, **kw: calls.append(cmd) or R(), + ) + response = executor.execute_request( + { + "function_name": "f", + "function_code": "def f():\n return 40 + 2", + "system_dependencies": ["ffmpeg"], + "serialization_format": "json", + } + ) + assert response["success"] is True + assert response["json_result"] == 42 + assert calls[0][:2] == ["apt-get", "update"] + assert "ffmpeg" in calls[1] + + def test_apt_update_runs_once(self, monkeypatch): + import shutil + + monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") + monkeypatch.setattr(executor, "_apt_updated", False) + calls = [] + + class R: + returncode = 0 + stderr = "" + + monkeypatch.setattr( + executor.subprocess, + "run", + lambda cmd, **kw: calls.append(cmd) or R(), + ) + executor._install_system(["ffmpeg"]) + executor._install_system(["sox"]) + updates = [c for c in calls if c[:2] == ["apt-get", "update"]] + assert len(updates) == 1 + + +class TestStdoutTee: + def test_prints_reach_real_stdout_and_response(self, capfd): + response = executor.execute_request( + { + "function_name": "speak", + "function_code": "def speak():\n print('live line')\n return 1\n", + "args": [], + "kwargs": {}, + "serialization_format": "json", + } + ) + assert response["success"] + # captured for the job response + assert "live line" in response["stdout"] + # and written through to the container's stdout for log streams + assert "live line" in capfd.readouterr().out + + +class TestCloudpickleLoading: + def test_available_returns_module(self): + module = executor._load_cloudpickle() + assert module is not None + assert hasattr(module, "dumps") + + def test_missing_without_install_returns_none(self, monkeypatch): + import builtins + + real_import = builtins.__import__ + + def no_cloudpickle(name, *args, **kwargs): + if name == "cloudpickle": + raise ImportError("not installed") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", no_cloudpickle) + assert executor._load_cloudpickle(install=False) is None + + +class TestInstallHelpers: + def test_install_empty_is_noop(self): + assert executor._install([], "nothing") is None + + def test_install_failure_returns_message(self, monkeypatch): + from unittest.mock import MagicMock + + result = MagicMock(returncode=1, stderr="resolver exploded") + monkeypatch.setattr( + executor.subprocess, "run", MagicMock(return_value=result) + ) + message = executor._install(["ghost-package"], "deps") + assert "resolver exploded" in message + + def test_install_system_requires_apt(self, monkeypatch): + monkeypatch.setattr(executor.shutil, "which", lambda _: None) + message = executor._install_system(["ffmpeg"]) + assert "apt-get" in message + + def test_install_system_empty_is_noop(self): + assert executor._install_system([]) is None diff --git a/tests/test_apps/test_job.py b/tests/test_apps/test_job.py new file mode 100644 index 000000000..cc8a4e242 --- /dev/null +++ b/tests/test_apps/test_job.py @@ -0,0 +1,161 @@ +"""tests for the shared Apps SDK queue job.""" + +from typing import Any, Callable, Dict, Optional + +import pytest + +import runpod +from runpod.apps.app import _clear_registry +from runpod.apps.errors import InvalidResourceError +from runpod.apps.job import Job + + +@pytest.fixture(autouse=True) +def clean_registry(): + _clear_registry() + yield + _clear_registry() + + +class FakeTarget: + def __init__(self): + self.calls = [] + + async def job_status(self, job_id: str) -> Dict[str, Any]: + self.calls.append(("status", job_id)) + return {"id": job_id, "status": "IN_PROGRESS", "workerId": "worker-1"} + + async def wait( + self, + job_data: Dict[str, Any], + *, + timeout: float, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, + ) -> Any: + self.calls.append(("wait", job_data["id"], timeout)) + final = { + "id": job_data["id"], + "status": "COMPLETED", + "output": {"answer": 42}, + } + if on_status is not None: + on_status(final) + return final["output"] + + async def cancel_job(self, job_id: str) -> Dict[str, Any]: + self.calls.append(("cancel", job_id)) + return {"id": job_id, "status": "CANCELLED"} + + async def retry_job(self, job_id: str) -> Dict[str, Any]: + self.calls.append(("retry", job_id)) + return {"id": job_id, "status": "IN_QUEUE", "retries": 1} + + async def stream_job(self, job_id: str, *, timeout: float = 300.0): + self.calls.append(("stream", job_id)) + for chunk in ("a", "b", "c"): + yield chunk + + +def test_job_status_polls_target(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + assert job.status() == "IN_PROGRESS" + assert target.calls == [("status", "job-1")] + + +def test_job_status_skips_poll_once_terminal(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "COMPLETED"}, target) + + assert job.status() == "COMPLETED" + assert target.calls == [] + + +def test_job_result_reaches_terminal_state(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + assert job.result(timeout=12) == {"answer": 42} + assert job.done is True + assert job.status() == "COMPLETED" + # the wait carried timeout through; terminal status needed no extra poll + assert target.calls == [("wait", "job-1", 12)] + + +def test_job_output_aliases_result(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + assert job.output() == {"answer": 42} + + +def test_job_cancel_and_retry_update_state(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_PROGRESS"}, target) + + assert job.cancel() is job + assert job.done is True + assert job.retry() is job + assert job.done is False + assert target.calls == [("cancel", "job-1"), ("retry", "job-1")] + + +async def test_job_async_forms(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + assert await job.status.aio() == "IN_PROGRESS" + assert await job.result.aio() == {"answer": 42} + assert await job.cancel.aio() is job + assert await job.retry.aio() is job + + +def test_job_stream_sync(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + assert list(job.stream()) == ["a", "b", "c"] + assert target.calls == [("stream", "job-1")] + + +async def test_job_stream_async(): + target = FakeTarget() + job = Job({"id": "job-1", "status": "IN_QUEUE"}, target) + + chunks = [chunk async for chunk in job.stream.aio()] + assert chunks == ["a", "b", "c"] + + +def test_job_repr(): + job = Job({"id": "job-1", "status": "IN_QUEUE"}, FakeTarget()) + assert repr(job) == "Job(id='job-1', status='IN_QUEUE')" + + +def test_decorated_queue_reconnects_to_job(monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = runpod.App("demo") + + @app.queue(name="generate") + def generate(prompt: str): + return prompt + + job = generate.job("job-1") + + assert isinstance(job, runpod.Job) + assert job.id == "job-1" + assert job.done is False + + +def test_task_job_cannot_reconnect(monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = runpod.App("demo") + + @app.task(name="train") + def train(): + return None + + with pytest.raises(InvalidResourceError, match="cannot be reconnected"): + train.job("job-1") diff --git a/tests/test_apps/test_queue_worker.py b/tests/test_apps/test_queue_worker.py index 1630c2ac1..8ac14f8e9 100644 --- a/tests/test_apps/test_queue_worker.py +++ b/tests/test_apps/test_queue_worker.py @@ -113,6 +113,25 @@ async def test_no_input(self): ) assert await handler({}) == "ok" + async def test_sync_generator(self): + def fn(n): + for i in range(n): + yield i + + handler = worker._make_deployed_handler(self._handle(fn)) + assert inspect.isgeneratorfunction(handler) + assert list(handler({"input": {"n": 3}})) == [0, 1, 2] + + async def test_async_generator(self): + async def fn(n): + for i in range(n): + yield i + + handler = worker._make_deployed_handler(self._handle(fn)) + assert inspect.isasyncgenfunction(handler) + chunks = [c async for c in handler({"input": {"n": 2}})] + assert chunks == [0, 1] + class TestInitHook: def test_no_init(self): @@ -165,6 +184,20 @@ def test_worker_config_concurrent(self, monkeypatch): config = worker._worker_config(lambda job: job) assert config["concurrency_modifier"](1) == 4 + def test_worker_config_plain_handler(self, monkeypatch): + monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) + config = worker._worker_config(lambda job: job) + assert "return_aggregate_stream" not in config + + def test_worker_config_generator_handler(self, monkeypatch): + monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) + + def gen_handler(job): + yield job + + config = worker._worker_config(gen_handler) + assert config["return_aggregate_stream"] is True + class TestMain: def test_live_mode(self, monkeypatch): @@ -195,11 +228,55 @@ def test_deployed_mode(self, monkeypatch, tmp_path): handler = start.call_args[0][0]["handler"] assert inspect.iscoroutinefunction(handler) - def test_live_handler_delegates(self): - with patch( - "runpod.runtimes.task.runner.execute_request", - return_value={"ok": True}, - ) as execute: - result = worker._live_handler({"input": {"foo": 1}}) - assert result == {"ok": True} + async def test_live_handler_plain_function(self): + def fn(x): + return {"ok": x} + + with ( + patch( + "runpod.runtimes.executor.resolve_request", + return_value=((fn, [1], {}), None), + ), + patch( + "runpod.runtimes.executor.execute_request", + return_value={"success": True, "json_result": {"ok": 1}}, + ) as execute, + ): + chunks = [c async for c in worker._live_handler({"input": {"foo": 1}})] + assert chunks == [{"success": True, "json_result": {"ok": 1}}] execute.assert_called_once_with({"foo": 1}) + + async def test_live_handler_resolve_error(self): + with patch( + "runpod.runtimes.executor.resolve_request", + return_value=(None, {"success": False, "error": "boom"}), + ): + chunks = [c async for c in worker._live_handler({"input": {}})] + assert chunks == [{"success": False, "error": "boom"}] + + async def test_live_handler_streams_generator(self): + def fn(n): + for i in range(n): + yield i + + with patch( + "runpod.runtimes.executor.resolve_request", + return_value=((fn, [], {"n": 2}), None), + ): + chunks = [c async for c in worker._live_handler({"input": {}})] + assert all(c["__stream__"] for c in chunks) + assert all(c["success"] for c in chunks) + assert len(chunks) == 2 + + async def test_live_handler_streams_async_generator(self): + async def fn(n): + for i in range(n): + yield i + + with patch( + "runpod.runtimes.executor.resolve_request", + return_value=((fn, [], {"n": 3}), None), + ): + chunks = [c async for c in worker._live_handler({"input": {}})] + assert len(chunks) == 3 + assert all(c["__stream__"] for c in chunks) diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py index e4b31c5c1..b660c83e7 100644 --- a/tests/test_apps/test_targets.py +++ b/tests/test_apps/test_targets.py @@ -241,6 +241,57 @@ async def test_submit_and_wait(self, local_endpoint): assert job["id"] == "j1" assert await target.wait(job, timeout=10) == 42 + async def test_job_operations(self, local_endpoint): + local_endpoint.state["responses"][f"/{SENTINEL_ID}/status/j1"] = { + "id": "j1", + "status": "IN_PROGRESS", + } + local_endpoint.state["responses"][f"/{SENTINEL_ID}/cancel/j1"] = { + "id": "j1", + "status": "CANCELLED", + } + local_endpoint.state["responses"][f"/{SENTINEL_ID}/retry/j1"] = { + "id": "j1", + "status": "IN_QUEUE", + } + target = SentinelTarget("demo", "production", "chat") + + assert (await target.job_status("j1"))["status"] == "IN_PROGRESS" + assert (await target.cancel_job("j1"))["status"] == "CANCELLED" + assert (await target.retry_job("j1"))["status"] == "IN_QUEUE" + + requests = local_endpoint.state["requests"] + assert [request["path"] for request in requests] == [ + f"/{SENTINEL_ID}/status/j1", + f"/{SENTINEL_ID}/cancel/j1", + f"/{SENTINEL_ID}/retry/j1", + ] + assert requests[0]["headers"]["X-Flash-Environment"] == "production" + + async def test_stream_job(self, local_endpoint): + local_endpoint.state["responses"][f"/{SENTINEL_ID}/stream/j1"] = { + "status": "COMPLETED", + "stream": [{"output": "a"}, {"output": "b"}], + } + target = SentinelTarget("demo", "default", "chat") + + chunks = [c async for c in target.stream_job("j1", timeout=10)] + assert chunks == ["a", "b"] + request = local_endpoint.state["requests"][0] + assert request["path"] == f"/{SENTINEL_ID}/stream/j1" + assert request["headers"]["X-Flash-App"] == "demo" + + async def test_stream_job_failed(self, local_endpoint): + local_endpoint.state["responses"][f"/{SENTINEL_ID}/stream/j1"] = { + "status": "FAILED", + "error": "boom", + } + target = SentinelTarget("demo", "default", "chat") + + with pytest.raises(RemoteExecutionError, match="boom"): + async for _ in target.stream_job("j1", timeout=10): + pass + def test_payload_is_plain_kwargs(self): target = SentinelTarget("demo", "default", "chat") @@ -268,6 +319,27 @@ def fn(prompt): assert body["function_name"] == "fn" assert "def fn(" in body["function_code"] + def test_payload_args_are_plain_json(self): + target = LiveTarget("ep123", "chat") + + def fn(prompt, n): + return prompt + + payload = target.build_payload(fn, self._spec(), ("hi",), {"n": 2}) + body = payload["input"] + assert body["args"] == ["hi"] + assert body["kwargs"] == {"n": 2} + assert body["serialization_format"] == "json" + + def test_payload_rejects_non_json_args(self): + target = LiveTarget("ep123", "chat") + + def fn(x): + return x + + with pytest.raises(TypeError, match="json-serializable"): + target.build_payload(fn, self._spec(), (object(),), {}) + def test_unwrap_success_response(self): target = LiveTarget("ep123", "chat") output = {"success": True, "result": None, "json_result": {"x": 1}} @@ -287,6 +359,37 @@ def test_unwrap_passthrough(self): {"status": "COMPLETED", "output": {"plain": 1}} ) == {"plain": 1} + def test_unwrap_aggregated_plain_function(self): + # live handlers aggregate; a plain function is one unmarked envelope + target = LiveTarget("ep123", "chat") + output = [{"success": True, "json_result": {"x": 1}}] + assert target.unwrap( + {"status": "COMPLETED", "output": output} + ) == {"x": 1} + + def test_unwrap_aggregated_generator(self): + target = LiveTarget("ep123", "chat") + output = [ + {"success": True, "__stream__": True, "json_result": "a"}, + {"success": True, "__stream__": True, "json_result": "b"}, + ] + assert target.unwrap( + {"status": "COMPLETED", "output": output} + ) == ["a", "b"] + + async def test_stream_job_unwraps_chunks(self, local_endpoint): + local_endpoint.state["responses"]["/ep123/stream/j1"] = { + "status": "COMPLETED", + "stream": [ + {"output": {"success": True, "__stream__": True, "json_result": "a"}}, + {"output": {"success": True, "__stream__": True, "json_result": "b"}}, + ], + } + target = LiveTarget("ep123", "chat") + + chunks = [c async for c in target.stream_job("j1", timeout=10)] + assert chunks == ["a", "b"] + async def test_invoke(self, local_endpoint): target = LiveTarget("ep123", "chat") data = await target.invoke({"input": {"x": 1}}, timeout=10) @@ -294,6 +397,25 @@ async def test_invoke(self, local_endpoint): request = local_endpoint.state["requests"][0] assert request["path"] == "/ep123/runsync" + async def test_job_operations(self, local_endpoint): + local_endpoint.state["responses"]["/ep123/status/j1"] = { + "id": "j1", + "status": "IN_PROGRESS", + } + local_endpoint.state["responses"]["/ep123/cancel/j1"] = { + "id": "j1", + "status": "CANCELLED", + } + local_endpoint.state["responses"]["/ep123/retry/j1"] = { + "id": "j1", + "status": "IN_QUEUE", + } + target = LiveTarget("ep123", "chat") + + assert (await target.job_status("j1"))["status"] == "IN_PROGRESS" + assert (await target.cancel_job("j1"))["status"] == "CANCELLED" + assert (await target.retry_job("j1"))["status"] == "IN_QUEUE" + async def test_sync_source_skips_unchanged(self, monkeypatch): target = LiveTarget("ep123", "chat") diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py index ded81bf2f..645b2d0f3 100644 --- a/tests/test_apps/test_task_runner_http.py +++ b/tests/test_apps/test_task_runner_http.py @@ -1,7 +1,8 @@ """integration tests for the task runner http server. boots the real ThreadingHTTPServer on a random port and exercises the -protocol over actual sockets. +protocol over actual sockets. engine behavior (dependency install, +serialization, stdout capture) is covered in test_executor.py. """ import base64 @@ -125,91 +126,26 @@ def test_unknown_path_404(server): assert exc_info.value.code == 404 -class TestSystemDependencies: - def test_missing_apt_reports_error(self, monkeypatch): - import shutil +def test_single_file_source_is_self_contained(): + """the shipped runner must run without the runpod package installed.""" + import subprocess + import sys - monkeypatch.setattr(shutil, "which", lambda _: None) - response = task_runner.execute_request( - { - "function_name": "f", - "function_code": "def f():\n return 1", - "system_dependencies": ["ffmpeg"], - "serialization_format": "json", - } - ) - assert response["success"] is False - assert "apt-get is not available" in response["error"] - - def test_system_deps_installed_before_execution(self, monkeypatch): - import shutil - - monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(task_runner, "_apt_updated", False) - calls = [] - - class R: - returncode = 0 - stderr = "" - - monkeypatch.setattr( - task_runner.subprocess, - "run", - lambda cmd, **kw: calls.append(cmd) or R(), - ) - response = task_runner.execute_request( - { - "function_name": "f", - "function_code": "def f():\n return 40 + 2", - "system_dependencies": ["ffmpeg"], - "serialization_format": "json", - } - ) - assert response["success"] is True - assert response["json_result"] == 42 - assert calls[0][:2] == ["apt-get", "update"] - assert "ffmpeg" in calls[1] - - def test_apt_update_runs_once(self, monkeypatch): - import shutil - - monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(task_runner, "_apt_updated", False) - calls = [] + from runpod.apps.tasks import _runner_source - class R: - returncode = 0 - stderr = "" - - monkeypatch.setattr( - task_runner.subprocess, - "run", - lambda cmd, **kw: calls.append(cmd) or R(), - ) - task_runner._install_system(["ffmpeg"]) - task_runner._install_system(["sox"]) - updates = [c for c in calls if c[:2] == ["apt-get", "update"]] - assert len(updates) == 1 - - -class TestStdoutTee: - def test_prints_reach_real_stdout_and_response(self, capfd): - from runpod.runtimes.task.runner import execute_request - - response = execute_request( - { - "function_name": "speak", - "function_code": "def speak():\n print('live line')\n return 1\n", - "args": [], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"] - # captured for the job response - assert "live line" in response["stdout"] - # and written through to the container's stdout for log streams - assert "live line" in capfd.readouterr().out + source = _runner_source() + assert "from runpod.runtimes.executor import" in source + # compile in an empty namespace with runpod imports failing + probe = ( + "import sys\n" + "sys.modules['runpod'] = None\n" # forces the ImportError branch + "sys.argv = ['runner']\n" + + source.replace('if __name__ == "__main__":', "if False:") + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True + ) + assert result.returncode == 0, result.stderr class TestWatchdog: @@ -300,46 +236,3 @@ def fail(req, timeout=None): ) task_runner._terminate_self() assert exits == [0] - - -class TestCloudpickleLoading: - def test_available_returns_module(self): - module = task_runner._load_cloudpickle() - assert module is not None - assert hasattr(module, "dumps") - - def test_missing_without_install_returns_none(self, monkeypatch): - import builtins - - real_import = builtins.__import__ - - def no_cloudpickle(name, *args, **kwargs): - if name == "cloudpickle": - raise ImportError("not installed") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", no_cloudpickle) - assert task_runner._load_cloudpickle(install=False) is None - - -class TestInstallHelpers: - def test_install_empty_is_noop(self): - assert task_runner._install([], "nothing") is None - - def test_install_failure_returns_message(self, monkeypatch): - from unittest.mock import MagicMock - - result = MagicMock(returncode=1, stderr="resolver exploded") - monkeypatch.setattr( - task_runner.subprocess, "run", MagicMock(return_value=result) - ) - message = task_runner._install(["ghost-package"], "deps") - assert "resolver exploded" in message - - def test_install_system_requires_apt(self, monkeypatch): - monkeypatch.setattr(task_runner.shutil, "which", lambda _: None) - message = task_runner._install_system(["ffmpeg"]) - assert "apt-get" in message - - def test_install_system_empty_is_noop(self): - assert task_runner._install_system([]) is None diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index beacf36b7..91656f10f 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -21,7 +21,7 @@ from runpod.apps.app import _clear_registry from runpod.apps.errors import RemoteExecutionError from runpod.apps.spec import ResourceKind, ResourceSpec -from runpod.runtimes.task.runner import execute_request +from runpod.runtimes.executor import execute_request from runpod.apps.tasks import _pod_input, unwrap_task_response from runpod.apps.targets import PodTarget diff --git a/tests/test_init.py b/tests/test_init.py index f5ff3ca68..62a74fec2 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -100,7 +100,7 @@ def test_all_covers_expected_public_api(self): 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', # Apps surface 'Api', 'App', 'CpuInstanceType', 'DataCenter', 'EndpointNotFound', 'GpuGroup', - 'GpuType', 'Queue', 'delete', 'get', 'init', 'is_local', + 'GpuType', 'Job', 'Queue', 'delete', 'get', 'init', 'is_local', 'local_entrypoint', 'patch', 'post', 'put', 'schedule', # Serverless module 'serverless', diff --git a/tests/test_shared/test_auth.py b/tests/test_shared/test_auth.py index 00f858484..2cb93defb 100644 --- a/tests/test_shared/test_auth.py +++ b/tests/test_shared/test_auth.py @@ -3,6 +3,7 @@ """ import importlib +import os import unittest from unittest.mock import mock_open, patch @@ -24,6 +25,9 @@ def test_use_file_credentials(self, _mock_exists, mock_file): """ import runpod # pylint: disable=import-outside-toplevel - importlib.reload(runpod) + # the env var outranks file credentials; clear it so the file wins + with patch.dict(os.environ): + os.environ.pop("RUNPOD_API_KEY", None) + importlib.reload(runpod) self.assertEqual(runpod.api_key, "RUNPOD_API_KEY") assert mock_file.called From e5fea5ec68af3cc74d9ee365c9e951fe101dfd62 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:35:02 -0700 Subject: [PATCH 086/120] refactor: extract apps graphql documents into api query and mutation modules --- runpod/api/mutations/apps.py | 132 +++++++++++++++ runpod/api/queries/apps.py | 116 +++++++++++++ runpod/apps/api.py | 309 ++++++----------------------------- 3 files changed, 300 insertions(+), 257 deletions(-) create mode 100644 runpod/api/mutations/apps.py create mode 100644 runpod/api/queries/apps.py diff --git a/runpod/api/mutations/apps.py b/runpod/api/mutations/apps.py new file mode 100644 index 000000000..ff7287f5e --- /dev/null +++ b/runpod/api/mutations/apps.py @@ -0,0 +1,132 @@ +"""graphql mutations for the apps control-plane client.""" + +MUTATION_SAVE_ENDPOINT = """ +mutation saveEndpoint($input: EndpointInput!) { + saveEndpoint(input: $input) { + id + name + templateId + gpuIds + instanceIds + workersMin + workersMax + idleTimeout + aiKey + } +} +""" + +MUTATION_DELETE_ENDPOINT = """ +mutation deleteEndpoint($id: String!) { + deleteEndpoint(id: $id) +} +""" + +MUTATION_DEPLOY_CPU_POD = """ +mutation deployCpuPod($input: deployCpuPodInput!) { + deployCpuPod(input: $input) { id desiredStatus } +} +""" + +MUTATION_DEPLOY_POD = """ +mutation deployPod($input: PodFindAndDeployOnDemandInput) { + podFindAndDeployOnDemand(input: $input) { id desiredStatus } +} +""" + +MUTATION_TERMINATE_POD = """ +mutation terminatePod($input: PodTerminateInput!) { + podTerminate(input: $input) +} +""" + +MUTATION_CREATE_FLASH_APP = """ +mutation createFlashApp($input: CreateFlashAppInput!) { + createFlashApp(input: $input) { id name } +} +""" + +MUTATION_CREATE_FLASH_ENVIRONMENT = """ +mutation createFlashEnvironment($input: CreateFlashEnvironmentInput!) { + createFlashEnvironment(input: $input) { id name } +} +""" + +MUTATION_CREATE_NETWORK_VOLUME = """ +mutation createNetworkVolume($input: CreateNetworkVolumeInput!) { + createNetworkVolume(input: $input) { + id + name + size + dataCenterId + } +} +""" + +MUTATION_SAVE_REGISTRY_AUTH = """ +mutation SaveRegistryAuth($input: SaveRegistryAuthInput!) { + saveRegistryAuth(input: $input) { id name } +} +""" + +MUTATION_DELETE_REGISTRY_AUTH = """ +mutation DeleteRegistryAuth($registryAuthId: String!) { + deleteRegistryAuth(registryAuthId: $registryAuthId) +} +""" + +MUTATION_CREATE_SECRET = """ +mutation secretCreate($input: SecretCreateInput!) { + secretCreate(input: $input) { id name } +} +""" + +MUTATION_DELETE_SECRET = """ +mutation secretDelete($id: ID!) { + secretDelete(id: $id) +} +""" + +MUTATION_DELETE_FLASH_APP = """ +mutation deleteFlashApp($flashAppId: String!) { + deleteFlashApp(flashAppId: $flashAppId) +} +""" + +MUTATION_DELETE_FLASH_ENVIRONMENT = """ +mutation deleteFlashEnvironment($flashEnvironmentId: String!) { + deleteFlashEnvironment(flashEnvironmentId: $flashEnvironmentId) +} +""" + +MUTATION_CREATE_FLASH_AUTH_REQUEST = """ +mutation createFlashAuthRequest { + createFlashAuthRequest { + id + status + expiresAt + } +} +""" + +MUTATION_PREPARE_ARTIFACT_UPLOAD = """ +mutation PrepareArtifactUpload($input: PrepareFlashArtifactUploadInput!) { + prepareFlashArtifactUpload(input: $input) { + uploadUrl + objectKey + expiresAt + } +} +""" + +MUTATION_FINALIZE_ARTIFACT_UPLOAD = """ +mutation FinalizeArtifactUpload($input: FinalizeFlashArtifactUploadInput!) { + finalizeFlashArtifactUpload(input: $input) { id manifest } +} +""" + +MUTATION_DEPLOY_BUILD = """ +mutation deployBuildToEnvironment($input: DeployBuildToEnvironmentInput!) { + deployBuildToEnvironment(input: $input) { id name } +} +""" diff --git a/runpod/api/queries/apps.py b/runpod/api/queries/apps.py new file mode 100644 index 000000000..8089aa253 --- /dev/null +++ b/runpod/api/queries/apps.py @@ -0,0 +1,116 @@ +"""graphql queries for the apps control-plane client.""" + +QUERY_MY_ENDPOINTS = """ +query myEndpoints { + myself { + endpoints { + id + name + templateId + workersMin + workersMax + template { env { key value } } + } + } +} +""" + +QUERY_FLASH_APP_BY_NAME = """ +query getFlashAppByName($flashAppName: String!) { + flashAppByName(flashAppName: $flashAppName) { + id + name + flashEnvironments { + id + name + state + activeBuildId + endpoints { id name } + } + } +} +""" + +QUERY_GPU_STOCK = """ +query GpuStock($gpuTypesInput: GpuTypeFilter, $lowestPriceInput: GpuLowestPriceInput) { + gpuTypes(input: $gpuTypesInput) { + lowestPrice(input: $lowestPriceInput) { stockStatus } + } +} +""" + +QUERY_CPU_STOCK = """ +query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { + cpuFlavors(input: $cpuFlavorInput) { + specifics(input: $specificsInput) { stockStatus } + } +} +""" + +QUERY_NETWORK_VOLUMES = """ +query myVolumes { + myself { + networkVolumes { id name size dataCenterId } + } +} +""" + +QUERY_REGISTRY_AUTHS = """ +query myRegistryCreds { + myself { + containerRegistryCreds { id name } + } +} +""" + +QUERY_SECRETS = """ +query mySecrets { + myself { + secrets { id name description createdAt } + } +} +""" + +QUERY_FLASH_APPS = """ +query getFlashApps { + myself { + flashApps { + id + name + flashEnvironments { + id + name + state + createdAt + activeBuildId + } + flashBuilds { id createdAt } + } + } +} +""" + +QUERY_FLASH_ENVIRONMENT_BY_NAME = """ +query getFlashEnvironmentByName($input: FlashEnvironmentByNameInput!) { + flashEnvironmentByName(input: $input) { + id + name + state + activeBuildId + createdAt + endpoints { id name } + networkVolumes { id name } + } +} +""" + +QUERY_FLASH_AUTH_REQUEST_STATUS = """ +query flashAuthRequestStatus($flashAuthRequestId: String!) { + flashAuthRequestStatus(flashAuthRequestId: $flashAuthRequestId) { + id + status + expiresAt + apiKey + } +} +""" diff --git a/runpod/apps/api.py b/runpod/apps/api.py index f9fac098b..8e570a0cd 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -11,6 +11,8 @@ import aiohttp from ..api.graphql import run_graphql_query_async +from ..api.mutations import apps as app_mutations +from ..api.queries import apps as app_queries from ..error import QueryError @@ -24,7 +26,11 @@ def __init__(self, api_key: Optional[str] = None): self._api_key = api_key async def _execute( - self, query: str, variables: Optional[Dict[str, Any]] = None + self, + query: str, + variables: Optional[Dict[str, Any]] = None, + *, + anonymous: bool = False, ) -> Dict[str, Any]: import asyncio @@ -34,7 +40,10 @@ async def _execute( await asyncio.sleep(2 ** (attempt - 1)) try: response = await run_graphql_query_async( - query, api_key=self._api_key, variables=variables + query, + api_key=self._api_key, + variables=variables, + anonymous=anonymous, ) return response["data"] except (aiohttp.ClientError, OSError, asyncio.TimeoutError) as exc: @@ -44,108 +53,46 @@ async def _execute( assert last_exc is not None # loop always runs at least once raise last_exc - # -- endpoints (dev session provisioning) -- - async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: """create or update a serverless endpoint. include id to update.""" - mutation = """ - mutation saveEndpoint($input: EndpointInput!) { - saveEndpoint(input: $input) { - id - name - templateId - gpuIds - instanceIds - workersMin - workersMax - idleTimeout - aiKey - } - } - """ + mutation = app_mutations.MUTATION_SAVE_ENDPOINT data = await self._execute(mutation, {"input": endpoint_input}) return data["saveEndpoint"] async def delete_endpoint(self, endpoint_id: str) -> bool: - mutation = """ - mutation deleteEndpoint($id: String!) { - deleteEndpoint(id: $id) - } - """ + mutation = app_mutations.MUTATION_DELETE_ENDPOINT data = await self._execute(mutation, {"id": endpoint_id}) - return "deleteEndpoint" in data + return bool(data.get("deleteEndpoint")) async def list_my_endpoints(self) -> List[Dict[str, Any]]: - query = """ - query myEndpoints { - myself { - endpoints { - id - name - templateId - workersMin - workersMax - template { env { key value } } - } - } - } - """ + query = app_queries.QUERY_MY_ENDPOINTS data = await self._execute(query) return data["myself"]["endpoints"] - # -- pods (task execution) -- - async def deploy_task_pod( self, pod_input: Dict[str, Any], *, is_cpu: bool ) -> Dict[str, Any]: """deploy an on-demand pod for a task run.""" + pod_input = dict(pod_input) if is_cpu: # deployCpuPod takes a single instanceId instance_ids = pod_input.pop("instanceIds", None) if instance_ids: pod_input["instanceId"] = instance_ids[0] - mutation = """ - mutation deployCpuPod($input: deployCpuPodInput!) { - deployCpuPod(input: $input) { id desiredStatus } - } - """ + mutation = app_mutations.MUTATION_DEPLOY_CPU_POD data = await self._execute(mutation, {"input": pod_input}) return data["deployCpuPod"] - mutation = """ - mutation deployPod($input: PodFindAndDeployOnDemandInput) { - podFindAndDeployOnDemand(input: $input) { id desiredStatus } - } - """ + mutation = app_mutations.MUTATION_DEPLOY_POD data = await self._execute(mutation, {"input": pod_input}) return data["podFindAndDeployOnDemand"] async def terminate_pod(self, pod_id: str) -> None: - mutation = """ - mutation terminatePod($input: PodTerminateInput!) { - podTerminate(input: $input) - } - """ + mutation = app_mutations.MUTATION_TERMINATE_POD await self._execute(mutation, {"input": {"podId": pod_id}}) - # -- app lifecycle (deploy) -- - async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: - query = """ - query getFlashAppByName($flashAppName: String!) { - flashAppByName(flashAppName: $flashAppName) { - id - name - flashEnvironments { - id - name - state - activeBuildId - endpoints { id name } - } - } - } - """ + query = app_queries.QUERY_FLASH_APP_BY_NAME try: data = await self._execute(query, {"flashAppName": name}) except QueryError as exc: @@ -155,27 +102,17 @@ async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: return data["flashAppByName"] async def create_app(self, name: str) -> Dict[str, Any]: - mutation = """ - mutation createFlashApp($input: CreateFlashAppInput!) { - createFlashApp(input: $input) { id name } - } - """ + mutation = app_mutations.MUTATION_CREATE_FLASH_APP data = await self._execute(mutation, {"input": {"name": name}}) return data["createFlashApp"] async def create_environment(self, app_id: str, name: str) -> Dict[str, Any]: - mutation = """ - mutation createFlashEnvironment($input: CreateFlashEnvironmentInput!) { - createFlashEnvironment(input: $input) { id name } - } - """ + mutation = app_mutations.MUTATION_CREATE_FLASH_ENVIRONMENT data = await self._execute( mutation, {"input": {"flashAppId": app_id, "name": name}} ) return data["createFlashEnvironment"] - # -- stock (placement) -- - async def gpu_stock_status( self, gpu_id: str, @@ -188,13 +125,7 @@ async def gpu_stock_status( the serverless plane (includeAiApi) and the pod plane have different availability; pods=True queries pod stock (tasks). """ - query = """ - query GpuStock($gpuTypesInput: GpuTypeFilter, $lowestPriceInput: GpuLowestPriceInput) { - gpuTypes(input: $gpuTypesInput) { - lowestPrice(input: $lowestPriceInput) { stockStatus } - } - } - """ + query = app_queries.QUERY_GPU_STOCK data = await self._execute( query, { @@ -217,13 +148,7 @@ async def cpu_stock_status( ) -> Optional[str]: """stock signal for a cpu flavor in one datacenter.""" flavor = instance_id.split("-", 1)[0] - query = """ - query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { - cpuFlavors(input: $cpuFlavorInput) { - specifics(input: $specificsInput) { stockStatus } - } - } - """ + query = app_queries.QUERY_CPU_STOCK data = await self._execute( query, { @@ -239,32 +164,15 @@ async def cpu_stock_status( specifics = first.get("specifics") if isinstance(first, dict) else None return (specifics or {}).get("stockStatus") - # -- network volumes -- - async def list_network_volumes(self) -> List[Dict[str, Any]]: - query = """ - query myVolumes { - myself { - networkVolumes { id name size dataCenterId } - } - } - """ + query = app_queries.QUERY_NETWORK_VOLUMES data = await self._execute(query) return data["myself"].get("networkVolumes") or [] async def create_network_volume( self, name: str, size: int, data_center_id: str ) -> Dict[str, Any]: - mutation = """ - mutation createNetworkVolume($input: CreateNetworkVolumeInput!) { - createNetworkVolume(input: $input) { - id - name - size - dataCenterId - } - } - """ + mutation = app_mutations.MUTATION_CREATE_NETWORK_VOLUME data = await self._execute( mutation, { @@ -277,27 +185,15 @@ async def create_network_volume( ) return data["createNetworkVolume"] - # -- registry credentials -- - async def list_registry_auths(self) -> List[Dict[str, Any]]: - query = """ - query myRegistryCreds { - myself { - containerRegistryCreds { id name } - } - } - """ + query = app_queries.QUERY_REGISTRY_AUTHS data = await self._execute(query) return data["myself"].get("containerRegistryCreds") or [] async def create_registry_auth( self, name: str, username: str, password: str ) -> Dict[str, Any]: - mutation = """ - mutation SaveRegistryAuth($input: SaveRegistryAuthInput!) { - saveRegistryAuth(input: $input) { id name } - } - """ + mutation = app_mutations.MUTATION_SAVE_REGISTRY_AUTH data = await self._execute( mutation, { @@ -311,35 +207,19 @@ async def create_registry_auth( return data["saveRegistryAuth"] async def delete_registry_auth(self, auth_id: str) -> bool: - mutation = """ - mutation DeleteRegistryAuth($registryAuthId: String!) { - deleteRegistryAuth(registryAuthId: $registryAuthId) - } - """ + mutation = app_mutations.MUTATION_DELETE_REGISTRY_AUTH data = await self._execute(mutation, {"registryAuthId": auth_id}) - return "deleteRegistryAuth" in data - - # -- secrets -- + return bool(data.get("deleteRegistryAuth")) async def list_secrets(self) -> List[Dict[str, Any]]: - query = """ - query mySecrets { - myself { - secrets { id name description createdAt } - } - } - """ + query = app_queries.QUERY_SECRETS data = await self._execute(query) return data["myself"].get("secrets") or [] async def create_secret( self, name: str, value: str, description: str = "" ) -> Dict[str, Any]: - mutation = """ - mutation secretCreate($input: SecretCreateInput!) { - secretCreate(input: $input) { id name } - } - """ + mutation = app_mutations.MUTATION_CREATE_SECRET data = await self._execute( mutation, { @@ -353,36 +233,13 @@ async def create_secret( return data["secretCreate"] async def delete_secret(self, secret_id: str) -> bool: - mutation = """ - mutation secretDelete($id: ID!) { - secretDelete(id: $id) - } - """ + mutation = app_mutations.MUTATION_DELETE_SECRET data = await self._execute(mutation, {"id": secret_id}) - return "secretDelete" in data - - # -- app / environment management -- + return bool(data.get("secretDelete")) async def list_apps(self) -> List[Dict[str, Any]]: """all flash apps with their environments and builds.""" - query = """ - query getFlashApps { - myself { - flashApps { - id - name - flashEnvironments { - id - name - state - createdAt - activeBuildId - } - flashBuilds { id createdAt } - } - } - } - """ + query = app_queries.QUERY_FLASH_APPS data = await self._execute(query) return data["myself"].get("flashApps") or [] @@ -393,19 +250,7 @@ async def get_environment_by_name( app = await self.get_app_by_name(app_name) if app is None: return None - query = """ - query getFlashEnvironmentByName($input: FlashEnvironmentByNameInput!) { - flashEnvironmentByName(input: $input) { - id - name - state - activeBuildId - createdAt - endpoints { id name } - networkVolumes { id name } - } - } - """ + query = app_queries.QUERY_FLASH_ENVIRONMENT_BY_NAME try: data = await self._execute( query, @@ -418,74 +263,34 @@ async def get_environment_by_name( return data["flashEnvironmentByName"] async def delete_app(self, app_id: str) -> bool: - mutation = """ - mutation deleteFlashApp($flashAppId: String!) { - deleteFlashApp(flashAppId: $flashAppId) - } - """ + mutation = app_mutations.MUTATION_DELETE_FLASH_APP data = await self._execute(mutation, {"flashAppId": app_id}) - return "deleteFlashApp" in data + return bool(data.get("deleteFlashApp")) async def delete_environment(self, environment_id: str) -> bool: - mutation = """ - mutation deleteFlashEnvironment($flashEnvironmentId: String!) { - deleteFlashEnvironment(flashEnvironmentId: $flashEnvironmentId) - } - """ - data = await self._execute( - mutation, {"flashEnvironmentId": environment_id} - ) - return "deleteFlashEnvironment" in data - - # -- browser auth (login) -- + mutation = app_mutations.MUTATION_DELETE_FLASH_ENVIRONMENT + data = await self._execute(mutation, {"flashEnvironmentId": environment_id}) + return bool(data.get("deleteFlashEnvironment")) async def create_auth_request(self) -> Dict[str, Any]: """open a browser-approval auth request; no credentials needed.""" - mutation = """ - mutation createFlashAuthRequest { - createFlashAuthRequest { - id - status - expiresAt - } - } - """ - response = await run_graphql_query_async(mutation, anonymous=True) - return response["data"].get("createFlashAuthRequest") or {} + mutation = app_mutations.MUTATION_CREATE_FLASH_AUTH_REQUEST + data = await self._execute(mutation, anonymous=True) + return data.get("createFlashAuthRequest") or {} async def get_auth_request_status(self, request_id: str) -> Dict[str, Any]: - query = """ - query flashAuthRequestStatus($flashAuthRequestId: String!) { - flashAuthRequestStatus(flashAuthRequestId: $flashAuthRequestId) { - id - status - expiresAt - apiKey - } - } - """ - # anonymous by design: the server only releases the granted key - # to guest requests, so authenticating the poll (e.g. with a - # stale stored key) would return apiKey=null forever - response = await run_graphql_query_async( + query = app_queries.QUERY_FLASH_AUTH_REQUEST_STATUS + data = await self._execute( query, - variables={"flashAuthRequestId": request_id}, + {"flashAuthRequestId": request_id}, anonymous=True, ) - return response["data"].get("flashAuthRequestStatus") or {} + return data.get("flashAuthRequestStatus") or {} async def prepare_artifact_upload( self, app_id: str, tarball_size: int ) -> Dict[str, Any]: - mutation = """ - mutation PrepareArtifactUpload($input: PrepareFlashArtifactUploadInput!) { - prepareFlashArtifactUpload(input: $input) { - uploadUrl - objectKey - expiresAt - } - } - """ + mutation = app_mutations.MUTATION_PREPARE_ARTIFACT_UPLOAD data = await self._execute( mutation, {"input": {"flashAppId": app_id, "tarballSize": tarball_size}} ) @@ -494,11 +299,7 @@ async def prepare_artifact_upload( async def finalize_artifact_upload( self, app_id: str, object_key: str, manifest: Dict[str, Any] ) -> Dict[str, Any]: - mutation = """ - mutation FinalizeArtifactUpload($input: FinalizeFlashArtifactUploadInput!) { - finalizeFlashArtifactUpload(input: $input) { id manifest } - } - """ + mutation = app_mutations.MUTATION_FINALIZE_ARTIFACT_UPLOAD data = await self._execute( mutation, { @@ -511,14 +312,8 @@ async def finalize_artifact_upload( ) return data["finalizeFlashArtifactUpload"] - async def deploy_build( - self, environment_id: str, build_id: str - ) -> Dict[str, Any]: - mutation = """ - mutation deployBuildToEnvironment($input: DeployBuildToEnvironmentInput!) { - deployBuildToEnvironment(input: $input) { id name } - } - """ + async def deploy_build(self, environment_id: str, build_id: str) -> Dict[str, Any]: + mutation = app_mutations.MUTATION_DEPLOY_BUILD data = await self._execute( mutation, { From 6ed7ca28042ee6ef94eb7f9000771122f7bdf108 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:35:02 -0700 Subject: [PATCH 087/120] refactor: consolidate shared apps helpers into utils package --- runpod/apps/app.py | 45 ++----------- runpod/apps/auth.py | 12 +--- runpod/apps/context.py | 6 -- runpod/apps/datacenter.py | 7 +- runpod/apps/deploy.py | 121 +++++++++------------------------- runpod/apps/dev.py | 82 +++++++++-------------- runpod/apps/discovery.py | 7 +- runpod/apps/images.py | 10 +-- runpod/apps/manage.py | 14 ++-- runpod/apps/monitor.py | 12 +--- runpod/apps/placement.py | 6 +- runpod/apps/registry.py | 17 ++--- runpod/apps/utils/__init__.py | 1 + runpod/apps/utils/client.py | 16 +++++ runpod/apps/utils/events.py | 20 ++++++ runpod/apps/utils/lookup.py | 26 ++++++++ runpod/apps/volume.py | 25 ++----- 17 files changed, 165 insertions(+), 262 deletions(-) create mode 100644 runpod/apps/utils/__init__.py create mode 100644 runpod/apps/utils/client.py create mode 100644 runpod/apps/utils/events.py create mode 100644 runpod/apps/utils/lookup.py diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 112946bfd..b62027b5a 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -35,8 +35,6 @@ async def transcribe(audio_url: str): ... DEFAULT_ENV = "default" -# module-level registry populated as user modules import; `rp deploy` and -# `rp dev` read it after importing the target modules _REGISTRY: List["App"] = [] @@ -59,9 +57,7 @@ def __init__(self, name: str, *, env: str = DEFAULT_ENV): self.name = name self.env = env self._resources: Dict[str, Union[FunctionHandle, ApiHandle]] = {} - # event sink for task lifecycle rendering, set by dev sessions self._dev_events: Optional[object] = None - # populated by an `rp dev` session: resource name -> live target self._dev_targets: Dict[str, InvocationTarget] = {} _REGISTRY.append(self) @@ -69,17 +65,13 @@ def __init__(self, name: str, *, env: str = DEFAULT_ENV): def resources(self) -> Dict[str, Union[FunctionHandle, ApiHandle]]: return dict(self._resources) - def _register( - self, name: str, handle: Union[FunctionHandle, ApiHandle] - ) -> None: + def _register(self, name: str, handle: Union[FunctionHandle, ApiHandle]) -> None: if name in self._resources: raise InvalidResourceError( f"duplicate resource name '{name}' in app '{self.name}'" ) self._resources[name] = handle - # -- decorators -- - def queue( self, *, @@ -106,16 +98,7 @@ def queue( accelerate_downloads: bool = True, container_disk_gb: Optional[int] = None, ) -> Callable[[Callable], FunctionHandle]: - """declare a queue-based serverless endpoint from a function. - - image selects a custom base image for the workers; the deployed - code still arrives via the build artifact and is booted by the - runtime bootstrap, so any image with a python3 binary works. - - max_concurrency lets one worker process several jobs at once; - values above 1 only achieve real concurrency with async - functions. execution_timeout_ms caps a single job (0 = no cap). - """ + """declare a queue-based serverless endpoint from a function.""" def decorator(fn: Callable) -> FunctionHandle: spec = ResourceSpec( @@ -167,15 +150,7 @@ def task( accelerate_downloads: bool = True, container_disk_gb: Optional[int] = None, ) -> Callable[[Callable], FunctionHandle]: - """declare ephemeral pod compute from a function. - - tasks have no standing infrastructure: `.remote()` provisions a - pod, runs the body, returns the result, and terminates the pod. - they never require `rp deploy` (except to register a schedule). - - platform-cached models (`model=`) are only available on queue - and api resources; tasks download weights themselves. - """ + """declare ephemeral pod compute from a function.""" def decorator(fn: Callable) -> FunctionHandle: spec = ResourceSpec( @@ -279,19 +254,11 @@ def decorator(target: Any) -> ApiHandle: return decorator - # -- resolution -- - async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: - """resolve a resource spec to an invocation target. - - stateless by design: deployed resources resolve through the - sentinel (server-side name resolution); dev sessions register - live targets in memory; tasks provision per call. - """ + """resolve a resource spec to an invocation target.""" if spec.kind is ResourceKind.TASK: handle = self._resources.get(spec.name) fn = getattr(handle, "_fn", None) - # dev sessions attach an event sink for lifecycle rendering return PodTarget(spec, fn, events=self._dev_events) ctx = current_context() @@ -300,13 +267,9 @@ async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: target = self._dev_targets.get(spec.name) if target is not None: return target - # a dev session should have provisioned every endpoint at - # startup; a miss means the resource was added mid-session raise EndpointNotFound(self.name, spec.name) if ctx is Context.WORKER and os.getenv("RUNPOD_DEV_APP") == self.name: - # nested call inside a dev worker: siblings live on dev - # endpoints (named dev-{app}-{resource}), not the sentinel return await self._resolve_dev_sibling(spec) env_name = os.getenv("FLASH_ENVIRONMENT") or self.env diff --git a/runpod/apps/auth.py b/runpod/apps/auth.py index b84765922..5bca96d25 100644 --- a/runpod/apps/auth.py +++ b/runpod/apps/auth.py @@ -1,9 +1,4 @@ -"""browser-approval login flow. - -opens a console url tied to a short-lived auth request; the api key -lands when the user approves in the browser. no credentials are sent -before approval, so an expired local key can never poison the flow. -""" +"""browser-approval login flow.""" import asyncio import datetime as dt @@ -53,15 +48,14 @@ async def browser_login( client = api or AppsApiClient() request: Dict[str, Any] = await client.create_auth_request() request_id = request.get("id") + if not request_id: raise LoginError("auth request failed to initialize") if on_url is not None: on_url(auth_url(request_id)) - deadline = dt.datetime.now(dt.timezone.utc) + dt.timedelta( - seconds=timeout_seconds - ) + deadline = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=timeout_seconds) expires_at = _parse_expires_at(request.get("expiresAt")) if expires_at and expires_at < deadline: deadline = expires_at diff --git a/runpod/apps/context.py b/runpod/apps/context.py index 1c96fe726..198df12e7 100644 --- a/runpod/apps/context.py +++ b/runpod/apps/context.py @@ -70,11 +70,6 @@ def run(cls, coro: Coroutine[Any, Any, Any]) -> Any: loop = cls._ensure_loop() future = asyncio.run_coroutine_threadsafe(coro, loop) try: - # wait in short slices: an uninterruptible result() would - # block signal delivery, making ctrl-c useless while a - # remote call is in flight. TimeoutError is ambiguous (the - # wait slice and the coroutine's own timeouts share the - # type since 3.11), so future.done() decides while True: try: return future.result(timeout=0.2) @@ -82,7 +77,6 @@ def run(cls, coro: Coroutine[Any, Any, Any]) -> Any: if future.done(): raise except BaseException: - # ctrl-c (or anything else) abandons the in-flight call future.cancel() raise diff --git a/runpod/apps/datacenter.py b/runpod/apps/datacenter.py index 3a71007b8..d98236a9d 100644 --- a/runpod/apps/datacenter.py +++ b/runpod/apps/datacenter.py @@ -1,16 +1,12 @@ """datacenter selection for app resources. -only datacenters with storage support and S3 API support are listed; -resources may attach network volumes at any time, so every deployable -location must be able to host them. -""" +only datacenters with storage support and S3 API support are listed""" from enum import Enum from typing import List class DataCenter(str, Enum): - """runpod datacenters with storage and S3 API support.""" # north america US_CA_2 = "US-CA-2" @@ -31,6 +27,7 @@ class DataCenter(str, Enum): def from_string(cls, value: str) -> "DataCenter": """parse a datacenter id, accepting lowercase and underscores.""" normalized = value.strip().upper().replace("_", "-") + try: return cls(normalized) except ValueError: diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 63901ae8e..461ff254a 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -1,33 +1,18 @@ -"""deploy pipeline: discovered apps -> manifest -> artifact -> activated build. - -per app: - 1. validate resources - 2. build the manifest from the app registry - 3. vendor the runtime environment (runpod + all deps, resolved for - the worker platform) so cold starts install nothing - 4. package source + env + manifest into a tarball (.runpodignore - honored for source) - 5. get-or-create the flash app + environment - 6. upload via presigned url, finalize the build - 7. activate the build on the environment (hosts deliver the artifact - to workers via the flash volume) - 8. reconcile endpoints: one per queue/api resource, named exactly - the resource name and bound to the environment via - flashEnvironmentId (the relation the sentinel's - resolveFlashEndpoint walks). removed resources' endpoints are - deleted; tasks have no standing infra. -""" +"""deploy pipeline: discovered apps -> manifest -> artifact -> activated build.""" import base64 import fnmatch import json import logging +import os import tarfile import tempfile from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional +import runpod + from .api import AppsApiClient from .app import App from .build import ( @@ -40,6 +25,7 @@ from .errors import InvalidResourceError, ScheduleNotSupported from .schedule import SCHEDULES_ENABLED from .spec import ResourceKind +from .utils.events import emit log = logging.getLogger(__name__) @@ -60,12 +46,7 @@ def _module_path_for(fn, project_root: Path) -> str: - """dotted import path for fn's file relative to the project root. - - discovery imports files under synthetic module names, so - fn.__module__ is meaningless on the worker; the import path must be - derived from where the file sits in the shipped tree. - """ + """dotted import path for fn's file relative to the project root.""" import inspect module = getattr(fn, "__module__", "") or "" @@ -98,40 +79,41 @@ def build_manifest( ) -> Dict[str, Any]: """serialize an app's resources for the coordinator and workers.""" resources = [] + for handle in app.resources.values(): spec = handle.spec if spec.schedule and not SCHEDULES_ENABLED: raise ScheduleNotSupported() - if ( - spec.kind in (ResourceKind.QUEUE, ResourceKind.API) - and len(spec.name) < 3 - ): - # deployed queue/api resources become endpoints named - # exactly the resource name; the api floor is 3 chars + + if spec.kind in (ResourceKind.QUEUE, ResourceKind.API) and len(spec.name) < 3: raise InvalidResourceError( f"resource name '{spec.name}' is too short to deploy; " f"queue and api names must be at least 3 characters" ) + entry = spec.to_manifest() + fn = ( getattr(handle, "_fn", None) or getattr(handle, "_cls", None) or getattr(handle, "__wrapped__", None) ) + if fn is not None: entry["module"] = _module_path_for(fn, project_root) entry["qualname"] = getattr(fn, "__qualname__", fn.__name__) resources.append(entry) + manifest: Dict[str, Any] = { "version": MANIFEST_VERSION, "app": app.name, "pythonVersion": python_version, "resources": resources, } + if excluded_packages: - # packages the build expects from the worker image rather than - # the vendored env (size-prohibitive CUDA packages) manifest["excludedPackages"] = excluded_packages + return manifest @@ -208,8 +190,7 @@ def package_project( return output -# gpu runtime images carry the torch family (~9 GB uncompressed); the -# container disk must hold the image layers plus the unpacked artifact +# gpu images have some hefty dependencies pre-installed CPU_CONTAINER_DISK_GB = 10 GPU_CONTAINER_DISK_GB = 30 @@ -220,17 +201,6 @@ def _container_disk_gb(spec) -> int: return CPU_CONTAINER_DISK_GB if spec.is_cpu else GPU_CONTAINER_DISK_GB -def _account_api_key() -> Optional[str]: - import os - - key = os.getenv("RUNPOD_API_KEY") - if key: - return key - import runpod - - return runpod.api_key - - def _deployed_endpoint_input( app: App, spec, @@ -260,16 +230,14 @@ def _deployed_endpoint_input( # cross-resource calls from inside a worker go through the sentinel, # which authenticates with the account api key; without it any # .remote() from worker code would fail - api_key = _account_api_key() + api_key = runpod.api_key if api_key and "RUNPOD_API_KEY" not in template_env: template_env["RUNPOD_API_KEY"] = api_key # workers that spawn tasks must select runtime images from the same # channel the deploy used (the vendored env supplies the code, so # only the image channel needs to travel) - import os as _os - - tag = _os.environ.get("RUNPOD_RUNTIME_TAG") + tag = os.environ.get("RUNPOD_RUNTIME_TAG") if tag and "RUNPOD_RUNTIME_TAG" not in template_env: template_env["RUNPOD_RUNTIME_TAG"] = tag @@ -290,11 +258,10 @@ def _deployed_endpoint_input( "imageName": image_for_spec(spec, python_version=python_version), "containerDiskInGb": _container_disk_gb(spec), "dockerArgs": "", - "env": [ - {"key": k, "value": v} for k, v in template_env.items() - ], + "env": [{"key": k, "value": v} for k, v in template_env.items()], }, } + if spec.flashboot: payload["flashBootType"] = "FLASHBOOT" @@ -307,9 +274,7 @@ def _deployed_endpoint_input( [ { "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode( - _bootstrap_source().encode() - ).decode(), + "value": base64.b64encode(_bootstrap_source().encode()).decode(), }, {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, ] @@ -324,13 +289,9 @@ def _deployed_endpoint_input( payload["instanceIds"] = spec.cpu if not spec.datacenter: if any(i.startswith("cpu5") for i in spec.cpu or []): - payload["locations"] = ",".join( - dc.value for dc in CPU5_DATACENTERS - ) + payload["locations"] = ",".join(dc.value for dc in CPU5_DATACENTERS) else: - payload["locations"] = ",".join( - dc.value for dc in CPU3_DATACENTERS - ) + payload["locations"] = ",".join(dc.value for dc in CPU3_DATACENTERS) else: from .gpu import gpu_ids_value @@ -359,9 +320,7 @@ async def reconcile_endpoints( endpoints (matched by name) are updated in place; endpoints whose resources disappeared are deleted. """ - existing = { - e["name"]: e["id"] for e in environment.get("endpoints") or [] - } + existing = {e["name"]: e["id"] for e in environment.get("endpoints") or []} provisionable = { h.spec.name: h for h in app.resources.values() @@ -378,17 +337,13 @@ async def reconcile_endpoints( app, handle.spec, environment["id"], build_id, python_version ) await attach_endpoint_volumes(payload, handle.spec, resolver, app) - auth_id = await resolve_registry_auth( - handle.spec.registry_auth, api=client - ) + auth_id = await resolve_registry_auth(handle.spec.registry_auth, api=client) if auth_id: payload["template"]["containerRegistryAuthId"] = auth_id if handle.spec.model: from .model import model_reference - payload["modelReferences"] = [ - model_reference(handle.spec.model) - ] + payload["modelReferences"] = [model_reference(handle.spec.model)] if name in existing: payload["id"] = existing[name] result = await client.save_endpoint(payload) @@ -408,9 +363,7 @@ async def reconcile_endpoints( return endpoints -async def attach_endpoint_volumes( - payload: Dict[str, Any], spec, resolver, app -) -> None: +async def attach_endpoint_volumes(payload: Dict[str, Any], spec, resolver, app) -> None: """resolve a resource's volumes onto an endpoint payload. endpoints may span regions: one volume per datacenter, locations @@ -425,18 +378,12 @@ async def attach_endpoint_volumes( for volume in volumes: sharing = specs_sharing_volume([app], volume.name) or [spec] resolved.append(await resolver.resolve(volume, sharing)) - payload["networkVolumeIds"] = [ - {"networkVolumeId": r["id"]} for r in resolved - ] - payload["locations"] = ",".join( - dict.fromkeys(r["dataCenterId"] for r in resolved) - ) + payload["networkVolumeIds"] = [{"networkVolumeId": r["id"]} for r in resolved] + payload["locations"] = ",".join(dict.fromkeys(r["dataCenterId"] for r in resolved)) def _phase(events, name: str, detail: str = "") -> None: - handler = getattr(events, "phase", None) - if handler is not None: - handler(name, detail) + emit(events, "phase", name, detail) def build_artifact( @@ -537,9 +484,7 @@ async def deploy_app( log.info("created app %s (%s)", app.name, remote_app["id"]) app_id = remote_app["id"] - environments = { - e["name"]: e for e in remote_app.get("flashEnvironments") or [] - } + environments = {e["name"]: e for e in remote_app.get("flashEnvironments") or []} environment = environments.get(env_name) if environment is None: environment = await client.create_environment(app_id, env_name) @@ -549,9 +494,7 @@ async def deploy_app( await client.upload_tarball( upload["uploadUrl"], str(tar_path), progress=upload_progress ) - build = await client.finalize_artifact_upload( - app_id, upload["objectKey"], manifest - ) + build = await client.finalize_artifact_upload(app_id, upload["objectKey"], manifest) log.info("uploaded build %s", build["id"]) await client.deploy_build(environment["id"], build["id"]) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 8a0d3cbed..1e81398e5 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -6,16 +6,15 @@ api is the only source of truth; nothing is persisted locally. """ -import asyncio import base64 import logging -import sys from pathlib import Path from typing import Dict, List, Optional, Union from .api import AppsApiClient from .app import App from .handles import ApiHandle, FunctionHandle +from .utils.events import emit from .spec import ResourceKind, ResourceSpec from .targets import LiveTarget @@ -47,9 +46,7 @@ def _comparable(payload: Dict) -> Dict: clean.pop("id", None) template = clean.get("template") or {} template["env"] = [ - e - for e in template.get("env") or [] - if e.get("key") != GENERATION_ENV + e for e in template.get("env") or [] if e.get("key") != GENERATION_ENV ] return clean @@ -74,20 +71,21 @@ def _image_for(spec: ResourceSpec) -> str: """dev worker image: custom image, env override, or the builtin runtime image matched to the local python (dev requests carry pickled source, so client and worker versions should align).""" + if spec.image: return spec.image + override = _os.getenv("RUNPOD_DEV_IMAGE") if override: return override + from .images import image_for_spec, local_python_version return image_for_spec(spec, python_version=local_python_version()) def _bootstrap_source() -> str: - return ( - Path(__file__).parent.parent / "runtimes" / "bootstrap.py" - ).read_text() + return (Path(__file__).parent.parent / "runtimes" / "bootstrap.py").read_text() def _bootstrap_docker_args() -> str: @@ -126,6 +124,7 @@ def _cpu_locations(instance_ids: List[str]) -> str: return ",".join(dc.value for dc in CPU5_DATACENTERS) return ",".join(dc.value for dc in CPU3_DATACENTERS) + # template env var bumped on refresh; env is a version-triggering template # property, so changing it recreates all workers server-side GENERATION_ENV = "RUNPOD_DEV_GENERATION" @@ -137,6 +136,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: the template is nested in the saveEndpoint input so it is bound to the endpoint and cascades on deleteEndpoint. """ + payload: Dict = { "name": dev_endpoint_name(app.name, spec.name), "workersMin": spec.workers[0], @@ -148,8 +148,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: "template": { "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), - "containerDiskInGb": spec.container_disk_gb - or (10 if spec.is_cpu else 30), + "containerDiskInGb": spec.container_disk_gb or (10 if spec.is_cpu else 30), "dockerArgs": "", "env": [ {"key": GENERATION_ENV, "value": str(generation)}, @@ -172,13 +171,11 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: if spec.max_concurrency > 1 else [] ), - *( - {"key": k, "value": v} - for k, v in _render_env(spec.env).items() - ), + *({"key": k, "value": v} for k, v in _render_env(spec.env).items()), ], }, } + if spec.flashboot: payload["flashBootType"] = "FLASHBOOT" @@ -189,9 +186,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: [ { "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode( - _bootstrap_source().encode() - ).decode(), + "value": base64.b64encode(_bootstrap_source().encode()).decode(), }, {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, ] @@ -252,9 +247,7 @@ def __init__( self._volume_resolver = None def _emit(self, event: str, *args) -> None: - handler = getattr(self.events, event, None) - if handler is not None: - handler(*args) + emit(self.events, event, *args) async def _attach_volumes(self, payload: Dict, spec, app) -> None: """resolve the resource's volumes and registry auth onto a @@ -264,9 +257,7 @@ async def _attach_volumes(self, payload: Dict, spec, app) -> None: from .volume import VolumeResolver if spec.registry_auth: - auth_id = await resolve_registry_auth( - spec.registry_auth, api=self.api - ) + auth_id = await resolve_registry_auth(spec.registry_auth, api=self.api) payload["template"]["containerRegistryAuthId"] = auth_id if spec.model: from .model import model_reference @@ -275,20 +266,14 @@ async def _attach_volumes(self, payload: Dict, spec, app) -> None: if not spec.volume: return if self._volume_resolver is None: - self._volume_resolver = VolumeResolver( - self.api, events=self.events - ) - await attach_endpoint_volumes( - payload, spec, self._volume_resolver, app - ) + self._volume_resolver = VolumeResolver(self.api, events=self.events) + await attach_endpoint_volumes(payload, spec, self._volume_resolver, app) @property def _endpoint_ids(self) -> List[str]: return list(self._endpoints.values()) - def _provisionable( - self, app: App - ) -> List[Union[FunctionHandle, ApiHandle]]: + def _provisionable(self, app: App) -> List[Union[FunctionHandle, ApiHandle]]: """endpoints only; tasks have no standing infra to manage.""" return [ h @@ -332,9 +317,7 @@ async def start(self) -> None: endpoint_id = result["id"] log.info("adopted dev endpoint %s (%s)", name, endpoint_id) else: - self._emit( - "provisioning", spec.name, spec.kind.value, hardware - ) + self._emit("provisioning", spec.name, spec.kind.value, hardware) result = await self.api.save_endpoint(payload) endpoint_id = result["id"] log.info("provisioned dev endpoint %s (%s)", name, endpoint_id) @@ -359,6 +342,7 @@ async def refresh(self, apps: List[App]) -> None: property, so the platform recreates all workers and subsequent requests execute in fresh processes. added resources are provisioned, removed ones deleted.""" + self.generation += 1 self.apps = apps for app in apps: @@ -388,13 +372,17 @@ async def refresh(self, apps: List[App]) -> None: for name, (app, handle) in desired.items(): payload = _endpoint_input(app, handle.spec, self.generation) await self._attach_volumes(payload, handle.spec, app) + existing_id = self._endpoints.get(name) comparable = _comparable(payload) previous = self._payloads.get(name) + if existing_id: payload["id"] = existing_id + result = await self.api.save_endpoint(payload) endpoint_id = result["id"] + self._endpoints[name] = endpoint_id self._payloads[name] = comparable app._dev_targets[handle.spec.name] = LiveTarget( @@ -403,18 +391,19 @@ async def refresh(self, apps: List[App]) -> None: events=self.events, metrics_key=result.get("aiKey"), ) + spec = handle.spec hardware = ",".join(spec.cpu or spec.gpu or ["any"]) + if previous is None: - self._emit( - "resource_added", spec.name, spec.kind.value, hardware - ) + self._emit("resource_added", spec.name, spec.kind.value, hardware) elif previous != comparable: self._emit( "resource_changed", spec.name, _changed_fields(previous, comparable), ) + log.info( "refreshed dev endpoint %s (%s, generation %d)", name, @@ -431,25 +420,18 @@ async def stop(self, events: Optional[object] = None) -> None: """ sink = events if events is not None else self.events - def _tell(event: str, *args) -> None: - handler = getattr(sink, event, None) - if handler is not None: - handler(*args) - pending = list(self._endpoints.items()) - _tell("cleanup_started", len(pending)) + emit(sink, "cleanup_started", len(pending)) for name, endpoint_id in pending: resource = _resource_of(name) - _tell("deleting", resource) + emit(sink, "deleting", resource) try: await self.api.delete_endpoint(endpoint_id) - _tell("deleted", resource) + emit(sink, "deleted", resource) log.info("deleted dev endpoint %s (%s)", name, endpoint_id) except Exception as exc: - _tell("delete_failed", resource) - log.warning( - "failed to delete dev endpoint %s: %s", endpoint_id, exc - ) + emit(sink, "delete_failed", resource) + log.warning("failed to delete dev endpoint %s: %s", endpoint_id, exc) self._endpoints.clear() for app in self.apps: app._dev_targets.clear() diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index a61a8eff7..fabed41b1 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -40,6 +40,7 @@ def __init__(self, resource_name: str): f"has no side effects." ) + _SKIP_DIRS = { ".git", ".venv", @@ -144,13 +145,13 @@ def discover_apps(target: Path) -> List[App]: sys.path.remove(root_str) found = [a for a in get_registered_apps() if id(a) not in before] - # a file imported both by the scan (synthetic name) and by another - # file (real name, e.g. `from main import q1`) registers its app - # twice; deploying duplicates would be wasteful and confusing + + # dedupe by_name = {} for app in found: by_name.setdefault(app.name, app) found = list(by_name.values()) + if not found and failures: raise DiscoveryError( "no runpod.App found; some files failed to import:\n " diff --git a/runpod/apps/images.py b/runpod/apps/images.py index 122b4903c..10d1400f5 100644 --- a/runpod/apps/images.py +++ b/runpod/apps/images.py @@ -17,7 +17,7 @@ from .spec import ResourceKind, ResourceSpec # policy: non-EOL cpython versions with torch wheel support. 3.10 -# ages out at its 2026-10 EOL; new versions join once torch ships +# ages out at its 2026-10 EOL. new versions join once torch ships # wheels for them. SUPPORTED_PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") DEFAULT_PYTHON_VERSION = "3.12" @@ -73,12 +73,8 @@ def runtime_image( return f"{repo}:py{version}-{runtime_tag()}" -def image_for_spec( - spec: ResourceSpec, *, python_version: Optional[str] = None -) -> str: +def image_for_spec(spec: ResourceSpec, *, python_version: Optional[str] = None) -> str: """the image a resource runs on: its custom image or the builtin.""" if spec.image: return spec.image - return runtime_image( - spec.kind, gpu=not spec.is_cpu, python_version=python_version - ) + return runtime_image(spec.kind, gpu=not spec.is_cpu, python_version=python_version) diff --git a/runpod/apps/manage.py b/runpod/apps/manage.py index 7745fe4e8..9c260e1fb 100644 --- a/runpod/apps/manage.py +++ b/runpod/apps/manage.py @@ -11,6 +11,7 @@ from .api import AppsApiClient from .errors import AppError +from .utils.events import emit log = logging.getLogger(__name__) @@ -82,25 +83,20 @@ async def undeploy_environment( client = api or AppsApiClient() env = await get_environment(app_name, env_name, api=client) - def _tell(event: str, *args) -> None: - handler = getattr(events, event, None) - if handler is not None: - handler(*args) - result = UndeployResult() endpoints = env.get("endpoints") or [] - _tell("cleanup_started", len(endpoints)) + emit(events, "cleanup_started", len(endpoints)) for endpoint in endpoints: name = endpoint.get("name") or endpoint.get("id") - _tell("deleting", name) + emit(events, "deleting", name) try: await client.delete_endpoint(endpoint["id"]) result.endpoints_deleted += 1 - _tell("deleted", name) + emit(events, "deleted", name) log.info("deleted endpoint %s (%s)", name, endpoint["id"]) except Exception as exc: # noqa: BLE001 - collected for the caller result.failures.append(f"endpoint {name}: {exc}") - _tell("delete_failed", name) + emit(events, "delete_failed", name) if delete_env and not result.failures: try: diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index b1b067315..c8e8c25fc 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -14,6 +14,8 @@ from datetime import datetime, timezone from typing import Any, Dict, List, Optional +from .utils.events import emit + log = logging.getLogger(__name__) METRICS_POLL_INTERVAL = 2.0 @@ -57,16 +59,6 @@ def _filter_line(line: str) -> Optional[str]: return None -def emit(sink: Optional[object], event: str, *args: Any) -> None: - """invoke a handler on the sink if it exists.""" - handler = getattr(sink, event, None) - if handler is not None: - try: - handler(*args) - except Exception: # noqa: BLE001 - rendering must never break calls - log.debug("event sink %s failed", event, exc_info=True) - - class PodLogStream: """follows one pod's container logs and emits worker_log events. diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py index cbcdb0854..4f88ad772 100644 --- a/runpod/apps/placement.py +++ b/runpod/apps/placement.py @@ -14,6 +14,7 @@ from .datacenter import DataCenter from .errors import AppError from .gpu import GpuGroup +from .utils.client import default_client log = logging.getLogger(__name__) @@ -70,10 +71,7 @@ def __init__(self, api=None): self._fetched_cpu: Set[str] = set() async def _client(self): - if self._api is None: - from .api import AppsApiClient - - self._api = AppsApiClient() + self._api = default_client(self._api) return self._api async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: diff --git a/runpod/apps/registry.py b/runpod/apps/registry.py index cb054e1da..e237ef44d 100644 --- a/runpod/apps/registry.py +++ b/runpod/apps/registry.py @@ -13,6 +13,7 @@ def infer(): ... from typing import Optional from .errors import AppError +from .utils.lookup import find_by_id_or_name log = logging.getLogger(__name__) @@ -32,17 +33,11 @@ async def resolve_registry_auth( api = AppsApiClient() creds = await api.list_registry_auths() - for cred in creds: - if cred["id"] == name: - return cred["id"] - matches = [c for c in creds if c.get("name") == name] - if len(matches) > 1: - raise RegistryAuthError( - f"multiple registry credentials named '{name}' " - f"({', '.join(m['id'] for m in matches)}); reference by id" - ) - if matches: - return matches[0]["id"] + match = find_by_id_or_name( + creds, name, noun="registry credentials", error=RegistryAuthError + ) + if match: + return match["id"] available = ", ".join(sorted(c["name"] for c in creds)) or "(none)" raise RegistryAuthError( f"registry credential '{name}' not found. available: {available}. " diff --git a/runpod/apps/utils/__init__.py b/runpod/apps/utils/__init__.py new file mode 100644 index 000000000..5d1311e97 --- /dev/null +++ b/runpod/apps/utils/__init__.py @@ -0,0 +1 @@ +"""shared helpers for the apps surface.""" diff --git a/runpod/apps/utils/client.py b/runpod/apps/utils/client.py new file mode 100644 index 000000000..44cca305f --- /dev/null +++ b/runpod/apps/utils/client.py @@ -0,0 +1,16 @@ +"""lazy control-plane client construction.""" + +from typing import Optional + + +def default_client(api: Optional[object] = None): + """return api when given, else a fresh AppsApiClient. + + the import is deferred so modules that only sometimes touch the + control plane do not pull it in at import time. + """ + if api is not None: + return api + from ..api import AppsApiClient + + return AppsApiClient() diff --git a/runpod/apps/utils/events.py b/runpod/apps/utils/events.py new file mode 100644 index 000000000..90cef3623 --- /dev/null +++ b/runpod/apps/utils/events.py @@ -0,0 +1,20 @@ +"""duck-typed event sink dispatch.""" + +import logging +from typing import Any, Optional + +log = logging.getLogger(__name__) + + +def emit(sink: Optional[object], event: str, *args: Any) -> None: + """invoke the named handler on the sink if it exists. + + handler failures are swallowed and logged: a broken event + renderer must never break the operation it observes. + """ + handler = getattr(sink, event, None) + if handler is not None: + try: + handler(*args) + except Exception: # noqa: BLE001 - rendering must never break calls + log.debug("event sink %s failed", event, exc_info=True) diff --git a/runpod/apps/utils/lookup.py b/runpod/apps/utils/lookup.py new file mode 100644 index 000000000..7200dc944 --- /dev/null +++ b/runpod/apps/utils/lookup.py @@ -0,0 +1,26 @@ +"""shared name-or-id resolution for named platform resources.""" + +from typing import Any, Callable, Dict, List, Optional + + +def find_by_id_or_name( + items: List[Dict[str, Any]], + ref: str, + *, + noun: str, + error: Callable[[str], Exception], +) -> Optional[Dict[str, Any]]: + """find the item whose id equals ref, else the unique name match. + + returns None when nothing matches. raises error(message) when a + name matches more than one item, since the id is then needed to + disambiguate. + """ + for item in items: + if item["id"] == ref: + return item + matches = [item for item in items if item.get("name") == ref] + if len(matches) > 1: + ids = ", ".join(match["id"] for match in matches) + raise error(f"multiple {noun} named '{ref}' ({ids}); reference by id") + return matches[0] if matches else None diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index 4927b6904..610b19f87 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -22,6 +22,9 @@ def train(): from typing import Any, Dict, List, Optional from .errors import AppError +from .utils.client import default_client +from .utils.events import emit +from .utils.lookup import find_by_id_or_name log = logging.getLogger(__name__) @@ -109,10 +112,7 @@ def __init__(self, api=None, events: Optional[object] = None): self._stock = None async def _client(self): - if self._api is None: - from .api import AppsApiClient - - self._api = AppsApiClient() + self._api = default_client(self._api) return self._api async def resolve( @@ -127,25 +127,14 @@ async def resolve( if cached is not None: return cached - from .monitor import emit from .placement import PlacementError, StockMap, solve_placement client = await self._client() existing = await client.list_network_volumes() - record = None - for entry in existing: - if entry["id"] == volume.name: - record = entry - break - if record is None: - matches = [e for e in existing if e.get("name") == volume.name] - if len(matches) > 1: - raise VolumeError( - f"multiple volumes named '{volume.name}' " - f"({', '.join(m['id'] for m in matches)}); reference by id" - ) - record = matches[0] if matches else None + record = find_by_id_or_name( + existing, volume.name, noun="volumes", error=VolumeError + ) if self._stock is None: self._stock = StockMap(client) From 68df2cee1516057b750240da76d4fbf57a8608b8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:35:02 -0700 Subject: [PATCH 088/120] feat: lazy top-level imports and curated runtime vendoring --- runpod/__init__.py | 150 ++++++++++++++++++++++++----------------- runpod/apps/build.py | 102 ++++++++++++++++++---------- runpod/cli/__init__.py | 2 - 3 files changed, 158 insertions(+), 96 deletions(-) diff --git a/runpod/__init__.py b/runpod/__init__.py index 5303284fa..56c2a7176 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -1,55 +1,70 @@ """Allows runpod to be imported as a module.""" +import importlib import logging import os -from . import serverless -from .api.ctl_commands import ( - create_container_registry_auth, - create_endpoint, - create_pod, - create_template, - delete_container_registry_auth, - get_endpoints, - get_gpu, - get_gpus, - get_pod, - get_pods, - get_user, - resume_pod, - stop_pod, - terminate_pod, - update_container_registry_auth, - update_endpoint_template, - update_user_settings, -) -from .cli.groups.config.functions import ( - check_credentials, - get_credentials, - set_credentials, -) -from .endpoint import AsyncioEndpoint, AsyncioJob, Endpoint -from .apps import ( - Api, - App, - DataCenter, - EndpointNotFound, - Job, - Model, - Queue, - Secret, - Volume, - is_local, - local_entrypoint, - schedule, -) -from .apps.gpu import CpuInstanceType, GpuGroup, GpuType -from .apps.markers import delete, get, init, patch, post, put -from .serverless.modules.rp_logger import RunPodLogger from .version import __version__ +# public names resolved on first access (PEP 562). keeping the top-level +# import light means `import runpod` (and the deployed worker that does +# it) pulls only what it actually touches, not the whole sdk/cli closure. +_LAZY_ATTRS = { + # api control-plane commands + "create_container_registry_auth": "runpod.api.ctl_commands", + "create_endpoint": "runpod.api.ctl_commands", + "create_pod": "runpod.api.ctl_commands", + "create_template": "runpod.api.ctl_commands", + "delete_container_registry_auth": "runpod.api.ctl_commands", + "get_endpoints": "runpod.api.ctl_commands", + "get_gpu": "runpod.api.ctl_commands", + "get_gpus": "runpod.api.ctl_commands", + "get_pod": "runpod.api.ctl_commands", + "get_pods": "runpod.api.ctl_commands", + "get_user": "runpod.api.ctl_commands", + "resume_pod": "runpod.api.ctl_commands", + "stop_pod": "runpod.api.ctl_commands", + "terminate_pod": "runpod.api.ctl_commands", + "update_container_registry_auth": "runpod.api.ctl_commands", + "update_endpoint_template": "runpod.api.ctl_commands", + "update_user_settings": "runpod.api.ctl_commands", + # config helpers + "check_credentials": "runpod.cli.groups.config.functions", + "get_credentials": "runpod.cli.groups.config.functions", + "set_credentials": "runpod.cli.groups.config.functions", + # endpoint clients + "AsyncioEndpoint": "runpod.endpoint", + "AsyncioJob": "runpod.endpoint", + "Endpoint": "runpod.endpoint", + # apps surface + "Api": "runpod.apps", + "App": "runpod.apps", + "DataCenter": "runpod.apps", + "EndpointNotFound": "runpod.apps", + "Job": "runpod.apps", + "Model": "runpod.apps", + "Queue": "runpod.apps", + "Secret": "runpod.apps", + "Volume": "runpod.apps", + "is_local": "runpod.apps", + "local_entrypoint": "runpod.apps", + "schedule": "runpod.apps", + "CpuInstanceType": "runpod.apps.gpu", + "GpuGroup": "runpod.apps.gpu", + "GpuType": "runpod.apps.gpu", + "delete": "runpod.apps.markers", + "get": "runpod.apps.markers", + "init": "runpod.apps.markers", + "patch": "runpod.apps.markers", + "post": "runpod.apps.markers", + "put": "runpod.apps.markers", + # logger + "RunPodLogger": "runpod.serverless.modules.rp_logger", + # submodule + "serverless": "runpod.serverless", +} + __all__ = [ - # API functions "create_container_registry_auth", "create_endpoint", "create_pod", @@ -67,15 +82,12 @@ "update_container_registry_auth", "update_endpoint_template", "update_user_settings", - # Config functions "check_credentials", "get_credentials", "set_credentials", - # Endpoint classes "AsyncioEndpoint", "AsyncioJob", "Endpoint", - # Apps surface "Api", "App", "CpuInstanceType", @@ -94,33 +106,51 @@ "post", "put", "schedule", - # Serverless module "serverless", - # Logger class "RunPodLogger", - # Version "__version__", - # Module variables "SSH_KEY_PATH", "profile", "api_key", "endpoint_url_base", ] + +def _load_api_key(): + """resolve the api key from the environment, then stored credentials. + + the credential lookup is deferred (and tolerant of a missing cli + dependency closure) so a bare `import runpod` never drags in the + config/cli modules; the deployed worker gets its key from the env. + """ + key = os.environ.get("RUNPOD_API_KEY") + if key: + return key + try: + creds = importlib.import_module( + "runpod.cli.groups.config.functions" + ).get_credentials(profile) + except Exception: # noqa: BLE001 - no stored key is a valid state + return None + return creds["api_key"] if creds else None + + +def __getattr__(name): + module_path = _LAZY_ATTRS.get(name) + if module_path is None: + raise AttributeError(f"module 'runpod' has no attribute {name!r}") + module = importlib.import_module(module_path) + value = module if module_path == f"runpod.{name}" else getattr(module, name) + globals()[name] = value + return value + + # ------------------------------- Config Paths ------------------------------- # SSH_KEY_PATH = os.path.expanduser("~/.runpod/ssh") - profile = "default" # pylint: disable=invalid-name -if os.environ.get("RUNPOD_API_KEY"): - api_key = os.environ.get("RUNPOD_API_KEY") -else: - _credentials = get_credentials(profile) - if _credentials is not None: - api_key = _credentials["api_key"] # pylint: disable=invalid-name - else: - api_key = None # pylint: disable=invalid-name +api_key = _load_api_key() # pylint: disable=invalid-name endpoint_url_base = os.environ.get( "RUNPOD_ENDPOINT_BASE_URL", "https://api.runpod.ai/v2" diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 7e7b4c068..477b15364 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -1,20 +1,4 @@ -"""deploy-time environment build: vendor dependencies into the artifact. - -cold starts do no dependency resolution. everything a resource needs to -run (including the runpod runtime itself) is resolved on the build -machine into a site-packages tree that ships inside the artifact under -env/. workers put it on sys.path and start. - -the exception is size-prohibitive CUDA packages (torch and friends) -that the gpu worker images already provide: vendoring them would add -gigabytes to every artifact for nothing. they are excluded here, -recorded in the manifest, and the worker bootstrap verifies they exist -in the image at startup (installing only if genuinely absent). - -wheels are selected for the worker platform (manylinux/amd64) and the -target python version, not the build machine's, so a mac or windows -build produces a linux-correct environment. -""" +"""deploy-time environment build: vendor dependencies into the artifact.""" import logging import os @@ -51,6 +35,26 @@ } ) +# the worker runtime's actual dependency closure, a small subset of the +# runpod package's full dependency list. the runpod package is vendored +# without its dependencies (sdk/cli-only packages like boto3, paramiko, +# and cryptography never load in a worker), so these must be vendored +# explicitly to keep a dependency-free worker small. +RUNTIME_REQUIREMENTS = ( + "aiohttp[speedups]", + "aiohttp-retry", + "backoff", + "cloudpickle", + "py-cpuinfo", + "requests", + "tomli", + "tomlkit", + "tqdm-loggable", +) + +# api workers additionally serve over fastapi/uvicorn +API_RUNTIME_REQUIREMENTS = ("fastapi[standard]", "uvicorn>=0.30") + # runpod's serverless limit for build artifacts MAX_ARTIFACT_MB = 1500 @@ -136,7 +140,8 @@ def _is_pinnable_name(spec: str) -> bool: """true if the spec is a plain requirement (name, optional pin), as opposed to a url, git ref, or filesystem path.""" return not any(token in spec for token in ("/", "\\", ":", "@")) or bool( - _REQ_NAME_RE.match(spec) and re.match(r"^[A-Za-z0-9._-]+(\[[^\]]+\])?([<>=!~;].*)?$", spec) + _REQ_NAME_RE.match(spec) + and re.match(r"^[A-Za-z0-9._-]+(\[[^\]]+\])?([<>=!~;].*)?$", spec) ) @@ -180,8 +185,29 @@ def sync_running_package(env_dir: Path) -> None: ) +def _ensure_pip() -> None: + """make `python -m pip` work in venvs created without pip (uv).""" + probe = subprocess.run( + [sys.executable, "-m", "pip", "--version"], + capture_output=True, + ) + if probe.returncode == 0: + return + bootstrap = subprocess.run( + [sys.executable, "-m", "ensurepip", "--upgrade"], + capture_output=True, + text=True, + ) + if bootstrap.returncode != 0: + raise BuildError( + f"pip is not available in this environment and ensurepip " + f"failed: {bootstrap.stderr[-500:]}" + ) + + def _build_wheel(spec: str, scratch_dir: Path) -> Path: """build a wheel from a url/path spec for binary-only vendoring.""" + _ensure_pip() wheel_dir = scratch_dir / "wheels" wheel_dir.mkdir(parents=True, exist_ok=True) env = dict(os.environ) @@ -313,22 +339,34 @@ def build_environment( # size-prohibitive packages come preinstalled on the builtin gpu # images; a custom image carries no such guarantee, so any custom # image in the app disables auto-exclusion and everything vendors - auto_exclude = not any( - h.spec.image for h in app.resources.values() - ) + auto_exclude = not any(h.spec.image for h in app.resources.values()) vendored, excluded = split_exclusions( user_requirements, exclude, auto_exclude=auto_exclude ) - runtime = [runtime_requirement(scratch), "cloudpickle"] from .spec import ResourceKind - if any( - h.spec.kind is ResourceKind.API for h in app.resources.values() - ): - runtime.append("uvicorn>=0.30") + runtime_deps = list(RUNTIME_REQUIREMENTS) + if any(h.spec.kind is ResourceKind.API for h in app.resources.values()): + runtime_deps.extend(API_RUNTIME_REQUIREMENTS) + + package_spec = os.environ.get("RUNPOD_PACKAGE_SPEC") + progress = getattr(events, "vendor_progress", None) + + # the runpod package ships without its own dependency closure; the + # worker runtime pulls only runtime_deps. with an override the pinned + # version is installed no-deps; otherwise the client's own package + # tree is overlaid below, so no index install of runpod is needed. + if package_spec: + vendor( + env_dir, + [runtime_requirement(scratch)], + python_version, + no_deps=True, + progress=progress, + ) - all_requirements = runtime + vendored + all_requirements = runtime_deps + vendored log.info( "vendoring %d packages for python %s (%d excluded: %s)", len(all_requirements), @@ -336,13 +374,9 @@ def build_environment( len(excluded), ", ".join(excluded) or "none", ) - vendor( - env_dir, - all_requirements, - python_version, - progress=getattr(events, "vendor_progress", None), - ) - if not os.environ.get("RUNPOD_PACKAGE_SPEC"): + vendor(env_dir, all_requirements, python_version, progress=progress) + + if not package_spec: sync_running_package(env_dir) _verify_runtime(env_dir) diff --git a/runpod/cli/__init__.py b/runpod/cli/__init__.py index 08c52a52d..a76134899 100644 --- a/runpod/cli/__init__.py +++ b/runpod/cli/__init__.py @@ -2,8 +2,6 @@ import threading -from .groups import config as config, ssh as ssh - STOP_EVENT = threading.Event() From 2a47e0ea979c76ed4e461f2b8efd5d78ca4f4f4a Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:35:02 -0700 Subject: [PATCH 089/120] feat: per-job queue options via with_options --- runpod/apps/handles.py | 45 ++++++++++++++-- runpod/apps/stubs.py | 44 +++++++++++++--- runpod/apps/targets.py | 43 +++++++++++++++ tests/test_apps/test_dispatch.py | 89 ++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 9 deletions(-) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index bec331411..08e8fca34 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -55,6 +55,8 @@ def __init__( self._fn = fn self.spec = spec self._init_fn: Optional[Callable] = None + # per-job options merged into the run payload; set by with_options + self._job_options: Dict[str, Any] = {} # adopt a schedule stamped by @schedule below the app decorator stamped = getattr(fn, SCHEDULE_ATTR, None) @@ -70,6 +72,38 @@ def __init__( self.__doc__ = getattr(fn, "__doc__", None) self.__wrapped__ = fn + def with_options( + self, + *, + webhook: Optional[str] = None, + execution_timeout: Optional[int] = None, + ttl: Optional[int] = None, + low_priority: Optional[bool] = None, + s3_config: Optional[Dict[str, Any]] = None, + ) -> "FunctionHandle": + """bind per-job options for the next call, returning a new handle. + + transcribe.with_options(webhook="https://...").spawn(url) + + options apply per invocation and stack across chained calls; + the original handle is untouched. queue only: tasks run on a + dedicated pod and take no job payload options. + """ + from .targets import build_job_options, merge_job_options + + if self.spec.kind is ResourceKind.TASK: + raise InvalidResourceError( + "per-job options apply only to @app.queue functions; " + "tasks run on a dedicated pod" + ) + new_options = build_job_options( + webhook, execution_timeout, ttl, low_priority, s3_config + ) + clone = FunctionHandle(self._app, self._fn, self.spec) + clone._init_fn = self._init_fn + clone._job_options = merge_job_options(self._job_options, new_options) + return clone + def init(self, fn: Callable) -> Callable: """register a worker-startup hook; runs before the worker is ready, never locally.""" @@ -100,7 +134,7 @@ async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) - return await target.invoke(payload) + return await target.invoke(self._apply_options(payload)) async def _stream_async(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: """invoke a generator function remotely, yielding partial outputs @@ -121,7 +155,7 @@ async def _stream_async(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) - data = await target.submit(payload) + data = await target.submit(self._apply_options(payload)) async for chunk in target.stream_job(data["id"]): yield chunk @@ -139,11 +173,16 @@ def _guard_generator(self) -> None: f"use .remote(...) instead" ) + def _apply_options(self, payload: Dict[str, Any]) -> Dict[str, Any]: + if not self._job_options: + return payload + return {**payload, **self._job_options} + async def _spawn_async(self, *args: Any, **kwargs: Any) -> Any: self._guard_discovery() target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) - data = await target.submit(payload) + data = await target.submit(self._apply_options(payload)) # queue targets return raw job data; task targets return a TaskJob if isinstance(data, dict): return Job(data, target) diff --git a/runpod/apps/stubs.py b/runpod/apps/stubs.py index c81b66923..04579d019 100644 --- a/runpod/apps/stubs.py +++ b/runpod/apps/stubs.py @@ -51,25 +51,57 @@ class Queue(_StubBase): def __init__(self, **kwargs: Any): super().__init__(**kwargs) + self._job_options: dict = {} self.remote = Invoker(self._remote_async) self.stream = StreamInvoker(self._stream_async) self.spawn = Invoker(self._spawn_async) self.job = Invoker(self._job_async) - async def _remote_async(self, **kwargs: Any) -> Any: + def with_options( + self, + *, + webhook: Optional[str] = None, + execution_timeout: Optional[int] = None, + ttl: Optional[int] = None, + low_priority: Optional[bool] = None, + s3_config: Optional[dict] = None, + ) -> "Queue": + """bind per-job options for the next call, returning a new stub. + + options apply per invocation and stack across chained calls; the + original stub is untouched. + """ + from .targets import build_job_options, merge_job_options + + new_options = build_job_options( + webhook, execution_timeout, ttl, low_priority, s3_config + ) + clone = Queue( + app=self.app_name, + name=self.resource_name, + id=self.resource_id, + env=self.env, + ) + clone._job_options = merge_job_options(self._job_options, new_options) + return clone + + def _payload(self, kwargs: dict) -> dict: payload = {"input": kwargs or {"__empty": True}} - return await self._target().invoke(payload) + if self._job_options: + payload.update(self._job_options) + return payload + + async def _remote_async(self, **kwargs: Any) -> Any: + return await self._target().invoke(self._payload(kwargs)) async def _spawn_async(self, **kwargs: Any) -> Job: - payload = {"input": kwargs or {"__empty": True}} target = self._target() - data = await target.submit(payload) + data = await target.submit(self._payload(kwargs)) return Job(data, target) async def _stream_async(self, **kwargs: Any) -> AsyncIterator[Any]: - payload = {"input": kwargs or {"__empty": True}} target = self._target() - data = await target.submit(payload) + data = await target.submit(self._payload(kwargs)) async for chunk in target.stream_job(data["id"]): yield chunk diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 067201ba9..1f84e60d4 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -85,6 +85,49 @@ def args_to_input(fn: Callable, args: tuple, kwargs: dict) -> Dict[str, Any]: return body +def build_job_options( + webhook: Optional[str] = None, + execution_timeout: Optional[int] = None, + ttl: Optional[int] = None, + low_priority: Optional[bool] = None, + s3_config: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """shape per-job queue options into the wire payload. + + these ride alongside `input` on the run/runsync request, matching + the raw data-plane api's job payload. + """ + options: Dict[str, Any] = {} + if webhook is not None: + options["webhook"] = webhook + policy: Dict[str, Any] = {} + if execution_timeout is not None: + policy["executionTimeout"] = execution_timeout + if ttl is not None: + policy["ttl"] = ttl + if low_priority is not None: + policy["lowPriority"] = low_priority + if policy: + options["policy"] = policy + if s3_config is not None: + options["s3Config"] = s3_config + return options + + +def merge_job_options( + base: Dict[str, Any], new: Dict[str, Any] +) -> Dict[str, Any]: + """combine two option sets; policy fields accumulate rather than + overwrite so chained with_options calls compose.""" + merged = dict(base) + for key, value in new.items(): + if key == "policy" and isinstance(merged.get("policy"), dict): + merged["policy"] = {**merged["policy"], **value} + else: + merged[key] = value + return merged + + def unwrap_job_output(data: Dict[str, Any]) -> Any: """extract output from a runsync-style response, raising on failure.""" if data.get("status") == "FAILED" or data.get("error"): diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index 19c0f82af..a6dd03889 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -129,6 +129,76 @@ async def q(x): assert result == {"ok": True} + def test_remote_with_options_merges_payload(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = {"ok": 1} + q.with_options( + webhook="https://hook", execution_timeout=5000, low_priority=True + ).remote(2) + + mock_invoke.assert_awaited_once_with( + { + "input": {"x": 2}, + "webhook": "https://hook", + "policy": {"executionTimeout": 5000, "lowPriority": True}, + } + ) + + def test_spawn_with_options_merges_payload(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + with patch.object( + SentinelTarget, "submit", new_callable=AsyncMock + ) as mock_submit: + mock_submit.return_value = {"id": "job-1", "status": "IN_QUEUE"} + q.with_options(s3_config={"bucketName": "b"}).spawn(1) + + mock_submit.assert_awaited_once_with( + {"input": {"x": 1}, "s3Config": {"bucketName": "b"}} + ) + + def test_with_options_returns_new_handle(self, monkeypatch): + for var in ("RUNPOD_ENDPOINT_ID", "RUNPOD_POD_ID", "RUNPOD_DEV_SESSION"): + monkeypatch.delenv(var, raising=False) + app = App("my-app") + + @app.queue(name="q") + def q(x): + return x + + bound = q.with_options(webhook="https://hook") + assert bound is not q + assert q._job_options == {} + # policy fields deep-merge across chained calls + chained = q.with_options(execution_timeout=1).with_options(ttl=2) + assert chained._job_options == {"policy": {"executionTimeout": 1, "ttl": 2}} + + def test_with_options_rejects_tasks(self): + app = App("my-app") + + @app.task(name="t") + def t(x): + return x + + with pytest.raises(InvalidResourceError, match="only to @app.queue"): + t.with_options(webhook="https://hook") + def test_worker_executes_own_body(self, monkeypatch): monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") @@ -243,6 +313,25 @@ def test_queue_stub_remote(self): assert result == {"ok": 1} mock_invoke.assert_awaited_once_with({"input": {"prompt": "hi"}}) + def test_queue_stub_with_options(self): + stub = runpod.Queue(app="other-app", name="q") + bound = stub.with_options(webhook="https://hook", ttl=1000) + assert bound is not stub + assert stub._job_options == {} + with patch.object( + SentinelTarget, "invoke", new_callable=AsyncMock + ) as mock_invoke: + mock_invoke.return_value = {"ok": 1} + bound.remote(prompt="hi") + + mock_invoke.assert_awaited_once_with( + { + "input": {"prompt": "hi"}, + "webhook": "https://hook", + "policy": {"ttl": 1000}, + } + ) + def test_queue_stub_spawn_returns_job(self): stub = runpod.Queue(app="other-app", name="q") with patch.object( From e8521ed296c2abab8c8510d56eb0c5d80e556bdc Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:35:02 -0700 Subject: [PATCH 090/120] test: shell example runner with dev, deploy, and local modes --- tests/e2e/examples/01_hello_world.py | 4 + tests/e2e/examples/02_invocation_styles.py | 4 + tests/e2e/examples/03_gpu.py | 4 + tests/e2e/examples/04_dependencies.py | 4 + tests/e2e/examples/05_api_service.py | 4 + tests/e2e/examples/06_tasks.py | 4 + tests/e2e/examples/07_pipelines.py | 4 + tests/e2e/examples/08_volumes.py | 4 + tests/e2e/examples/09_secrets.py | 4 + tests/e2e/examples/10_cached_models.py | 4 + tests/e2e/examples/11_train_eval.py | 4 + tests/e2e/examples/12_custom_image.py | 4 + tests/e2e/examples/README.md | 24 ++++- tests/e2e/examples/run_all.py | 77 ------------- tests/e2e/examples/run_all.sh | 119 +++++++++++++++++++++ tests/e2e/matrix/main.py | 16 +++ tests/e2e/matrix/run.py | 29 +++++ 17 files changed, 231 insertions(+), 82 deletions(-) delete mode 100644 tests/e2e/examples/run_all.py create mode 100755 tests/e2e/examples/run_all.sh diff --git a/tests/e2e/examples/01_hello_world.py b/tests/e2e/examples/01_hello_world.py index 5b64bb121..fc14acaa6 100644 --- a/tests/e2e/examples/01_hello_world.py +++ b/tests/e2e/examples/01_hello_world.py @@ -20,3 +20,7 @@ def main(): result = hello.remote("world") print("result:", result) assert result == "hello world" + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/02_invocation_styles.py b/tests/e2e/examples/02_invocation_styles.py index 30ae03a37..9aeb52197 100644 --- a/tests/e2e/examples/02_invocation_styles.py +++ b/tests/e2e/examples/02_invocation_styles.py @@ -35,3 +35,7 @@ async def go(): assert square.local(7) == 49 print("all four invocation styles ok") + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/03_gpu.py b/tests/e2e/examples/03_gpu.py index 993bd8857..055cd9e71 100644 --- a/tests/e2e/examples/03_gpu.py +++ b/tests/e2e/examples/03_gpu.py @@ -31,3 +31,7 @@ def main(): assert info["cuda"] is True assert "4090" in info["device"] assert info["mode"] == "example" + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/04_dependencies.py b/tests/e2e/examples/04_dependencies.py index a7ad41f36..4c06728e1 100644 --- a/tests/e2e/examples/04_dependencies.py +++ b/tests/e2e/examples/04_dependencies.py @@ -32,3 +32,7 @@ def main(): version = jq_version.remote() print("jq:", version) assert version.startswith("jq-") + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/05_api_service.py b/tests/e2e/examples/05_api_service.py index f5b460639..b918d3e26 100644 --- a/tests/e2e/examples/05_api_service.py +++ b/tests/e2e/examples/05_api_service.py @@ -39,3 +39,7 @@ def main(): value = Counter.get("/value") print("value:", value) assert value["count"] == 5 + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/06_tasks.py b/tests/e2e/examples/06_tasks.py index e60b00765..af12fb345 100644 --- a/tests/e2e/examples/06_tasks.py +++ b/tests/e2e/examples/06_tasks.py @@ -29,3 +29,7 @@ def main(): info = gpu_task.remote() print("gpu task:", info) assert info["cuda"] is True + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/07_pipelines.py b/tests/e2e/examples/07_pipelines.py index 74519dc96..0fa0b8010 100644 --- a/tests/e2e/examples/07_pipelines.py +++ b/tests/e2e/examples/07_pipelines.py @@ -35,3 +35,7 @@ def main(): print("pipeline:", result) assert result["total"] == 4 assert result["tokens"][0] == "the" + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/08_volumes.py b/tests/e2e/examples/08_volumes.py index 4b6364eb1..106acdabe 100644 --- a/tests/e2e/examples/08_volumes.py +++ b/tests/e2e/examples/08_volumes.py @@ -37,3 +37,7 @@ def main(): result = read.remote() print("read back:", result) assert result == message + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/09_secrets.py b/tests/e2e/examples/09_secrets.py index 242060c26..f17170489 100644 --- a/tests/e2e/examples/09_secrets.py +++ b/tests/e2e/examples/09_secrets.py @@ -31,3 +31,7 @@ def main(): print("peek:", result) assert result["token_present"] is True assert result["mode"] == "example" + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/10_cached_models.py b/tests/e2e/examples/10_cached_models.py index 0032503dc..95aae5504 100644 --- a/tests/e2e/examples/10_cached_models.py +++ b/tests/e2e/examples/10_cached_models.py @@ -31,3 +31,7 @@ def main(): print("result:", result) assert result["path"].startswith("/runpod/model-store/huggingface/") assert result["text"] + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/11_train_eval.py b/tests/e2e/examples/11_train_eval.py index c988ba46c..022210a64 100644 --- a/tests/e2e/examples/11_train_eval.py +++ b/tests/e2e/examples/11_train_eval.py @@ -67,3 +67,7 @@ def main(): print("eval:", fit) assert abs(fit["weight"] - 3.0) < 0.3 assert abs(fit["bias"] - 1.0) < 0.3 + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/12_custom_image.py b/tests/e2e/examples/12_custom_image.py index 77782b069..15037d72e 100644 --- a/tests/e2e/examples/12_custom_image.py +++ b/tests/e2e/examples/12_custom_image.py @@ -22,3 +22,7 @@ def main(): version = python_version.remote() print("worker python:", version) assert version == "3.12" + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/examples/README.md b/tests/e2e/examples/README.md index 86c531346..d535d4f4b 100644 --- a/tests/e2e/examples/README.md +++ b/tests/e2e/examples/README.md @@ -1,13 +1,27 @@ # e2e example suite self-asserting apps exercising every sdk feature against production. -each file runs via `rp dev --once` and fails loudly when a -result is wrong, so the set doubles as a release gate. +each file fails loudly when a result is wrong, so the set doubles as a +release gate. every example carries both a `@runpod.local_entrypoint` +and an `if __name__ == "__main__"` block, so the same `main()` runs in +three modes: + +| mode | command | what it exercises | +| ------ | -------------------- | --------------------------------------------- | +| dev | `rp dev --once` | ephemeral endpoints, entrypoint, teardown | +| deploy | `rp deploy ` | persistent endpoints for the app | +| invoke | `python3 ` | `main()` against the deployed endpoints | + +`run_all.sh` runs every example through all three (the deploy phase does +deploy -> invoke -> undeploy per file, leaving nothing standing): ```bash -python tests/e2e/examples/run_all.py # everything -python tests/e2e/examples/run_all.py --jobs 4 # parallel -python tests/e2e/examples/run_all.py 01 07 # subset by prefix +./tests/e2e/examples/run_all.sh # everything, every mode +./tests/e2e/examples/run_all.sh 01 07 # subset by prefix + +SKIP_DEV=1 ./tests/e2e/examples/run_all.sh # deploy/invoke only +SKIP_DEPLOY=1 ./tests/e2e/examples/run_all.sh # dev only +KEEP=1 ./tests/e2e/examples/run_all.sh # leave endpoints up ``` prerequisites: diff --git a/tests/e2e/examples/run_all.py b/tests/e2e/examples/run_all.py deleted file mode 100644 index f1bca092f..000000000 --- a/tests/e2e/examples/run_all.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -"""run every example against prod as an end-to-end suite. - -each example is executed with `rp dev --once`, which provisions -real endpoints/pods, runs the example's local_entrypoint (every example -asserts on its results), and cleans up. requires `rp login` and, for -09_secrets, the demo secret: - - rp secret add ex-demo-secret --value anything - -usage: - python tests/e2e/examples/run_all.py # everything - python tests/e2e/examples/run_all.py 01 05 08 # by prefix - python tests/e2e/examples/run_all.py --jobs 4 # parallel -""" - -import argparse -import subprocess -import sys -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -HERE = Path(__file__).parent -TIMEOUT_SECONDS = 2400 - - -def find_examples(prefixes): - files = sorted( - p - for p in HERE.glob("[0-9][0-9]_*.py") - if not prefixes or any(p.name.startswith(pre) for pre in prefixes) - ) - if not files: - sys.exit(f"no examples match {prefixes}") - return files - - -def run_one(path: Path): - start = time.monotonic() - proc = subprocess.run( - ["rp", "dev", str(path), "--once"], - capture_output=True, - text=True, - timeout=TIMEOUT_SECONDS, - ) - elapsed = time.monotonic() - start - ok = proc.returncode == 0 - return path.name, ok, elapsed, proc.stdout + proc.stderr - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("prefixes", nargs="*", help="example prefixes, e.g. 01 08") - parser.add_argument("--jobs", type=int, default=1, help="parallel sessions") - args = parser.parse_args() - - files = find_examples(args.prefixes) - print(f"running {len(files)} examples (jobs={args.jobs})\n") - - results = [] - with ThreadPoolExecutor(max_workers=args.jobs) as pool: - for name, ok, elapsed, output in pool.map(run_one, files): - mark = "PASS" if ok else "FAIL" - print(f" {mark} {name} ({elapsed:.0f}s)") - results.append((name, ok, output)) - - failures = [(n, out) for n, ok, out in results if not ok] - print(f"\n{len(results) - len(failures)}/{len(results)} passed") - for name, output in failures: - print(f"\n--- {name} ---") - print("\n".join(output.splitlines()[-30:])) - sys.exit(1 if failures else 0) - - -if __name__ == "__main__": - main() diff --git a/tests/e2e/examples/run_all.sh b/tests/e2e/examples/run_all.sh new file mode 100755 index 000000000..619991555 --- /dev/null +++ b/tests/e2e/examples/run_all.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# run every example against prod as an end-to-end suite, in three modes. +# +# each example is a self-asserting app with both a +# @runpod.local_entrypoint and an `if __name__ == "__main__"` block, so +# the same main() runs under all three: +# +# dev rp dev --once ephemeral endpoints, run entrypoint, tear down +# deploy rp deploy persistent endpoints for the app +# invoke python3 run main() against the deployed endpoints +# +# the deploy phase runs deploy -> invoke -> undeploy per file, so no +# endpoints are left standing. requires `rp login` and, for 09_secrets: +# +# rp secret add ex-demo-secret --value anything +# +# output streams live so you see the exact cli output. +# +# usage: +# ./run_all.sh # every example, every mode +# ./run_all.sh 01 05 08 # subset by prefix +# +# env toggles: +# SKIP_DEV=1 skip the rp dev phase +# SKIP_DEPLOY=1 skip the rp deploy/python3 phase +# KEEP=1 leave deployed endpoints up (skip undeploy) + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +files=() +for f in "$HERE"/[0-9][0-9]_*.py; do + name="$(basename "$f")" + if [ "$#" -eq 0 ]; then + files+=("$f") + else + for pre in "$@"; do + if [[ "$name" == "$pre"* ]]; then + files+=("$f") + break + fi + done + fi +done + +if [ "${#files[@]}" -eq 0 ]; then + echo "no examples match: $*" >&2 + exit 1 +fi + +passed=() +failed=() + +app_name_of() { + sed -nE 's/.*App\("([^"]+)".*/\1/p' "$1" | head -1 +} + +# run one step; stream output live, record pass/fail as " ". +run_step() { + local mode="$1" name="$2" + shift 2 + echo "===================================================================" + echo ">>> [$mode] $name" + echo "===================================================================" + local start rc elapsed + start=$(date +%s) + "$@" + rc=$? + elapsed=$(( $(date +%s) - start )) + if [ "$rc" -eq 0 ]; then + echo "<<< PASS [$mode] $name (${elapsed}s)" + passed+=("$mode $name") + else + echo "<<< FAIL [$mode] $name (${elapsed}s, exit $rc)" + failed+=("$mode $name") + fi + echo + return "$rc" +} + +echo "running ${#files[@]} examples" +echo + +# ---------------------------------------------------------------- dev +if [ -z "${SKIP_DEV:-}" ]; then + for f in "${files[@]}"; do + run_step dev "$(basename "$f")" rp dev "$f" --once + done +fi + +# ------------------------------------------------- deploy -> invoke -> undeploy +if [ -z "${SKIP_DEPLOY:-}" ]; then + for f in "${files[@]}"; do + name="$(basename "$f")" + app="$(app_name_of "$f")" + + if run_step deploy "$name" rp deploy "$f"; then + run_step invoke "$name" python3 "$f" + if [ -z "${KEEP:-}" ] && [ -n "$app" ]; then + echo ">>> cleanup: rp undeploy -a $app -y" + rp undeploy -a "$app" -y || echo "!!! cleanup failed for $app" >&2 + echo + fi + fi + done +fi + +# ------------------------------------------------------------- summary +total=$(( ${#passed[@]} + ${#failed[@]} )) +echo "===================================================================" +echo "${#passed[@]}/${total} steps passed" +if [ "${#failed[@]}" -gt 0 ]; then + for s in "${failed[@]}"; do + echo " FAIL $s" + done +fi + +[ "${#failed[@]}" -eq 0 ] diff --git a/tests/e2e/matrix/main.py b/tests/e2e/matrix/main.py index a82298026..070022bcf 100644 --- a/tests/e2e/matrix/main.py +++ b/tests/e2e/matrix/main.py @@ -90,6 +90,12 @@ def t_mul(a: int, b: int): return {"product": a * b} +@app.task(name="t-gen", cpu="cpu3c-1-2") +def t_gen(n: int): + for i in range(n): + yield i * i + + @app.task(name="t-gpu", gpu=runpod.GpuGroup.ADA_24) def t_gpu(size: int): import torch @@ -115,3 +121,13 @@ def bump(self, body: dict): @runpod.get("/stats") def stats(self): return {"counter": self.counter} + + @runpod.get("/tokens") + def tokens(self): + from fastapi.responses import StreamingResponse + + def gen(): + for word in ("alpha", "beta", "gamma"): + yield word + "\n" + + return StreamingResponse(gen(), media_type="text/plain") diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py index 71e8bb795..125710c5e 100644 --- a/tests/e2e/matrix/run.py +++ b/tests/e2e/matrix/run.py @@ -22,6 +22,7 @@ q_gpu, q_stream, q_stream_async, + t_gen, t_gpu, t_mul, ) @@ -79,6 +80,13 @@ async def check_api(): return {"bump": r1, "stats": r2} +async def check_api_streaming_response(): + # api routes may stream; the client delivers the body intact + r = await Svc.get.aio("/tokens") + assert r == "alpha\nbeta\ngamma\n", repr(r) + return {"body": r} + + async def check_spawn(): job = await q_basic.spawn.aio(4) r = await job.result.aio() @@ -124,6 +132,24 @@ async def check_stream_from_job(): return chunks +async def check_task_generator_aggregates(): + r = await t_gen.remote.aio(4) + assert r == [0, 1, 4, 9], r + return r + + +async def check_task_stream_rejected(): + from runpod.apps.errors import InvalidResourceError + + try: + async for _ in t_gen.stream.aio(2): + pass + except InvalidResourceError as exc: + assert "do not stream" in str(exc), exc + return {"rejected": True} + raise AssertionError("task .stream() should raise InvalidResourceError") + + CHECKS = { "q-basic": check_q_basic, "q-deps": check_q_deps, @@ -139,6 +165,9 @@ async def check_stream_from_job(): "stream-async-gen": check_stream_async_gen, "stream-aggregate": check_stream_aggregate, "stream-from-job": check_stream_from_job, + "task-gen-aggregate": check_task_generator_aggregates, + "task-stream-rejected": check_task_stream_rejected, + "api-streaming-response": check_api_streaming_response, } From 8adc6c278257911aa1f92853b27c6fb710c2fb30 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:07:10 -0700 Subject: [PATCH 091/120] fix: address codeql and review findings across apps and runtimes --- runpod/__init__.py | 46 +++------------------------------- runpod/apps/api.py | 3 ++- runpod/apps/build.py | 11 +++++--- runpod/apps/dev.py | 16 +++++++++--- runpod/apps/discovery.py | 3 +++ runpod/apps/gpu.py | 1 + runpod/apps/targets.py | 3 ++- runpod/apps/tasks.py | 2 ++ runpod/apps/watch.py | 1 + runpod/rp_cli/main.py | 9 +++++++ runpod/rp_cli/update.py | 18 +++++++++++-- runpod/runtimes/api/server.py | 38 ++++++++++++++++++++++------ runpod/runtimes/executor.py | 3 +++ runpod/runtimes/task/runner.py | 16 +++++++++--- tests/test_apps/test_dev.py | 2 +- tests/test_init.py | 4 +-- 16 files changed, 109 insertions(+), 67 deletions(-) diff --git a/runpod/__init__.py b/runpod/__init__.py index 56c2a7176..ddbfa21e3 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -64,50 +64,10 @@ "serverless": "runpod.serverless", } +# lazy names resolve through __getattr__, so __all__ derives from the +# lazy table plus the eagerly-defined module attributes __all__ = [ - "create_container_registry_auth", - "create_endpoint", - "create_pod", - "create_template", - "delete_container_registry_auth", - "get_endpoints", - "get_gpu", - "get_gpus", - "get_pod", - "get_pods", - "get_user", - "resume_pod", - "stop_pod", - "terminate_pod", - "update_container_registry_auth", - "update_endpoint_template", - "update_user_settings", - "check_credentials", - "get_credentials", - "set_credentials", - "AsyncioEndpoint", - "AsyncioJob", - "Endpoint", - "Api", - "App", - "CpuInstanceType", - "DataCenter", - "EndpointNotFound", - "GpuGroup", - "GpuType", - "Job", - "Queue", - "delete", - "get", - "init", - "is_local", - "local_entrypoint", - "patch", - "post", - "put", - "schedule", - "serverless", - "RunPodLogger", + *_LAZY_ATTRS, "__version__", "SSH_KEY_PATH", "profile", diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 8e570a0cd..e69bfc781 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -50,7 +50,8 @@ async def _execute( # transport-level failures (ssl hiccups, resets, dns) # are transient; graphql/auth errors propagate untouched last_exc = exc - assert last_exc is not None # loop always runs at least once + if last_exc is None: # pragma: no cover - loop always runs once + raise RuntimeError("graphql transport retry loop exited cleanly") raise last_exc async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 477b15364..537449035 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -336,10 +336,13 @@ def build_environment( env_dir.mkdir(parents=True, exist_ok=True) user_requirements = collect_requirements(project_root, app) - # size-prohibitive packages come preinstalled on the builtin gpu - # images; a custom image carries no such guarantee, so any custom - # image in the app disables auto-exclusion and everything vendors - auto_exclude = not any(h.spec.image for h in app.resources.values()) + # size-prohibitive packages come preinstalled only on the builtin + # gpu images. custom images carry no guarantee, and builtin cpu + # images ship without them, so every resource must be gpu-on-builtin + # for auto-exclusion to be sound (the artifact is shared app-wide) + auto_exclude = all( + h.spec.gpu and not h.spec.image for h in app.resources.values() + ) vendored, excluded = split_exclusions( user_requirements, exclude, auto_exclude=auto_exclude ) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 1e81398e5..c1480c803 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -26,12 +26,22 @@ def dev_endpoint_name(app_name: str, resource_name: str) -> str: - return f"{DEV_PREFIX}-{app_name}-{resource_name}" + """unique endpoint name for a dev (app, resource) pair. + + dash-joined names alone are ambiguous (("a-b","c") vs ("a","b-c")), + so a short digest of the exact pair disambiguates. + """ + import hashlib + + digest = hashlib.sha256( + f"{app_name}/{resource_name}".encode() + ).hexdigest()[:6] + return f"{DEV_PREFIX}-{app_name}-{resource_name}-{digest}" def _resource_of(endpoint_name: str) -> str: - """resource name from a dev endpoint name (last dash segment).""" - return endpoint_name.rsplit("-", 1)[-1] + """resource name from a dev endpoint name (display only).""" + return endpoint_name.rsplit("-", 2)[-2] def _comparable(payload: Dict) -> Dict: diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index fabed41b1..4179b6ffd 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -85,6 +85,9 @@ def _import_module(path: Path) -> None: def run() -> None: try: spec.loader.exec_module(module) + # BaseException on purpose: user module top-level code can raise + # SystemExit/KeyboardInterrupt; those must surface as discovery + # errors on the caller thread, not kill this worker thread except BaseException as exc: # noqa: BLE001 - reported to caller error.append(exc) diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py index 5c7c12cb7..21a261fb0 100644 --- a/runpod/apps/gpu.py +++ b/runpod/apps/gpu.py @@ -218,6 +218,7 @@ def gpu_ids_value(gpu: Optional[List[str]]) -> str: pool = pool_for_gpu_type(GpuType(value)) value = pool.value if pool is not None else value except ValueError: + # not a known GpuType: pass the raw pool/id string through pass if value not in pools: pools.append(value) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 1f84e60d4..08a9136c4 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -257,7 +257,8 @@ async def _request_json( # decided retryability last_exc = exc continue - assert last_exc is not None # loop always runs at least once + if last_exc is None: # pragma: no cover - loop always runs once + raise RuntimeError("request retry loop exited cleanly") raise last_exc diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 7ddcd1cde..cb46fc6da 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -229,6 +229,8 @@ async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: if resp.status == 200: return except (aiohttp.ClientError, asyncio.TimeoutError): + # pod not reachable yet; the surrounding deadline + # decides when to stop trying pass await asyncio.sleep(READY_POLL_INTERVAL) raise TimeoutError( diff --git a/runpod/apps/watch.py b/runpod/apps/watch.py index 7ea12d3cc..5f931bda4 100644 --- a/runpod/apps/watch.py +++ b/runpod/apps/watch.py @@ -18,6 +18,7 @@ def _snapshot(roots: Iterable[Path]) -> Dict[str, float]: try: state[str(root)] = root.stat().st_mtime except OSError: + # file deleted mid-scan; picked up on the next pass pass continue for path in root.rglob("*.py"): diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 13ee4bbfc..16e396ed4 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -315,6 +315,11 @@ def dev(module, once): os.environ["RUNPOD_DEV_SESSION"] = "1" def _scan(): + # each rescan registers entrypoints anew; clearing first keeps + # get_entrypoint() reflecting the current file, not stale runs + from runpod.apps.entrypoint import _clear_entrypoints + + _clear_entrypoints() try: apps = discover_apps(module) except DiscoveryError as exc: @@ -382,6 +387,7 @@ async def _wait_for_rerun(watcher: "FileWatcher") -> str: else: return "enter" except _queue.Empty: + # no keypress this tick; fall through to the sleep pass await asyncio.sleep(0.5) @@ -408,6 +414,8 @@ def _finish(exc: "BaseException | None") -> None: def _runner() -> None: try: run_entrypoint(fn) + # BaseException on purpose: SystemExit/KeyboardInterrupt from + # the entrypoint must reach the loop, not die on this thread except BaseException as exc: # noqa: BLE001 - marshalled to the loop # bind the exception now: `as exc` is unbound once the # except block exits, before the loop callback runs @@ -497,6 +505,7 @@ async def _run(): try: asyncio.run(_run()) except KeyboardInterrupt: + # ctrl-c ends the log follow; not an error pass diff --git a/runpod/rp_cli/update.py b/runpod/rp_cli/update.py index 18a6f7a42..5097bd9e6 100644 --- a/runpod/rp_cli/update.py +++ b/runpod/rp_cli/update.py @@ -87,10 +87,23 @@ def fetch_pypi_metadata() -> Tuple[str, Set[str]]: def install_command(version: str) -> list: - """the installer invocation, preferring uv when present.""" + """the installer invocation targeting the running interpreter. + + `uv pip install` is used only with an explicit --python pointing at + this interpreter: bare uv resolves against an activated venv, which + is not necessarily where this cli runs from. + """ package_spec = f"runpod=={version}" if shutil.which("uv"): - return ["uv", "pip", "install", package_spec, "--quiet"] + return [ + "uv", + "pip", + "install", + "--python", + sys.executable, + package_spec, + "--quiet", + ] return [sys.executable, "-m", "pip", "install", package_spec, "--quiet"] @@ -135,6 +148,7 @@ def _write_cache(latest_version: str) -> None: encoding="utf-8", ) except OSError: + # cache write is best-effort; the check reruns next invocation pass diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py index 534648d89..e1d0c42c5 100644 --- a/runpod/runtimes/api/server.py +++ b/runpod/runtimes/api/server.py @@ -151,17 +151,41 @@ async def endpoint(request: Request): def _build_factory_app(handle) -> Any: - """call the user's asgi factory and ensure /ping exists.""" + """call the user's asgi factory and ensure /ping exists. + + fastapi/starlette apps get a /ping route added when missing; other + asgi callables are wrapped so /ping is answered before delegating. + """ app = handle._asgi_factory() - routes = getattr(app, "routes", []) - if not any(getattr(r, "path", None) == "/ping" for r in routes): + routes = getattr(app, "routes", None) + if routes is not None and hasattr(app, "get"): + if not any(getattr(r, "path", None) == "/ping" for r in routes): - @app.get("/ping") - async def ping(): - return {"status": "healthy"} + @app.get("/ping") + async def ping(): + return {"status": "healthy"} - return app + return app + + async def with_ping(scope, receive, send): + if scope["type"] == "http" and scope["path"] == "/ping": + body = b'{"status": "healthy"}' + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [ + (b"content-type", b"application/json"), + (b"content-length", str(len(body)).encode()), + ], + } + ) + await send({"type": "http.response.body", "body": body}) + return + await app(scope, receive, send) + + return with_ping def _install_live_dependencies(request: dict) -> None: diff --git a/runpod/runtimes/executor.py b/runpod/runtimes/executor.py index 1e2b50a1a..bb1c7b626 100644 --- a/runpod/runtimes/executor.py +++ b/runpod/runtimes/executor.py @@ -248,6 +248,8 @@ def write(self, s): stream.write(s) stream.flush() except (ValueError, OSError): + # one sink failing (closed pipe) must not lose the + # write on the other sinks pass return len(s) @@ -256,6 +258,7 @@ def flush(self): try: stream.flush() except (ValueError, OSError): + # closed sinks are skipped, same as write pass diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py index b0694890b..0b3e6d9c7 100644 --- a/runpod/runtimes/task/runner.py +++ b/runpod/runtimes/task/runner.py @@ -49,6 +49,9 @@ _job_lock = threading.Lock() _job_state = {"status": "NONE", "response": None} _last_contact = {"ts": None} # set at server start +# inline /execute requests in flight; the watchdog must not terminate +# the pod while one runs (long executes outlive the idle timeout) +_inline_executions = {"count": 0} def _touch_contact(): @@ -57,9 +60,9 @@ def _touch_contact(): _last_contact["ts"] = time.time() -def _should_self_terminate(status, last_contact, now, idle_timeout): +def _should_self_terminate(status, last_contact, now, idle_timeout, inline=0): """the watchdog decision: kill only provably-abandoned pods.""" - if status == "RUNNING": + if status == "RUNNING" or inline > 0: return False if last_contact is None: return False @@ -117,6 +120,7 @@ def _watchdog(): _last_contact["ts"], time.time(), IDLE_TIMEOUT, + inline=_inline_executions["count"], ): _terminate_self() @@ -166,7 +170,13 @@ def do_POST(self): # noqa: N802 - BaseHTTPRequestHandler api self._send(401, {"error": "unauthorized"}) return if self.path == "/execute": - self._send(200, execute_request(self._read_request())) + with _job_lock: + _inline_executions["count"] += 1 + try: + self._send(200, execute_request(self._read_request())) + finally: + with _job_lock: + _inline_executions["count"] -= 1 return if self.path == "/submit": request = self._read_request() diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 45f25c3bd..4ad35661e 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -35,7 +35,7 @@ def q(): pass payload = _endpoint_input(app, q.spec) - assert payload["name"] == "dev-a-q" + assert payload["name"].startswith("dev-a-q-") assert payload["instanceIds"] == ["cpu3c-1-2"] assert "EU-RO-1" in payload["locations"] assert "US-KS-2" in payload["locations"] diff --git a/tests/test_init.py b/tests/test_init.py index 62a74fec2..52339fce2 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -100,8 +100,8 @@ def test_all_covers_expected_public_api(self): 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', # Apps surface 'Api', 'App', 'CpuInstanceType', 'DataCenter', 'EndpointNotFound', 'GpuGroup', - 'GpuType', 'Job', 'Queue', 'delete', 'get', 'init', 'is_local', - 'local_entrypoint', 'patch', 'post', 'put', 'schedule', + 'GpuType', 'Job', 'Model', 'Queue', 'Secret', 'Volume', 'delete', 'get', + 'init', 'is_local', 'local_entrypoint', 'patch', 'post', 'put', 'schedule', # Serverless module 'serverless', # Logger class From 3df2237581cbb4d12c527237adbd1dc061d47575 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:42:29 -0700 Subject: [PATCH 092/120] fix: address remaining apps sdk review findings --- examples/apps/gpu_inference.py | 5 ++--- examples/apps/hello_world.py | 3 +-- examples/apps/streaming.py | 3 +-- examples/apps/train_and_eval.py | 5 ++--- examples/apps/web_service.py | 9 ++++----- runpod/apps/discovery.py | 5 ++--- runpod/rp_cli/main.py | 10 +++++----- tests/e2e/examples/01_hello_world.py | 3 +-- tests/e2e/examples/02_invocation_styles.py | 3 +-- tests/e2e/examples/03_gpu.py | 3 +-- tests/e2e/examples/04_dependencies.py | 3 +-- tests/e2e/examples/05_api_service.py | 9 ++++----- tests/e2e/examples/06_tasks.py | 3 +-- tests/e2e/examples/07_pipelines.py | 3 +-- tests/e2e/examples/08_volumes.py | 5 ++--- tests/e2e/examples/09_secrets.py | 8 +++++--- tests/e2e/examples/10_cached_models.py | 5 ++--- tests/e2e/examples/11_train_eval.py | 5 ++--- tests/e2e/examples/12_custom_image.py | 3 +-- tests/e2e/examples/13_streaming.py | 3 +-- 20 files changed, 40 insertions(+), 56 deletions(-) diff --git a/examples/apps/gpu_inference.py b/examples/apps/gpu_inference.py index 5e5bacb5e..c5dcd1ebc 100644 --- a/examples/apps/gpu_inference.py +++ b/examples/apps/gpu_inference.py @@ -9,9 +9,8 @@ """ import runpod -from runpod import App -app = App("gpu-demo") +app = runpod.App("gpu-demo") @app.queue(gpu="4090", dependencies=["numpy"]) @@ -29,7 +28,7 @@ def matmul_benchmark(size: int = 4096): torch.cuda.synchronize() start = time.time() for _ in range(10): - a @ b + result = a @ b torch.cuda.synchronize() elapsed = time.time() - start diff --git a/examples/apps/hello_world.py b/examples/apps/hello_world.py index 0bee9c5c5..530371389 100644 --- a/examples/apps/hello_world.py +++ b/examples/apps/hello_world.py @@ -13,9 +13,8 @@ """ import runpod -from runpod import App -app = App("hello") +app = runpod.App("hello") @app.queue(cpu="cpu3c-1-2") diff --git a/examples/apps/streaming.py b/examples/apps/streaming.py index 35267bcff..1f682d959 100644 --- a/examples/apps/streaming.py +++ b/examples/apps/streaming.py @@ -13,9 +13,8 @@ import time import runpod -from runpod import App -app = App("streaming") +app = runpod.App("streaming") @app.queue(cpu="cpu3c-1-2") diff --git a/examples/apps/train_and_eval.py b/examples/apps/train_and_eval.py index db7b4c443..8459973a4 100644 --- a/examples/apps/train_and_eval.py +++ b/examples/apps/train_and_eval.py @@ -11,11 +11,10 @@ """ import runpod -from runpod import App, Volume -app = App("trainer") +app = runpod.App("trainer") -checkpoints = Volume("checkpoints", size=10) +checkpoints = runpod.Volume("checkpoints", size=10) @app.task(gpu="4090", volume=checkpoints) diff --git a/examples/apps/web_service.py b/examples/apps/web_service.py index fe6f6baa6..ec1dbb365 100644 --- a/examples/apps/web_service.py +++ b/examples/apps/web_service.py @@ -12,25 +12,24 @@ class itself, as shown in the entrypoint. """ import runpod -from runpod import App, get, init, post -app = App("web") +app = runpod.App("web") @app.api(cpu="cpu3c-1-2") class Counter: - @init + @runpod.init def setup(self): # runs once when a worker starts, before any request self.count = 0 print("worker ready, counter at 0") - @post("/bump") + @runpod.post("/bump") async def bump(self, body: dict): self.count += body.get("by", 1) return {"count": self.count} - @get("/value") + @runpod.get("/value") async def value(self): return {"count": self.count} diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 4179b6ffd..2ba9a71cf 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -85,10 +85,9 @@ def _import_module(path: Path) -> None: def run() -> None: try: spec.loader.exec_module(module) - # BaseException on purpose: user module top-level code can raise - # SystemExit/KeyboardInterrupt; those must surface as discovery + # user module control-flow exceptions must surface as discovery # errors on the caller thread, not kill this worker thread - except BaseException as exc: # noqa: BLE001 - reported to caller + except (Exception, KeyboardInterrupt, SystemExit) as exc: error.append(exc) thread = threading.Thread(target=run, daemon=True) diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 16e396ed4..001c2bf1a 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -401,7 +401,7 @@ async def _run_entrypoint_cancellable(fn) -> None: free, and cancellation simply abandons the in-flight call. """ loop = asyncio.get_running_loop() - done: asyncio.Future = loop.create_future() + done: asyncio.Future[None] = loop.create_future() def _finish(exc: "BaseException | None") -> None: if done.done(): @@ -414,9 +414,9 @@ def _finish(exc: "BaseException | None") -> None: def _runner() -> None: try: run_entrypoint(fn) - # BaseException on purpose: SystemExit/KeyboardInterrupt from - # the entrypoint must reach the loop, not die on this thread - except BaseException as exc: # noqa: BLE001 - marshalled to the loop + # entrypoint control-flow exceptions must reach the loop, + # not die on this thread + except (Exception, KeyboardInterrupt, SystemExit) as exc: # bind the exception now: `as exc` is unbound once the # except block exits, before the loop callback runs loop.call_soon_threadsafe(_finish, exc) @@ -424,7 +424,7 @@ def _runner() -> None: loop.call_soon_threadsafe(_finish, None) threading.Thread(target=_runner, daemon=True).start() - await done + return await done async def _session() -> int: nonlocal apps, entrypoint diff --git a/tests/e2e/examples/01_hello_world.py b/tests/e2e/examples/01_hello_world.py index fc14acaa6..e4c16a79f 100644 --- a/tests/e2e/examples/01_hello_world.py +++ b/tests/e2e/examples/01_hello_world.py @@ -4,9 +4,8 @@ """ import runpod -from runpod import App -app = App("ex-hello") +app = runpod.App("ex-hello") @app.queue(cpu="cpu3c-1-2") diff --git a/tests/e2e/examples/02_invocation_styles.py b/tests/e2e/examples/02_invocation_styles.py index 9aeb52197..43a73ef20 100644 --- a/tests/e2e/examples/02_invocation_styles.py +++ b/tests/e2e/examples/02_invocation_styles.py @@ -6,9 +6,8 @@ import asyncio import runpod -from runpod import App -app = App("ex-invoke") +app = runpod.App("ex-invoke") @app.queue(cpu="cpu3c-1-2") diff --git a/tests/e2e/examples/03_gpu.py b/tests/e2e/examples/03_gpu.py index 055cd9e71..5999f9c9f 100644 --- a/tests/e2e/examples/03_gpu.py +++ b/tests/e2e/examples/03_gpu.py @@ -4,9 +4,8 @@ """ import runpod -from runpod import App -app = App("ex-gpu") +app = runpod.App("ex-gpu") @app.queue(gpu="4090", env={"MODE": "example"}) diff --git a/tests/e2e/examples/04_dependencies.py b/tests/e2e/examples/04_dependencies.py index 4c06728e1..054ee3139 100644 --- a/tests/e2e/examples/04_dependencies.py +++ b/tests/e2e/examples/04_dependencies.py @@ -4,9 +4,8 @@ """ import runpod -from runpod import App -app = App("ex-deps") +app = runpod.App("ex-deps") @app.queue(cpu="cpu3c-1-2", dependencies=["pyfiglet"]) diff --git a/tests/e2e/examples/05_api_service.py b/tests/e2e/examples/05_api_service.py index b918d3e26..be8d0d899 100644 --- a/tests/e2e/examples/05_api_service.py +++ b/tests/e2e/examples/05_api_service.py @@ -4,25 +4,24 @@ """ import runpod -from runpod import App, get, init, post -app = App("ex-api") +app = runpod.App("ex-api") @app.api(cpu="cpu3c-1-2") class Counter: - @init + @runpod.init def setup(self): # runs once per worker before it takes traffic self.count = 0 print("counter initialized") - @post("/bump") + @runpod.post("/bump") async def bump(self, body: dict): self.count += body.get("by", 1) return {"count": self.count} - @get("/value") + @runpod.get("/value") async def value(self): return {"count": self.count} diff --git a/tests/e2e/examples/06_tasks.py b/tests/e2e/examples/06_tasks.py index af12fb345..5f96962fe 100644 --- a/tests/e2e/examples/06_tasks.py +++ b/tests/e2e/examples/06_tasks.py @@ -4,9 +4,8 @@ """ import runpod -from runpod import App -app = App("ex-tasks") +app = runpod.App("ex-tasks") @app.task(cpu="cpu3c-1-2") diff --git a/tests/e2e/examples/07_pipelines.py b/tests/e2e/examples/07_pipelines.py index 0fa0b8010..a797699a8 100644 --- a/tests/e2e/examples/07_pipelines.py +++ b/tests/e2e/examples/07_pipelines.py @@ -4,9 +4,8 @@ """ import runpod -from runpod import App -app = App("ex-pipeline") +app = runpod.App("ex-pipeline") @app.queue(cpu="cpu3c-1-2") diff --git a/tests/e2e/examples/08_volumes.py b/tests/e2e/examples/08_volumes.py index 106acdabe..834bb976f 100644 --- a/tests/e2e/examples/08_volumes.py +++ b/tests/e2e/examples/08_volumes.py @@ -7,11 +7,10 @@ """ import runpod -from runpod import App, Volume -app = App("ex-volumes") +app = runpod.App("ex-volumes") -scratch = Volume("ex-scratch", size=10) +scratch = runpod.Volume("ex-scratch", size=10) @app.task(cpu="cpu3c-1-2", volume=scratch) diff --git a/tests/e2e/examples/09_secrets.py b/tests/e2e/examples/09_secrets.py index f17170489..f0bc688dd 100644 --- a/tests/e2e/examples/09_secrets.py +++ b/tests/e2e/examples/09_secrets.py @@ -7,12 +7,14 @@ """ import runpod -from runpod import App, Secret -app = App("ex-secrets") +app = runpod.App("ex-secrets") -@app.queue(cpu="cpu3c-1-2", env={"DEMO_TOKEN": Secret("ex-demo-secret"), "MODE": "example"}) +@app.queue( + cpu="cpu3c-1-2", + env={"DEMO_TOKEN": runpod.Secret("ex-demo-secret"), "MODE": "example"}, +) def peek(): import os diff --git a/tests/e2e/examples/10_cached_models.py b/tests/e2e/examples/10_cached_models.py index 95aae5504..a406e9140 100644 --- a/tests/e2e/examples/10_cached_models.py +++ b/tests/e2e/examples/10_cached_models.py @@ -4,11 +4,10 @@ """ import runpod -from runpod import App, Model -app = App("ex-models") +app = runpod.App("ex-models") -tiny = Model("sshleifer/tiny-gpt2") +tiny = runpod.Model("sshleifer/tiny-gpt2") @app.queue(gpu="4090", model=tiny, dependencies=["transformers"]) diff --git a/tests/e2e/examples/11_train_eval.py b/tests/e2e/examples/11_train_eval.py index 022210a64..0470bb4e4 100644 --- a/tests/e2e/examples/11_train_eval.py +++ b/tests/e2e/examples/11_train_eval.py @@ -4,11 +4,10 @@ """ import runpod -from runpod import App, Volume -app = App("ex-train-eval") +app = runpod.App("ex-train-eval") -models = Volume("ex-models", size=10) +models = runpod.Volume("ex-models", size=10) @app.task(gpu="4090", volume=models) diff --git a/tests/e2e/examples/12_custom_image.py b/tests/e2e/examples/12_custom_image.py index 15037d72e..6bedae0ea 100644 --- a/tests/e2e/examples/12_custom_image.py +++ b/tests/e2e/examples/12_custom_image.py @@ -5,9 +5,8 @@ """ import runpod -from runpod import App -app = App("ex-image") +app = runpod.App("ex-image") @app.queue(cpu="cpu3c-1-2", image="python:3.12-slim") diff --git a/tests/e2e/examples/13_streaming.py b/tests/e2e/examples/13_streaming.py index a2cb52157..cf317ce14 100644 --- a/tests/e2e/examples/13_streaming.py +++ b/tests/e2e/examples/13_streaming.py @@ -6,9 +6,8 @@ import asyncio import runpod -from runpod import App -app = App("ex-stream") +app = runpod.App("ex-stream") @app.queue(cpu="cpu3c-1-2") From ae6182da74021f33cb15a5ea2abfff95e1b9325e Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:49:27 -0700 Subject: [PATCH 093/120] fix: clean up apps sdk imports --- examples/apps/gpu_inference.py | 2 +- runpod/apps/api.py | 2 +- runpod/apps/placement.py | 2 +- runpod/apps/volume.py | 2 +- runpod/rp_cli/console.py | 1 - runpod/runtimes/api/server.py | 2 +- tests/e2e/matrix/main.py | 3 +-- tests/e2e/matrix/run.py | 1 - tests/test_apps/test_api_client.py | 1 - tests/test_apps/test_api_server.py | 1 - tests/test_apps/test_bootstrap.py | 1 - tests/test_apps/test_build.py | 1 - tests/test_apps/test_deploy.py | 1 - tests/test_apps/test_images.py | 6 +++--- tests/test_apps/test_logs.py | 3 --- tests/test_apps/test_monitor.py | 4 ---- tests/test_apps/test_shim.py | 2 -- tests/test_apps/test_tasks.py | 1 - 18 files changed, 9 insertions(+), 27 deletions(-) diff --git a/examples/apps/gpu_inference.py b/examples/apps/gpu_inference.py index c5dcd1ebc..7eaa9f093 100644 --- a/examples/apps/gpu_inference.py +++ b/examples/apps/gpu_inference.py @@ -28,7 +28,7 @@ def matmul_benchmark(size: int = 4096): torch.cuda.synchronize() start = time.time() for _ in range(10): - result = a @ b + _ = a @ b torch.cuda.synchronize() elapsed = time.time() - start diff --git a/runpod/apps/api.py b/runpod/apps/api.py index e69bfc781..2483fdd8b 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -362,7 +362,7 @@ async def _reader(): try: await self._put_tarball(upload_url, _reader(), total, timeout) return - except (aiohttp.ClientError, OSError) as exc: + except (aiohttp.ClientError, OSError): if attempt == attempts - 1: raise await asyncio.sleep(2**attempt) diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py index 4f88ad772..3ef7498aa 100644 --- a/runpod/apps/placement.py +++ b/runpod/apps/placement.py @@ -204,7 +204,7 @@ def solve_placement( raise PlacementError( f"cannot place volume '{volume_name}': no datacenter can host " f"every resource using it\n" + "\n".join(lines) + "\n" - f"use separate volumes or compatible hardware" + "use separate volumes or compatible hardware" ) # maximin: rank each DC by the worst resource's stock there, diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index 610b19f87..481ca7d03 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -127,7 +127,7 @@ async def resolve( if cached is not None: return cached - from .placement import PlacementError, StockMap, solve_placement + from .placement import StockMap, solve_placement client = await self._client() existing = await client.list_network_volumes() diff --git a/runpod/rp_cli/console.py b/runpod/rp_cli/console.py index 8b766a123..f165ff56e 100644 --- a/runpod/rp_cli/console.py +++ b/runpod/rp_cli/console.py @@ -18,7 +18,6 @@ TextColumn, ) from rich.spinner import SPINNERS -from rich.text import Text from rich.theme import Theme # -- brand ---------------------------------------------------------------- diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py index e1d0c42c5..a675fe31a 100644 --- a/runpod/runtimes/api/server.py +++ b/runpod/runtimes/api/server.py @@ -92,7 +92,7 @@ def _build_class_app(handle) -> Any: """ from contextlib import asynccontextmanager - from fastapi import FastAPI, Request + from fastapi import FastAPI cls = handle._cls instance = cls() diff --git a/tests/e2e/matrix/main.py b/tests/e2e/matrix/main.py index 070022bcf..00ccaf622 100644 --- a/tests/e2e/matrix/main.py +++ b/tests/e2e/matrix/main.py @@ -5,9 +5,8 @@ """ import runpod -from runpod import App -app = App("e2e-matrix") +app = runpod.App("e2e-matrix") # -- queue permutations ------------------------------------------------ diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py index 125710c5e..e5f93b7bb 100644 --- a/tests/e2e/matrix/run.py +++ b/tests/e2e/matrix/run.py @@ -14,7 +14,6 @@ from main import ( # noqa: E402 Svc, - app, q_basic, q_caller, q_custom, diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py index 095ad90f5..445d97b0a 100644 --- a/tests/test_apps/test_api_client.py +++ b/tests/test_apps/test_api_client.py @@ -1,6 +1,5 @@ """unit tests for the apps control-plane client.""" -import asyncio from unittest.mock import AsyncMock, MagicMock, patch import aiohttp diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py index 10804879c..e3ee2496b 100644 --- a/tests/test_apps/test_api_server.py +++ b/tests/test_apps/test_api_server.py @@ -8,7 +8,6 @@ pytest.importorskip("fastapi") from fastapi.testclient import TestClient -import runpod from runpod.apps.app import _clear_registry from runpod.runtimes.api import server diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py index 5eadedc0c..f2790c8df 100644 --- a/tests/test_apps/test_bootstrap.py +++ b/tests/test_apps/test_bootstrap.py @@ -2,7 +2,6 @@ import base64 import json -import textwrap import pytest diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index 30af8ece3..4fbe8f13e 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -1,6 +1,5 @@ """tests for deploy-time environment vendoring.""" -from pathlib import Path from unittest.mock import patch import pytest diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index f8223061c..365d412be 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -1,6 +1,5 @@ """tests for discovery, manifest building, and packaging.""" -import io import json import tarfile import textwrap diff --git a/tests/test_apps/test_images.py b/tests/test_apps/test_images.py index 87e7dfc2a..d9c8a88bc 100644 --- a/tests/test_apps/test_images.py +++ b/tests/test_apps/test_images.py @@ -1,5 +1,7 @@ """tests for runtime image selection.""" +import sys + import pytest from runpod.apps.images import ( @@ -63,12 +65,10 @@ def test_returns_supported_version(self): def test_unsupported_local_python_fails_loudly(self, monkeypatch): # cloudpickle payloads are version-bound; a silent fallback # would break at deserialization time on the worker - import runpod.apps.images as images - class FakeVersion: major, minor = 3, 9 - monkeypatch.setattr(images.sys, "version_info", FakeVersion) + monkeypatch.setattr(sys, "version_info", FakeVersion) with pytest.raises(RuntimeError, match="3.9"): local_python_version() diff --git a/tests/test_apps/test_logs.py b/tests/test_apps/test_logs.py index f32b661b7..fffc9df02 100644 --- a/tests/test_apps/test_logs.py +++ b/tests/test_apps/test_logs.py @@ -3,9 +3,6 @@ import json from unittest.mock import AsyncMock, MagicMock, patch -import pytest - -from runpod.apps import logs as pod_logs_mod from runpod.apps.logs import pod_logs, stream_pod_logs, tail_summary diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index c1cb27192..f4c5eb530 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -3,8 +3,6 @@ import asyncio from unittest.mock import AsyncMock, patch -import pytest - from runpod.apps.monitor import WorkerMonitor, emit, format_worker_counts @@ -252,8 +250,6 @@ async def test_start_without_metrics_key_is_noop(self): await monitor.stop() async def test_metrics_polling_reports_counts(self): - import runpod - sink = Sink() monitor = WorkerMonitor("ep1", "calc", sink, metrics_key="mk") payload = {"workers": {"initializing": 1, "ready": 0}} diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py index 5b55a21ba..8339609ca 100644 --- a/tests/test_apps/test_shim.py +++ b/tests/test_apps/test_shim.py @@ -3,8 +3,6 @@ import base64 import subprocess -import pytest - from runpod.apps.shim import shell_launcher diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 91656f10f..893a434d1 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -16,7 +16,6 @@ import cloudpickle import pytest -import runpod from runpod.apps import App from runpod.apps.app import _clear_registry from runpod.apps.errors import RemoteExecutionError From 1ad2a484e4c1408bb0ca28f0b8659ba0920ba5d8 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:06:57 -0700 Subject: [PATCH 094/120] fix: resolve apps sdk codeql findings --- runpod/apps/app.py | 2 +- runpod/apps/deploy.py | 28 +++------------- runpod/apps/dev.py | 47 +++++---------------------- runpod/apps/discovery.py | 23 +------------ runpod/apps/discovery_state.py | 21 ++++++++++++ runpod/apps/handles.py | 5 +-- runpod/apps/logs.py | 4 +-- runpod/apps/monitor.py | 6 ++-- runpod/apps/shim.py | 12 +++++++ runpod/apps/targets.py | 28 ++-------------- runpod/apps/tasks.py | 3 +- runpod/apps/utils/names.py | 11 +++++++ runpod/apps/utils/network.py | 33 +++++++++++++++++++ runpod/apps/volume.py | 19 +++++++++++ runpod/rp_cli/update.py | 7 ++-- runpod/runtimes/executor.py | 8 ++--- tests/test_apps/test_executor.py | 4 +-- tests/test_apps/test_monitor.py | 2 +- tests/test_apps/test_tasks.py | 16 ++++----- tests/test_cli/test_update_command.py | 4 +-- 20 files changed, 140 insertions(+), 143 deletions(-) create mode 100644 runpod/apps/discovery_state.py create mode 100644 runpod/apps/utils/names.py create mode 100644 runpod/apps/utils/network.py diff --git a/runpod/apps/app.py b/runpod/apps/app.py index b62027b5a..009e6a10e 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -284,7 +284,7 @@ async def _resolve_dev_sibling(self, spec: ResourceSpec) -> InvocationTarget: return target from .api import AppsApiClient - from .dev import dev_endpoint_name + from .utils.names import dev_endpoint_name wanted = dev_endpoint_name(self.name, spec.name) endpoints = await AppsApiClient().list_my_endpoints() diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 461ff254a..a294577bb 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -24,6 +24,7 @@ ) from .errors import InvalidResourceError, ScheduleNotSupported from .schedule import SCHEDULES_ENABLED +from .shim import bootstrap_docker_args, bootstrap_source from .spec import ResourceKind from .utils.events import emit @@ -268,18 +269,16 @@ def _deployed_endpoint_input( if spec.image: # custom image: inject the bootstrap; the vendored env in the # artifact provides the runtime once the bootstrap unpacks it - from .dev import _bootstrap_docker_args, _bootstrap_source - payload["template"]["env"].extend( [ { "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode(_bootstrap_source().encode()).decode(), + "value": base64.b64encode(bootstrap_source().encode()).decode(), }, {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, ] ) - payload["template"]["dockerArgs"] = _bootstrap_docker_args() + payload["template"]["dockerArgs"] = bootstrap_docker_args() if spec.kind is ResourceKind.API: payload["type"] = "LB" @@ -328,7 +327,7 @@ async def reconcile_endpoints( } from .registry import resolve_registry_auth - from .volume import VolumeResolver + from .volume import VolumeResolver, attach_endpoint_volumes resolver = VolumeResolver(client) endpoints: Dict[str, str] = {} @@ -363,25 +362,6 @@ async def reconcile_endpoints( return endpoints -async def attach_endpoint_volumes(payload: Dict[str, Any], spec, resolver, app) -> None: - """resolve a resource's volumes onto an endpoint payload. - - endpoints may span regions: one volume per datacenter, locations - derived from the resolved volumes so lists cannot disagree. - """ - from .volume import specs_sharing_volume, volume_list - - volumes = volume_list(spec.volume) - if not volumes: - return - resolved = [] - for volume in volumes: - sharing = specs_sharing_volume([app], volume.name) or [spec] - resolved.append(await resolver.resolve(volume, sharing)) - payload["networkVolumeIds"] = [{"networkVolumeId": r["id"]} for r in resolved] - payload["locations"] = ",".join(dict.fromkeys(r["dataCenterId"] for r in resolved)) - - def _phase(events, name: str, detail: str = "") -> None: emit(events, "phase", name, detail) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index c1480c803..9ebd52f44 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -8,36 +8,20 @@ import base64 import logging -from pathlib import Path +import os as _os from typing import Dict, List, Optional, Union from .api import AppsApiClient from .app import App from .handles import ApiHandle, FunctionHandle -from .utils.events import emit +from .shim import bootstrap_docker_args, bootstrap_source from .spec import ResourceKind, ResourceSpec from .targets import LiveTarget +from .utils.events import emit +from .utils.names import dev_endpoint_name log = logging.getLogger(__name__) -DEV_PREFIX = "dev" - -import os as _os - - -def dev_endpoint_name(app_name: str, resource_name: str) -> str: - """unique endpoint name for a dev (app, resource) pair. - - dash-joined names alone are ambiguous (("a-b","c") vs ("a","b-c")), - so a short digest of the exact pair disambiguates. - """ - import hashlib - - digest = hashlib.sha256( - f"{app_name}/{resource_name}".encode() - ).hexdigest()[:6] - return f"{DEV_PREFIX}-{app_name}-{resource_name}-{digest}" - def _resource_of(endpoint_name: str) -> str: """resource name from a dev endpoint name (display only).""" @@ -94,18 +78,6 @@ def _image_for(spec: ResourceSpec) -> str: return image_for_spec(spec, python_version=local_python_version()) -def _bootstrap_source() -> str: - return (Path(__file__).parent.parent / "runtimes" / "bootstrap.py").read_text() - - -def _bootstrap_docker_args() -> str: - """shell command that materializes and starts the bootstrap on a - custom image.""" - from .shim import shell_launcher - - return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") - - def _render_env(env): from .secret import render_env @@ -113,10 +85,10 @@ def _render_env(env): def _client_api_key() -> str: - from .targets import _api_key + from .utils.network import api_key try: - return _api_key() + return api_key() except RuntimeError: # payload construction must not require credentials (tests, # dry runs); the session itself fails loudly on its first call @@ -196,7 +168,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: [ { "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode(_bootstrap_source().encode()).decode(), + "value": base64.b64encode(bootstrap_source().encode()).decode(), }, {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, ] @@ -208,7 +180,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: payload["template"]["env"].append( {"key": "RUNPOD_PACKAGE_SPEC", "value": package_spec} ) - payload["template"]["dockerArgs"] = _bootstrap_docker_args() + payload["template"]["dockerArgs"] = bootstrap_docker_args() if spec.kind is ResourceKind.API: payload["type"] = "LB" if spec.datacenter: @@ -262,9 +234,8 @@ def _emit(self, event: str, *args) -> None: async def _attach_volumes(self, payload: Dict, spec, app) -> None: """resolve the resource's volumes and registry auth onto a dev endpoint payload.""" - from .deploy import attach_endpoint_volumes from .registry import resolve_registry_auth - from .volume import VolumeResolver + from .volume import VolumeResolver, attach_endpoint_volumes if spec.registry_auth: auth_id = await resolve_registry_auth(spec.registry_auth, api=self.api) diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 2ba9a71cf..4db1c907c 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -14,33 +14,12 @@ from typing import List from .app import App, get_registered_apps +from .discovery_state import DISCOVERY_ENV log = logging.getLogger(__name__) IMPORT_TIMEOUT_SECONDS = 30 -# set while discovery imports modules; handle invocation checks it so -# module-level .remote()/.spawn() calls fail fast with a precise -# diagnosis instead of attempting network calls mid-scan -DISCOVERY_ENV = "RUNPOD_DISCOVERY_SCAN" - - -def in_discovery() -> bool: - return bool(os.environ.get(DISCOVERY_ENV)) - - -class DiscoveryInvocationError(Exception): - """a handle was invoked at module level during a discovery scan.""" - - def __init__(self, resource_name: str): - super().__init__( - f"'{resource_name}' was invoked at import time. move calls " - f"into a function, an entrypoint, or an " - f'`if __name__ == "__main__":` guard so importing the file ' - f"has no side effects." - ) - - _SKIP_DIRS = { ".git", ".venv", diff --git a/runpod/apps/discovery_state.py b/runpod/apps/discovery_state.py new file mode 100644 index 000000000..f793bdc2a --- /dev/null +++ b/runpod/apps/discovery_state.py @@ -0,0 +1,21 @@ +"""state shared by discovery scans and invocation guards.""" + +import os + +DISCOVERY_ENV = "RUNPOD_DISCOVERY_SCAN" + + +def in_discovery() -> bool: + return bool(os.environ.get(DISCOVERY_ENV)) + + +class DiscoveryInvocationError(Exception): + """a handle was invoked at module level during a discovery scan.""" + + def __init__(self, resource_name: str): + super().__init__( + f"'{resource_name}' was invoked at import time. move calls " + f"into a function, an entrypoint, or an " + f'`if __name__ == "__main__":` guard so importing the file ' + f"has no side effects." + ) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 08e8fca34..9587d8b11 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -31,6 +31,7 @@ def load_model(): ... # worker startup hook ) from .context import Context, current_context +from .discovery_state import DiscoveryInvocationError, in_discovery from .errors import InvalidResourceError from .invoker import Invoker, StreamInvoker from .job import Job @@ -199,8 +200,6 @@ async def _job_async(self, job_id: str) -> Job: return Job({"id": job_id, "status": "UNKNOWN"}, target) def _guard_discovery(self) -> None: - from .discovery import DiscoveryInvocationError, in_discovery - if in_discovery(): raise DiscoveryInvocationError(self.spec.name) @@ -239,8 +238,6 @@ def __call__(self, path: str, body: Any = None, **kwargs: Any) -> Any: return block(self.aio(path, body, **kwargs)) async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: - from .discovery import DiscoveryInvocationError, in_discovery - if in_discovery(): raise DiscoveryInvocationError(self._handle.spec.name) target = await self._handle._app._resolve(self._handle.spec) diff --git a/runpod/apps/logs.py b/runpod/apps/logs.py index 482e0967e..912bc0b6a 100644 --- a/runpod/apps/logs.py +++ b/runpod/apps/logs.py @@ -21,9 +21,9 @@ def _headers() -> Dict[str, str]: - from .targets import _api_key + from .utils.network import api_key - return {"Authorization": f"Bearer {_api_key()}"} + return {"Authorization": f"Bearer {api_key()}"} async def pod_logs( diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index c8e8c25fc..3affb28e9 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -214,13 +214,13 @@ async def stop(self) -> None: await stream.stop() async def _poll_metrics(self) -> None: - from .targets import _endpoint_url_base, _get_json + from .utils.network import endpoint_url_base, get_json - url = f"{_endpoint_url_base()}/{self.endpoint_id}/metrics" + url = f"{endpoint_url_base()}/{self.endpoint_id}/metrics" headers = {"Authorization": f"Bearer {self.metrics_key}"} while True: try: - data = await _get_json(url, headers, 10.0) + data = await get_json(url, headers, 10.0) except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - observability is best-effort diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py index 13cfa8f57..c997315a2 100644 --- a/runpod/apps/shim.py +++ b/runpod/apps/shim.py @@ -16,6 +16,8 @@ pair of single quotes. """ +from pathlib import Path + # well-known interpreter locations beyond PATH _PYTHON_CANDIDATES = ( "python3", @@ -49,3 +51,13 @@ def shell_launcher(env_var: str, dest: str) -> str: f'exec "$PY" {dest}' ) return f"sh -c '{script}'" + + +def bootstrap_source() -> str: + """return the runtime bootstrap injected into custom images.""" + return (Path(__file__).parent.parent / "runtimes" / "bootstrap.py").read_text() + + +def bootstrap_docker_args() -> str: + """return the command that starts the injected runtime bootstrap.""" + return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 08a9136c4..4f18391e2 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -7,7 +7,6 @@ import inspect import json -import os from abc import ABC, abstractmethod from typing import Any, AsyncIterator, Callable, Dict, Optional @@ -17,6 +16,9 @@ from .errors import EndpointNotFound, RemoteExecutionError from .protocol import FORMAT_JSON, FunctionRequest, FunctionResponse from .spec import ResourceSpec +from .utils.events import emit +from .utils.network import api_key as _api_key +from .utils.network import endpoint_url_base as _endpoint_url_base # the sentinel pseudo-endpoint id; ai-api resolves the real endpoint from # the X-Flash-App / X-Flash-Environment / X-Flash-Endpoint headers. @@ -28,25 +30,6 @@ FINAL_STATUSES = frozenset({"COMPLETED", "FAILED", "CANCELLED", "TIMED_OUT"}) -def _api_key() -> str: - key = os.getenv("RUNPOD_API_KEY") - if not key: - import runpod - - key = runpod.api_key - if not key: - raise RuntimeError( - "no api key configured. run `rp login` or set RUNPOD_API_KEY." - ) - return key - - -def _endpoint_url_base() -> str: - import runpod - - return runpod.endpoint_url_base.rstrip("/") - - def _lb_domain() -> str: """host portion of the data-plane base url, for lb subdomain urls.""" base = _endpoint_url_base() @@ -657,8 +640,6 @@ async def invoke( ) -> Any: import time - from .monitor import emit - monitor = self._monitor() emit(self.events, "dispatch", self.resource_name) start = time.monotonic() @@ -739,8 +720,6 @@ async def request( ) -> Any: import time - from .monitor import emit - emit(self.events, "dispatch", self.resource_name, f"{method} {path}") start = time.monotonic() try: @@ -810,7 +789,6 @@ async def invoke( ) -> Any: import time - from .monitor import emit from .tasks import TaskExecution, unwrap_task_response name = self.spec.name diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index cb46fc6da..c04a82544 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -16,6 +16,7 @@ import asyncio import base64 import logging +import os as _os import secrets import time from datetime import datetime, timedelta, timezone @@ -35,8 +36,6 @@ # safety net: pods self-terminate server-side after this long DEFAULT_MAX_LIFETIME = timedelta(hours=1) -import os as _os - READY_POLL_INTERVAL = 2.0 READY_TIMEOUT = 600.0 RESULT_POLL_INTERVAL = 2.0 diff --git a/runpod/apps/utils/names.py b/runpod/apps/utils/names.py new file mode 100644 index 000000000..e711dd8ae --- /dev/null +++ b/runpod/apps/utils/names.py @@ -0,0 +1,11 @@ +"""stable names for app infrastructure.""" + +import hashlib + +DEV_PREFIX = "dev" + + +def dev_endpoint_name(app_name: str, resource_name: str) -> str: + """return an unambiguous endpoint name for an app resource.""" + digest = hashlib.sha256(f"{app_name}/{resource_name}".encode()).hexdigest()[:6] + return f"{DEV_PREFIX}-{app_name}-{resource_name}-{digest}" diff --git a/runpod/apps/utils/network.py b/runpod/apps/utils/network.py new file mode 100644 index 000000000..d681766ea --- /dev/null +++ b/runpod/apps/utils/network.py @@ -0,0 +1,33 @@ +"""shared authentication and lightweight data-plane requests.""" + +import os +from typing import Any, Dict + +import aiohttp + + +def api_key() -> str: + key = os.getenv("RUNPOD_API_KEY") + if not key: + import runpod + + key = runpod.api_key + if not key: + raise RuntimeError( + "no api key configured. run `rp login` or set RUNPOD_API_KEY." + ) + return key + + +def endpoint_url_base() -> str: + import runpod + + return runpod.endpoint_url_base.rstrip("/") + + +async def get_json(url: str, headers: Dict[str, str], timeout: float) -> Dict[str, Any]: + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + async with session.get(url, headers=headers) as response: + response.raise_for_status() + return await response.json() diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index 481ca7d03..fcf3e8dc3 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -203,3 +203,22 @@ def specs_sharing_volume(apps: List, name: str) -> List: if ref.name == name: out.append(handle.spec) return out + + +async def attach_endpoint_volumes( + payload: Dict[str, Any], spec, resolver: VolumeResolver, app +) -> None: + """resolve a resource's volumes onto an endpoint payload.""" + volumes = volume_list(spec.volume) + if not volumes: + return + resolved = [] + for volume in volumes: + sharing = specs_sharing_volume([app], volume.name) or [spec] + resolved.append(await resolver.resolve(volume, sharing)) + payload["networkVolumeIds"] = [ + {"networkVolumeId": result["id"]} for result in resolved + ] + payload["locations"] = ",".join( + dict.fromkeys(result["dataCenterId"] for result in resolved) + ) diff --git a/runpod/rp_cli/update.py b/runpod/rp_cli/update.py index 5097bd9e6..82258b7b1 100644 --- a/runpod/rp_cli/update.py +++ b/runpod/rp_cli/update.py @@ -28,7 +28,7 @@ _newer_version: Optional[str] = None _check_done = threading.Event() -_started = False +_started = threading.Event() _start_lock = threading.Lock() @@ -212,11 +212,10 @@ def _is_interactive() -> bool: def start_background_check() -> None: """spawn the daemon check thread once per process (ttys only).""" - global _started # noqa: PLW0603 with _start_lock: - if _started: + if _started.is_set(): return - _started = True + _started.set() if not _is_interactive(): return atexit.register(_print_update_notice) diff --git a/runpod/runtimes/executor.py b/runpod/runtimes/executor.py index bb1c7b626..104704a04 100644 --- a/runpod/runtimes/executor.py +++ b/runpod/runtimes/executor.py @@ -19,6 +19,7 @@ import shutil import subprocess import sys +import threading import traceback from contextlib import redirect_stdout @@ -53,12 +54,11 @@ def _install(packages, label): return None -_apt_updated = False +_apt_updated = threading.Event() def _install_system(packages): """install apt packages; requires a debian-family image with root.""" - global _apt_updated if not packages: return None if shutil.which("apt-get") is None: @@ -68,13 +68,13 @@ def _install_system(packages): f"them in" ) env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") - if not _apt_updated: + if not _apt_updated.is_set(): update = subprocess.run( ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env ) if update.returncode != 0: return f"apt-get update failed: {update.stderr[-2000:]}" - _apt_updated = True + _apt_updated.set() result = subprocess.run( ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], capture_output=True, diff --git a/tests/test_apps/test_executor.py b/tests/test_apps/test_executor.py index 01205e702..3fa76c729 100644 --- a/tests/test_apps/test_executor.py +++ b/tests/test_apps/test_executor.py @@ -87,7 +87,7 @@ def test_system_deps_installed_before_execution(self, monkeypatch): import shutil monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(executor, "_apt_updated", False) + monkeypatch.setattr(executor, "_apt_updated", executor.threading.Event()) calls = [] class R: @@ -116,7 +116,7 @@ def test_apt_update_runs_once(self, monkeypatch): import shutil monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(executor, "_apt_updated", False) + monkeypatch.setattr(executor, "_apt_updated", executor.threading.Event()) calls = [] class R: diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index f4c5eb530..f8546dfa6 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -258,7 +258,7 @@ async def fake_get(url, headers, timeout): assert headers["Authorization"] == "Bearer mk" return payload - with patch("runpod.apps.targets._get_json", fake_get): + with patch("runpod.apps.utils.network.get_json", fake_get): await monitor.start() await asyncio.sleep(0.05) await monitor.stop() diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 893a434d1..9dc58a7ec 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -1,14 +1,5 @@ """tests for the task runner and pod task transport.""" -# the sync bridge finalizes coroutines on a background loop thread; -# AsyncMock coroutines observed there trip the unraisable checker -# non-deterministically under full-suite gc timing -import pytest as _pytest - -pytestmark = _pytest.mark.filterwarnings( - "ignore::pytest.PytestUnraisableExceptionWarning" -) - import base64 import json from unittest.mock import AsyncMock, MagicMock, patch @@ -24,6 +15,13 @@ from runpod.apps.tasks import _pod_input, unwrap_task_response from runpod.apps.targets import PodTarget +# the sync bridge finalizes coroutines on a background loop thread; +# asyncmock coroutines observed there trip the unraisable checker +# non-deterministically under full-suite gc timing +pytestmark = pytest.mark.filterwarnings( + "ignore::pytest.PytestUnraisableExceptionWarning" +) + @pytest.fixture(autouse=True) def clean_registry(): diff --git a/tests/test_cli/test_update_command.py b/tests/test_cli/test_update_command.py index 7c941e654..ed3eb8dbb 100644 --- a/tests/test_cli/test_update_command.py +++ b/tests/test_cli/test_update_command.py @@ -239,7 +239,7 @@ def test_no_notice_when_check_incomplete(self, monkeypatch, capsys): class TestStartBackgroundCheck: def test_starts_once(self, monkeypatch): - monkeypatch.setattr(upd, "_started", False) + monkeypatch.setattr(upd, "_started", threading.Event()) monkeypatch.setattr(upd, "_is_interactive", lambda: True) threads = [] monkeypatch.setattr( @@ -252,7 +252,7 @@ def test_starts_once(self, monkeypatch): assert len(threads) == 1 def test_skips_non_tty(self, monkeypatch): - monkeypatch.setattr(upd, "_started", False) + monkeypatch.setattr(upd, "_started", threading.Event()) monkeypatch.setattr(upd, "_is_interactive", lambda: False) threads = [] monkeypatch.setattr( From 08c8c83f1c4e0858c933121c129d8b76966e74f2 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:45:47 -0700 Subject: [PATCH 095/120] refactor: consume independently released sdk runtimes --- .github/workflows/runtimes.yml | 162 -------- pyproject.toml | 2 - runpod/apps/build.py | 33 +- runpod/apps/deploy.py | 32 +- runpod/apps/dev.py | 28 +- runpod/apps/images.py | 9 +- runpod/apps/shim.py | 68 ++-- runpod/apps/tasks.py | 45 +-- runpod/runtimes/README.md | 75 ---- runpod/runtimes/__init__.py | 0 runpod/runtimes/api/Dockerfile | 29 -- runpod/runtimes/api/__init__.py | 0 runpod/runtimes/api/server.py | 321 ---------------- runpod/runtimes/bootstrap.py | 443 ---------------------- runpod/runtimes/executor.py | 291 -------------- runpod/runtimes/gpu-base/Dockerfile | 29 -- runpod/runtimes/queue/Dockerfile | 28 -- runpod/runtimes/queue/__init__.py | 0 runpod/runtimes/queue/worker.py | 192 ---------- runpod/runtimes/task/Dockerfile | 28 -- runpod/runtimes/task/__init__.py | 0 runpod/runtimes/task/runner.py | 218 ----------- tests/e2e/examples/README.md | 7 +- tests/test_apps/test_api_server.py | 262 ------------- tests/test_apps/test_bootstrap.py | 459 ----------------------- tests/test_apps/test_build.py | 13 +- tests/test_apps/test_dev.py | 33 ++ tests/test_apps/test_dispatch.py | 56 --- tests/test_apps/test_executor.py | 195 ---------- tests/test_apps/test_queue_worker.py | 282 -------------- tests/test_apps/test_shim.py | 72 ++-- tests/test_apps/test_task_runner_http.py | 238 ------------ tests/test_apps/test_tasks.py | 98 +---- 33 files changed, 175 insertions(+), 3573 deletions(-) delete mode 100644 .github/workflows/runtimes.yml delete mode 100644 runpod/runtimes/README.md delete mode 100644 runpod/runtimes/__init__.py delete mode 100644 runpod/runtimes/api/Dockerfile delete mode 100644 runpod/runtimes/api/__init__.py delete mode 100644 runpod/runtimes/api/server.py delete mode 100644 runpod/runtimes/bootstrap.py delete mode 100644 runpod/runtimes/executor.py delete mode 100644 runpod/runtimes/gpu-base/Dockerfile delete mode 100644 runpod/runtimes/queue/Dockerfile delete mode 100644 runpod/runtimes/queue/__init__.py delete mode 100644 runpod/runtimes/queue/worker.py delete mode 100644 runpod/runtimes/task/Dockerfile delete mode 100644 runpod/runtimes/task/__init__.py delete mode 100644 runpod/runtimes/task/runner.py delete mode 100644 tests/test_apps/test_api_server.py delete mode 100644 tests/test_apps/test_bootstrap.py delete mode 100644 tests/test_apps/test_executor.py delete mode 100644 tests/test_apps/test_queue_worker.py delete mode 100644 tests/test_apps/test_task_runner_http.py diff --git a/.github/workflows/runtimes.yml b/.github/workflows/runtimes.yml deleted file mode 100644 index 64deccd5a..000000000 --- a/.github/workflows/runtimes.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: Runtimes - -# runtime images embed the runpod package, so they are published when a -# release ships (release-please merge -> github release), not per pr. -# the feature branch is a prerelease channel while the apps sdk is in -# development; runtime tests for prs run in ci via the full test suite. -on: - release: - types: [published] - push: - branches: [feat/apps-sdk] - paths: - - "runpod/runtimes/**" - - ".github/workflows/runtimes.yml" - workflow_dispatch: - -concurrency: - group: runtimes-${{ github.ref }} - cancel-in-progress: true - -env: - REGISTRY: docker.io - TAG_SUFFIX: latest - -jobs: - changes: - runs-on: ubuntu-latest - outputs: - gpu-base: ${{ steps.filter.outputs.gpu-base || 'true' }} - steps: - - uses: actions/checkout@v7 - - # branch pushes rebuild the multi-gb torch base only when its - # definition changed; releases and manual runs always rebuild so - # published images never lag the release content - - uses: dorny/paths-filter@v3 - if: github.event_name == 'push' - id: filter - with: - filters: | - gpu-base: - - 'runpod/runtimes/gpu-base/**' - - '.github/workflows/runtimes.yml' - - test-runner: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install test deps - run: | - pip install -e . - pip install pytest pytest-asyncio cloudpickle - - - name: Run runtime tests - run: pytest tests/test_apps/test_tasks.py tests/test_apps/test_task_runner_http.py tests/test_apps/test_bootstrap.py -q -o addopts="" -p no:unraisableexception - - build-gpu-base: - runs-on: ubuntu-latest - needs: [changes, test-runner] - if: needs.changes.outputs.gpu-base == 'true' - strategy: - # a transient registry/mirror failure on one leg must not cancel - # the other python versions - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - steps: - - uses: actions/checkout@v7 - - - name: Clear space - run: | - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" - docker system prune -af - - - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push gpu base image - uses: docker/build-push-action@v6 - with: - context: . - file: runpod/runtimes/gpu-base/Dockerfile - platforms: linux/amd64 - build-args: | - PYTHON_VERSION=${{ matrix.python-version }} - push: true - tags: | - runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }} - cache-from: type=gha,scope=gpu-base-py${{ matrix.python-version }} - cache-to: type=gha,mode=max,scope=gpu-base-py${{ matrix.python-version }} - - build-runtimes: - runs-on: ubuntu-latest - # gpu legs build FROM the registry's gpu-base tag, so a skipped or - # failed gpu-base rebuild (e.g. torch mirror outage) must not block - # runtime images: the previously published base keeps working - needs: [test-runner, build-gpu-base] - if: always() && needs.test-runner.result == 'success' - strategy: - fail-fast: false - matrix: - kind: [task, queue, api] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - accelerator: [cpu, gpu] - steps: - - uses: actions/checkout@v7 - - - name: Clear space - if: matrix.accelerator == 'gpu' - run: | - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" - docker system prune -af - - - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Compute image names - id: names - run: | - if [ "${{ matrix.accelerator }}" = "gpu" ]; then - echo "base=runpod/gpu-base:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }}" >> "$GITHUB_OUTPUT" - repo="runpod/${{ matrix.kind }}-gpu" - else - echo "base=python:${{ matrix.python-version }}-slim" >> "$GITHUB_OUTPUT" - repo="runpod/${{ matrix.kind }}" - fi - tags="$repo:py${{ matrix.python-version }}-${{ env.TAG_SUFFIX }}" - # releases also pin a version tag for rollback - if [ "${{ github.event_name }}" = "release" ]; then - tags="$tags,$repo:py${{ matrix.python-version }}-${{ github.event.release.tag_name }}" - fi - echo "tags=$tags" >> "$GITHUB_OUTPUT" - - - name: Build and push runtime image - uses: docker/build-push-action@v6 - with: - context: . - file: runpod/runtimes/${{ matrix.kind }}/Dockerfile - platforms: linux/amd64 - build-args: | - BASE_IMAGE=${{ steps.names.outputs.base }} - push: true - tags: ${{ steps.names.outputs.tags }} - cache-from: type=gha,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} - cache-to: type=gha,mode=max,scope=${{ matrix.kind }}-${{ matrix.accelerator }}-py${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index 4a8c870c2..d514756a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,8 +52,6 @@ include-package-data = true runpod = [ "serverless/binaries/gpu_test", "serverless/binaries/README.md", - "runtimes/**/*.py", - "runtimes/**/Dockerfile*", ] [tool.setuptools_scm] diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 537449035..967f0d60c 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -35,11 +35,9 @@ } ) -# the worker runtime's actual dependency closure, a small subset of the -# runpod package's full dependency list. the runpod package is vendored -# without its dependencies (sdk/cli-only packages like boto3, paramiko, -# and cryptography never load in a worker), so these must be vendored -# explicitly to keep a dependency-free worker small. +# the worker's runpod dependency closure, a small subset of the full sdk +# dependency list. the package is vendored without sdk/cli-only packages +# that never load in a worker. RUNTIME_REQUIREMENTS = ( "aiohttp[speedups]", "aiohttp-retry", @@ -145,8 +143,8 @@ def _is_pinnable_name(spec: str) -> bool: ) -def runtime_requirement(scratch_dir: Path) -> str: - """the runpod runtime spec to vendor into the artifact. +def runpod_requirement(scratch_dir: Path) -> str: + """the runpod sdk spec to vendor into the artifact. RUNPOD_PACKAGE_SPEC overrides the published package (version pins, git refs, tarball urls, local checkouts). non-index specs are @@ -167,11 +165,9 @@ def runtime_requirement(scratch_dir: Path) -> str: def sync_running_package(env_dir: Path) -> None: """overlay the running runpod package onto the vendored env. - the package is pure python (py3-none-any), so the client's - installed tree is valid on the worker platform. this pins the - vendored runtime to the exact client version: wire protocols, - manifest handling, and worker code can never drift, and prerelease - clients work before their version reaches pypi. + the package is pure python (py3-none-any), so the client's installed + tree is valid on the worker platform. this pins sdk contracts and + manifest handling to the client that produced the artifact. """ import runpod as _runpod @@ -306,16 +302,6 @@ def vendor( ) -def _verify_runtime(env_dir: Path) -> None: - """the vendored runpod must include the worker runtimes.""" - if not (env_dir / "runpod" / "runtimes").is_dir(): - raise BuildError( - "the vendored runpod package has no runtimes modules; the " - "resolved version predates them. set RUNPOD_PACKAGE_SPEC to a " - "version that includes runpod.runtimes" - ) - - def build_environment( app, project_root: Path, @@ -363,7 +349,7 @@ def build_environment( if package_spec: vendor( env_dir, - [runtime_requirement(scratch)], + [runpod_requirement(scratch)], python_version, no_deps=True, progress=progress, @@ -381,7 +367,6 @@ def build_environment( if not package_spec: sync_running_package(env_dir) - _verify_runtime(env_dir) return BuildResult( env_dir=env_dir, diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index a294577bb..30d9723ff 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -1,6 +1,5 @@ """deploy pipeline: discovered apps -> manifest -> artifact -> activated build.""" -import base64 import fnmatch import json import logging @@ -24,7 +23,7 @@ ) from .errors import InvalidResourceError, ScheduleNotSupported from .schedule import SCHEDULES_ENABLED -from .shim import bootstrap_docker_args, bootstrap_source +from .shim import runtime_launcher from .spec import ResourceKind from .utils.events import emit @@ -235,12 +234,14 @@ def _deployed_endpoint_input( if api_key and "RUNPOD_API_KEY" not in template_env: template_env["RUNPOD_API_KEY"] = api_key - # workers that spawn tasks must select runtime images from the same - # channel the deploy used (the vendored env supplies the code, so - # only the image channel needs to travel) - tag = os.environ.get("RUNPOD_RUNTIME_TAG") - if tag and "RUNPOD_RUNTIME_TAG" not in template_env: - template_env["RUNPOD_RUNTIME_TAG"] = tag + # nested tasks use the same runtime and sdk channels as the endpoint + for key in ( + "RUNPOD_RUNTIME_TAG", + "RUNPOD_RUNTIME_PACKAGE_SPEC", + "RUNPOD_PACKAGE_SPEC", + ): + if value := os.environ.get(key): + template_env.setdefault(key, value) if spec.max_concurrency > 1: template_env["RUNPOD_MAX_CONCURRENCY"] = str(spec.max_concurrency) @@ -267,18 +268,11 @@ def _deployed_endpoint_input( payload["flashBootType"] = "FLASHBOOT" if spec.image: - # custom image: inject the bootstrap; the vendored env in the - # artifact provides the runtime once the bootstrap unpacks it - payload["template"]["env"].extend( - [ - { - "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode(bootstrap_source().encode()).decode(), - }, - {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, - ] + # custom images install the runtime before attaching the artifact + payload["template"]["env"].append( + {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value} ) - payload["template"]["dockerArgs"] = bootstrap_docker_args() + payload["template"]["dockerArgs"] = runtime_launcher(spec.kind.value) if spec.kind is ResourceKind.API: payload["type"] = "LB" diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 9ebd52f44..da3d2d0ec 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -6,7 +6,6 @@ api is the only source of truth; nothing is persisted locally. """ -import base64 import logging import os as _os from typing import Dict, List, Optional, Union @@ -14,7 +13,7 @@ from .api import AppsApiClient from .app import App from .handles import ApiHandle, FunctionHandle -from .shim import bootstrap_docker_args, bootstrap_source +from .shim import runtime_launcher from .spec import ResourceKind, ResourceSpec from .targets import LiveTarget from .utils.events import emit @@ -162,25 +161,14 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: payload["flashBootType"] = "FLASHBOOT" if spec.image: - # custom image: inject the bootstrap so the worker runtime starts - # regardless of what the image contains - payload["template"]["env"].extend( - [ - { - "key": "RUNPOD_BOOTSTRAP_B64", - "value": base64.b64encode(bootstrap_source().encode()).decode(), - }, - {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value}, - ] + # custom images install and start the independently released runtime + payload["template"]["env"].append( + {"key": "RUNPOD_RUNTIME_KIND", "value": spec.kind.value} ) - # the bootstrap pip-installs the runpod package on bare images; - # forward a pinned spec (e.g. a git branch during prerelease) - package_spec = _os.environ.get("RUNPOD_PACKAGE_SPEC") - if package_spec: - payload["template"]["env"].append( - {"key": "RUNPOD_PACKAGE_SPEC", "value": package_spec} - ) - payload["template"]["dockerArgs"] = bootstrap_docker_args() + for key in ("RUNPOD_RUNTIME_PACKAGE_SPEC", "RUNPOD_PACKAGE_SPEC"): + if value := _os.environ.get(key): + payload["template"]["env"].append({"key": key, "value": value}) + payload["template"]["dockerArgs"] = runtime_launcher(spec.kind.value) if spec.kind is ResourceKind.API: payload["type"] = "LB" if spec.datacenter: diff --git a/runpod/apps/images.py b/runpod/apps/images.py index 10d1400f5..fb00e1947 100644 --- a/runpod/apps/images.py +++ b/runpod/apps/images.py @@ -1,10 +1,9 @@ """runtime image selection. -one place maps (resource kind, gpu/cpu, python version) to the runtime -image built by the runtimes workflow. cpu images are python:X.Y-slim -based; gpu images additionally preinstall the torch family (matching -the build-time exclusion set) so deployed artifacts never install it -at cold start. +one place maps (resource kind, gpu/cpu, python version) to images +published from github.com/runpod-workers/sdk-runtime. cpu images are +python:X.Y-slim based; gpu images additionally preinstall the torch +family used by the build-time exclusion set. RUNPOD_RUNTIME_TAG selects the image channel (latest, dev, or a pinned version). diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py index c997315a2..8c71a54da 100644 --- a/runpod/apps/shim.py +++ b/runpod/apps/shim.py @@ -1,24 +1,16 @@ -"""shell launcher for booting injected runtime scripts on custom images. - -builds the dockerArgs command that materializes a base64 env payload -into a file and runs it with the image's python. constraints: - - - POSIX sh only (no bash on busybox/alpine images) - - python may not be on sh's PATH (conda/venv images), so well-known - interpreter locations are probed - - no base64 binary assumed: the discovered python does the decode - - no python at all is a configuration error: the whole feature is - running the user's python function on their image, so a pythonless - image gets a loud, clear failure - -the launcher must stay single-quote-free internally: the host parses -dockerArgs with a shell lexer and the whole script rides inside one -pair of single quotes. -""" +"""shell launcher for runtime modules on custom images. + +constraints: -from pathlib import Path + - posix sh only + - python may live outside the image's path + - the runtime package may already be baked into the image + - package overrides support prerelease and pinned runtime builds + +commands stay single-quote-free internally because the host parses +`dockerArgs` with a shell lexer and wraps the script in single quotes. +""" -# well-known interpreter locations beyond PATH _PYTHON_CANDIDATES = ( "python3", "python", @@ -32,9 +24,20 @@ "/usr/local/bin/python", ) +_RUNTIME_MODULES = { + "api": "runpod_sdk_runtime.bootstrap", + "queue": "runpod_sdk_runtime.bootstrap", + "task": "runpod_sdk_runtime.task.runner", +} + + +def runtime_launcher(kind: str) -> str: + """dockerArgs command that installs and starts a runtime module.""" + try: + module = _RUNTIME_MODULES[kind] + except KeyError as exc: + raise ValueError(f"unknown runtime kind: {kind}") from exc -def shell_launcher(env_var: str, dest: str) -> str: - """dockerArgs command that decodes $env_var into dest and execs it.""" probes = " ".join(_PYTHON_CANDIDATES) script = ( f'PY=""; ' @@ -45,19 +48,14 @@ def shell_launcher(env_var: str, dest: str) -> str: f'echo "[shim] FATAL: no python interpreter found in this image. "' f'"custom images must include python3." >&2; ' f"exit 1; fi; " - f'echo "${env_var}" | "$PY" -c ' - f'"import base64,sys;sys.stdout.buffer.write(base64.b64decode(sys.stdin.read()))" ' - f"> {dest} && " - f'exec "$PY" {dest}' + f'RUNTIME_SPEC="${{RUNPOD_RUNTIME_PACKAGE_SPEC:-runpod-sdk-runtime}}"; ' + f'if [ -n "${{RUNPOD_RUNTIME_PACKAGE_SPEC:-}}" ] || ' + f'! "$PY" -c "import runpod_sdk_runtime" >/dev/null 2>&1; then ' + f'"$PY" -m pip install -q --upgrade "$RUNTIME_SPEC" || exit 1; ' + f"fi; " + f'if [ -n "${{RUNPOD_PACKAGE_SPEC:-}}" ]; then ' + f'"$PY" -m pip install -q --upgrade "$RUNPOD_PACKAGE_SPEC" || exit 1; ' + f"fi; " + f'exec "$PY" -m {module}' ) return f"sh -c '{script}'" - - -def bootstrap_source() -> str: - """return the runtime bootstrap injected into custom images.""" - return (Path(__file__).parent.parent / "runtimes" / "bootstrap.py").read_text() - - -def bootstrap_docker_args() -> str: - """return the command that starts the injected runtime bootstrap.""" - return shell_launcher("RUNPOD_BOOTSTRAP_B64", "/bootstrap.py") diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index c04a82544..3d221c7fe 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -1,10 +1,8 @@ """ephemeral pod execution for @app.task. lifecycle per call: - 1. deploy a pod on a task runtime image (runpod/task*, built from - runtimes/task in this repo). custom images get the runner source - delivered base64-encoded in the pod env and booted via dockerArgs, - so any image with a python3 binary works. + 1. deploy a pod on a runpod/task runtime image. custom images install + the independently released runtime package via dockerArgs. 2. wait for the runner's /ping via the pod http proxy 3. POST the FunctionRequest to /execute (remote) or /submit (spawn) 4. collect the response, terminate the pod @@ -14,13 +12,11 @@ """ import asyncio -import base64 import logging import os as _os import secrets import time from datetime import datetime, timedelta, timezone -from pathlib import Path from typing import Any, Dict, List, Optional import aiohttp @@ -41,24 +37,11 @@ RESULT_POLL_INTERVAL = 2.0 -def _runner_source() -> str: - """single-file runner for bare-image bootstrap. +def _runtime_command() -> str: + """shell command that installs and starts the task runtime.""" + from .shim import runtime_launcher - the shared executor is concatenated ahead of the task server so the - shipped file has no runpod imports; the server's executor import - fails on the pod and falls through to the names already in scope. - """ - runtimes = Path(__file__).parent.parent / "runtimes" - executor = (runtimes / "executor.py").read_text() - server = (runtimes / "task" / "runner.py").read_text() - return f"{executor}\n\n{server}" - - -def _bootstrap_command() -> str: - """shell command that materializes and starts the runner.""" - from .shim import shell_launcher - - return shell_launcher("RUNPOD_TASK_RUNNER_B64", "/task_runner.py") + return runtime_launcher("task") def cuda_versions_at_least(minimum: str) -> List[str]: @@ -102,8 +85,8 @@ def _proxy_url(pod_id: str) -> str: def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any]: """build the pod deploy input for one task execution. - runtime images have the runner baked in (CMD starts it); custom - images bootstrap the runner from a base64 env payload via dockerArgs. + runtime images have the runner baked in; custom images install and + start the runtime package via dockerArgs. """ terminate_after = ( datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME @@ -132,14 +115,10 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] } if spec.image: - # custom image: inject the runner source and boot it explicitly - env["RUNPOD_TASK_RUNNER_B64"] = base64.b64encode( - _runner_source().encode() - ).decode() - package_spec = _os.environ.get("RUNPOD_PACKAGE_SPEC") - if package_spec: - env["RUNPOD_PACKAGE_SPEC"] = package_spec - pod["dockerArgs"] = _bootstrap_command() + for key in ("RUNPOD_RUNTIME_PACKAGE_SPEC", "RUNPOD_PACKAGE_SPEC"): + if value := _os.environ.get(key): + env[key] = value + pod["dockerArgs"] = _runtime_command() pod["env"] = [{"key": k, "value": v} for k, v in env.items()] if spec.datacenter: diff --git a/runpod/runtimes/README.md b/runpod/runtimes/README.md deleted file mode 100644 index d2675e3c0..000000000 --- a/runpod/runtimes/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# runtimes - -worker runtime scripts and the docker images built from them. images -are built and pushed by the `runtimes` github actions workflow. - -## cold-start model - -deployed workers do no dependency resolution. `rp deploy` vendors the -full runtime environment (the runpod package plus every python -dependency, resolved for the worker platform) into the build artifact -under `env/`. at cold start the bootstrap: - -1. **locate** the app tree: a host-provided pre-unpacked directory if - available, else extract the artifact tarball once (marker file - short-circuits warm restarts) -2. **attach** `env/` and the source tree to the worker's PYTHONPATH, - vendored env first so it wins over image packages -3. **verify** size-excluded packages (torch and friends) exist in the - image, installing only if genuinely absent (loudly, as a fallback) -4. **serve** exec the worker runtime - -torch-family packages are excluded from artifacts by size and expected -from the gpu worker images; the exclusion list is recorded in the -manifest as `excludedPackages`. - -if any phase fails, the worker does not crash-loop silently: queue -workers answer jobs with a structured `BootstrapError`, api workers -answer http requests with it. - -live mode (`rp dev`) has no artifact; source arrives per request and -the bootstrap only ensures the runtime package is importable (a no-op -on the baked images). - -## images - -every runtime kind ships a cpu and a gpu variant per python version. -gpu variants build on `runpod/gpu-base:pyX.Y-{tag}` (python:X.Y-slim -plus the torch family), which preinstalls exactly the packages the -build auto-excludes from artifacts. the two lists must stay in sync: -`SIZE_PROHIBITIVE_PACKAGES` in `runpod/apps/build.py` and the pip -install in `gpu-base/Dockerfile`. - -| image | base | use | -|---|---|---| -| `runpod/gpu-base:py{3.10-3.14}-{tag}` | `python:X.Y-slim` + torch | shared gpu base | -| `runpod/queue:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu queue endpoints | -| `runpod/queue-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu queue endpoints | -| `runpod/api:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu api endpoints | -| `runpod/api-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu api endpoints | -| `runpod/task:py{3.10-3.14}-{tag}` | `python:X.Y-slim` | cpu tasks | -| `runpod/task-gpu:py{3.10-3.14}-{tag}` | gpu-base | gpu tasks | - -image selection lives in `runpod.apps.images`; dev sessions and tasks -pick the variant matching the local python so pickled payloads stay -compatible. - -custom images work everywhere: the sdk injects `bootstrap.py` (queue/ -api) or `runner.py` (task) base64-encoded in an env var and boots it -via `dockerArgs`. both scripts are stdlib-only, so any image with a -python3 binary works. for deployed resources the vendored env then -provides everything else, including the runpod package itself. builds -targeting custom images vendor the torch family too (no auto -exclusion), since only the builtin gpu images guarantee it. - -## task - -single-shot task runner for `@app.task` pods. one pod runs one -function: the sdk deploys a pod, waits for `/ping`, posts a -`FunctionRequest` to `/execute` (or `/submit` for spawn), collects the -result, and terminates the pod. - -the wire protocol (`FunctionRequest` / `FunctionResponse`) is defined -in `runpod.apps.protocol`; the runner and sdk must stay in sync with -it. `runner.py` deliberately avoids importing the runpod package so it -can run standalone on arbitrary images. diff --git a/runpod/runtimes/__init__.py b/runpod/runtimes/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/runpod/runtimes/api/Dockerfile b/runpod/runtimes/api/Dockerfile deleted file mode 100644 index ded53a686..000000000 --- a/runpod/runtimes/api/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# api (load-balanced) worker image. BASE_IMAGE is python:X.Y-slim for -# cpu or runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). -ARG BASE_IMAGE=python:3.12-slim -FROM ${BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 \ - RUNPOD_RUNTIME_KIND=api - -RUN apt-get update && apt-get install -y --no-install-recommends \ - git ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# dependency layer: cached until requirements.txt changes, so code-only -# edits skip the full dependency install -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle "uvicorn>=0.30" \ - && rm /tmp/requirements.txt - -# install the runpod package (worker loop + runtimes) from the build -# context. the context has no .git, so setuptools-scm needs a pretend -# version -COPY . /src -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ - pip install --no-cache-dir --no-deps /src && rm -rf /src - -EXPOSE 80 -CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/runpod/runtimes/api/__init__.py b/runpod/runtimes/api/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/runpod/runtimes/api/server.py b/runpod/runtimes/api/server.py deleted file mode 100644 index a675fe31a..000000000 --- a/runpod/runtimes/api/server.py +++ /dev/null @@ -1,321 +0,0 @@ -"""generic api (load-balanced) server for app endpoints. - -serves an asgi app on the port runpod's load balancer routes to -(PORT env, default 80), with /ping kept healthy for LB health checks. - -two serving modes, chosen at startup: - -deployed mode (rp deploy): - the build artifact is unpacked at RUNPOD_APP_DIR and - FLASH_RESOURCE_NAME identifies this resource. the server imports - the user's module, finds the ApiHandle, and builds the asgi app: - - class-based api: instantiate the class, run its @init method - before /ping reports healthy, mount each @get/@post route - - asgi factory: call the factory, serve what it returns - -live mode (rp dev): - no artifact. the client pushes the user's module source to - /_runpod/sync (once per source change); the matching api class is - materialized from it and serves the real routes, so dev requests - hit the same handlers a deployed worker would. /execute remains - for FunctionRequest payloads. -""" - -import importlib -import inspect -import json -import logging -import os -import sys -from typing import Any, Optional - -log = logging.getLogger("runpod.runtimes.api") - -APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") -MANIFEST_NAME = "runpod_manifest.json" -PORT = int(os.environ.get("PORT", "80")) - - -def _resource_name() -> str: - return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( - "RUNPOD_RESOURCE_NAME", "" - ) - - -def _is_deployed() -> bool: - return bool(_resource_name()) and os.path.isfile( - os.path.join(APP_DIR, MANIFEST_NAME) - ) - - -def _load_api_handle(): - """import the user's module and return the ApiHandle for this resource.""" - with open(os.path.join(APP_DIR, MANIFEST_NAME)) as f: - manifest = json.load(f) - - name = _resource_name() - entry = next( - (r for r in manifest.get("resources", []) if r.get("name") == name), - None, - ) - if entry is None: - raise RuntimeError( - f"resource '{name}' not in manifest " - f"(has: {[r.get('name') for r in manifest.get('resources', [])]})" - ) - - if APP_DIR not in sys.path: - sys.path.insert(0, APP_DIR) - module = importlib.import_module(entry["module"]) - - from runpod.apps.handles import ApiHandle - - for attr in vars(module).values(): - if isinstance(attr, ApiHandle) and attr.spec.name == name: - return attr - raise RuntimeError( - f"no @app.api handle named '{name}' found in module '{entry['module']}'" - ) - - -async def _maybe_await(value: Any) -> Any: - if inspect.isawaitable(value): - return await value - return value - - -def _build_class_app(handle) -> Any: - """construct a fastapi app from an ApiHandle's decorated class. - - the class is instantiated once per worker; @init runs before /ping - reports healthy so the LB only routes to ready workers. - """ - from contextlib import asynccontextmanager - - from fastapi import FastAPI - - cls = handle._cls - instance = cls() - ready = {"ok": False} - - @asynccontextmanager - async def lifespan(_app): - if handle._init_name: - await _maybe_await(getattr(instance, handle._init_name)()) - ready["ok"] = True - yield - - app = FastAPI(title=handle.spec.name, lifespan=lifespan) - - @app.get("/ping") - async def ping(): - from fastapi.responses import JSONResponse - - if not ready["ok"]: - return JSONResponse({"status": "initializing"}, status_code=204) - return {"status": "healthy"} - - _mount_routes(app, handle, instance) - return app - - -def _mount_routes(app: Any, handle, instance) -> None: - """add each @get/@post route of an ApiHandle's class to a fastapi app.""" - from fastapi import Request - - for route in handle.spec.routes: - method = getattr(route, "method", None) or route["method"] - path = getattr(route, "path", None) or route["path"] - handler_name = ( - getattr(route, "handler_name", None) or route["handler"] - ) - bound = getattr(instance, handler_name) - - def make_endpoint(fn): - async def endpoint(request: Request): - body = None - if request.method in ("POST", "PUT", "PATCH", "DELETE"): - try: - body = await request.json() - except Exception: # noqa: BLE001 - empty/non-json body - body = None - if body is not None: - return await _maybe_await(fn(body)) - return await _maybe_await(fn()) - - return endpoint - - app.add_api_route( - path, make_endpoint(bound), methods=[method], name=handler_name - ) - - -def _build_factory_app(handle) -> Any: - """call the user's asgi factory and ensure /ping exists. - - fastapi/starlette apps get a /ping route added when missing; other - asgi callables are wrapped so /ping is answered before delegating. - """ - app = handle._asgi_factory() - - routes = getattr(app, "routes", None) - if routes is not None and hasattr(app, "get"): - if not any(getattr(r, "path", None) == "/ping" for r in routes): - - @app.get("/ping") - async def ping(): - return {"status": "healthy"} - - return app - - async def with_ping(scope, receive, send): - if scope["type"] == "http" and scope["path"] == "/ping": - body = b'{"status": "healthy"}' - await send( - { - "type": "http.response.start", - "status": 200, - "headers": [ - (b"content-type", b"application/json"), - (b"content-length", str(len(body)).encode()), - ], - } - ) - await send({"type": "http.response.body", "body": body}) - return - await app(scope, receive, send) - - return with_ping - - -def _install_live_dependencies(request: dict) -> None: - """install a live resource's dependencies before its module runs. - - dev workers boot on the bare runtime image; deployed workers get - dependencies baked into the build artifact, so live mode installs - them at sync time to match. - """ - from runpod.runtimes.executor import _install, _install_system - - error = _install_system(request.get("system_dependencies")) - if error: - raise RuntimeError(error) - error = _install(request.get("dependencies"), "dependencies") - if error: - raise RuntimeError(error) - - -async def _materialize_live_api(source: str, resource: str) -> Any: - """exec shipped module source and build the api app for a resource. - - @init runs here, before the app is swapped in, so the first - routed request already sees initialized state. - """ - from fastapi import FastAPI - - from runpod.apps.handles import ApiHandle - from runpod.runtimes.executor import _materialize_source - - path = _materialize_source(source) - namespace: dict = {"__name__": "__runpod_live__", "__file__": path} - exec(compile(source, path, "exec"), namespace) # noqa: S102 - - handle = None - for value in namespace.values(): - if isinstance(value, ApiHandle) and value.spec.name == resource: - handle = value - break - if handle is None: - raise RuntimeError( - f"no @app.api resource named '{resource}' in shipped module" - ) - if handle._cls is None: - return _build_factory_app(handle) - - instance = handle._cls() - if handle._init_name: - await _maybe_await(getattr(instance, handle._init_name)()) - app = FastAPI(title=f"{resource} (live)") - _mount_routes(app, handle, instance) - return app - - -class _LiveDispatcher: - """asgi front for dev sessions. - - the client pushes module source to /_runpod/sync; the api app is - materialized from it (rebuilt when the source hash changes) and - serves every route. /ping and /execute always work. - """ - - def __init__(self): - self._core = self._build_core() - self._inner: Any = None - self._hash: Optional[str] = None - - def _build_core(self) -> Any: - import hashlib - - from fastapi import FastAPI - - app = FastAPI(title="runpod-live-api") - - @app.get("/ping") - async def ping(): - return {"status": "healthy"} - - @app.post("/execute") - async def execute(request: dict): - from runpod.runtimes.executor import execute_request - - return execute_request(request.get("input", request)) - - @app.post("/_runpod/sync") - async def sync(request: dict): - source = request.get("source") or "" - resource = request.get("resource") or "" - digest = hashlib.sha256(source.encode()).hexdigest() - if digest != self._hash: - _install_live_dependencies(request) - self._inner = await _materialize_live_api(source, resource) - self._hash = digest - return {"status": "synced", "hash": digest} - - return app - - async def __call__(self, scope, receive, send): - if scope["type"] != "http": - return await self._core(scope, receive, send) - path = scope.get("path", "") - if path in ("/ping", "/execute", "/_runpod/sync") or self._inner is None: - return await self._core(scope, receive, send) - return await self._inner(scope, receive, send) - - -def _build_live_app() -> Any: - return _LiveDispatcher() - - -def build_app() -> Any: - if _is_deployed(): - handle = _load_api_handle() - if handle._cls is not None: - return _build_class_app(handle) - return _build_factory_app(handle) - return _build_live_app() - - -def main() -> None: - import uvicorn - - uvicorn.run( - build_app(), - host="0.0.0.0", - port=PORT, - timeout_keep_alive=600, - log_level="info", - ) - - -if __name__ == "__main__": - main() diff --git a/runpod/runtimes/bootstrap.py b/runpod/runtimes/bootstrap.py deleted file mode 100644 index 1ba19a774..000000000 --- a/runpod/runtimes/bootstrap.py +++ /dev/null @@ -1,443 +0,0 @@ -"""stdlib-only entrypoint for app workers. - -cold starts do no dependency resolution: the build artifact ships a -vendored environment (env/ inside the artifact) containing the runpod -runtime and every python dependency, resolved at deploy time for the -worker platform. this entrypoint only has to make that environment -visible and exec the worker. - -deployed mode (FLASH_RESOURCE_NAME set): - locate find the app tree: a pre-unpacked directory if the host - provides one, else extract the artifact tarball once - attach put {app} and {app}/env at the front of PYTHONPATH - verify packages the build excluded by size (torch and friends) - must come from the image; genuinely absent ones are - installed as a fallback, loudly - system apt packages from the resource's system_dependencies; - these cannot ride in the artifact, so this is the one - install that legitimately remains at cold start - serve exec the worker runtime from the vendored environment - -live mode (no resource name, `rp dev`): - no artifact exists; source arrives per request. the runtime images - bake the runpod package in, custom images get it installed here. - serve exec the worker runtime - -if any phase fails the worker does not crash-loop silently: a minimal -job-take loop answers every job with a structured BootstrapError so -the failure surfaces in job responses (queue endpoints), or an http -server answers every request with the error (api endpoints). - -stdlib only: this file runs on the image's python before any -environment exists. it is baked into the runtime images as CMD and -injected into custom images via RUNPOD_BOOTSTRAP_B64 + dockerArgs. -""" - -import importlib.util -import json -import os -import subprocess -import sys -import tarfile -import time -import urllib.request - -ARTIFACT_PATH = os.environ.get( - "FLASH_BUILD_ARTIFACT_PATH", "/root/.runpod/artifact.tar.gz" -) -# a host-provided, already-unpacked app tree; when present, unpacking -# is skipped entirely -PREBUILT_APP_DIR = os.environ.get("RUNPOD_PREBUILT_APP_DIR", "") -APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") -ENV_SUBDIR = "env" -ARTIFACT_WAIT_SECONDS = int(os.environ.get("RUNPOD_ARTIFACT_WAIT", "300")) -MANIFEST_NAME = "runpod_manifest.json" -UNPACK_MARKER = ".rp-unpacked" - -WORKER_MODULES = { - "queue": "runpod.runtimes.queue.worker", - "api": "runpod.runtimes.api.server", -} - - -def _log(message): - sys.stderr.write(f"[bootstrap] {message}\n") - sys.stderr.flush() - - -class PhaseError(Exception): - def __init__(self, phase, detail): - self.phase = phase - self.detail = detail - super().__init__(f"{phase}: {detail}") - - -def _runtime_kind(): - return os.environ.get("RUNPOD_RUNTIME_KIND", "queue") - - -def _resource_name(): - return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( - "RUNPOD_RESOURCE_NAME", "" - ) - - -# ---------------------------------------------------------------- locate - - -def _locate(): - """return the app tree root, unpacking the artifact if needed.""" - if PREBUILT_APP_DIR and os.path.isdir(PREBUILT_APP_DIR): - _log(f"using pre-unpacked app tree at {PREBUILT_APP_DIR}") - return PREBUILT_APP_DIR - - marker = os.path.join(APP_DIR, UNPACK_MARKER) - if os.path.isfile(marker): - _log(f"app tree already unpacked at {APP_DIR}") - return APP_DIR - - deadline = time.monotonic() + ARTIFACT_WAIT_SECONDS - while not os.path.isfile(ARTIFACT_PATH): - if time.monotonic() >= deadline: - raise PhaseError( - "locate", - f"artifact not found at {ARTIFACT_PATH} after " - f"{ARTIFACT_WAIT_SECONDS}s; is this endpoint part of an " - f"app deployment?", - ) - time.sleep(2) - - os.makedirs(APP_DIR, exist_ok=True) - target = os.path.realpath(APP_DIR) - started = time.monotonic() - try: - with tarfile.open(ARTIFACT_PATH, mode="r:*") as tar: - if hasattr(tarfile, "data_filter"): - # 3.12+: the data filter rejects traversal, devices, - # and absolute paths - tar.extractall(path=APP_DIR, filter="data") - else: - members = tar.getmembers() - for member in members: - dest = os.path.realpath( - os.path.join(APP_DIR, member.name) - ) - if not dest.startswith(target + os.sep) and dest != target: - raise PhaseError( - "locate", f"unsafe tar member path: {member.name}" - ) - # links can point outside and later members write - # through them; artifacts never contain links - if member.issym() or member.islnk(): - raise PhaseError( - "locate", - f"unsafe tar link member: {member.name}", - ) - if not (member.isfile() or member.isdir()): - raise PhaseError( - "locate", - f"unsupported tar member type: {member.name}", - ) - tar.extractall(path=APP_DIR, members=members) # noqa: S202 - members validated above - except (OSError, tarfile.TarError) as exc: - raise PhaseError("locate", f"failed to extract artifact: {exc}") - - with open(marker, "w") as f: - f.write(str(time.time())) - _log( - f"artifact extracted to {APP_DIR} " - f"in {time.monotonic() - started:.1f}s" - ) - return APP_DIR - - -# ---------------------------------------------------------------- attach - - -def _attach(app_dir): - """paths that must lead sys.path in the worker process. - - the vendored env comes first so the artifact's runpod (matched to - the client that built it) wins over anything baked into the image. - """ - env_dir = os.path.join(app_dir, ENV_SUBDIR) - paths = [app_dir] - if os.path.isdir(env_dir): - paths.insert(0, env_dir) - else: - _log( - f"artifact has no vendored env at {env_dir}; " - f"falling back to image packages" - ) - return paths - - -def _manifest(app_dir): - path = os.path.join(app_dir, MANIFEST_NAME) - if not os.path.isfile(path): - raise PhaseError("attach", f"manifest not found at {path}") - with open(path) as f: - return json.load(f) - - -def _worker_importable(paths): - """can the worker module be imported with these paths leading?""" - module = WORKER_MODULES[_runtime_kind()] - probe = ( - "import sys; sys.path[:0] = sys.argv[1:]; " - f"import importlib; importlib.import_module({module!r})" - ) - result = subprocess.run( - [sys.executable, "-c", probe, *paths], - capture_output=True, - text=True, - ) - return result.returncode == 0, result.stderr[-2000:] - - -# ---------------------------------------------------------------- verify - - -def _pip_install(packages, phase): - env = dict(os.environ) - # tarball specs have no .git for setuptools-scm version inference - env.setdefault("SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD", "0.0.0.dev0") - result = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", "--upgrade", *packages], - capture_output=True, - text=True, - env=env, - ) - if result.returncode != 0: - raise PhaseError(phase, result.stderr[-3000:]) - - -def _resource_entry(manifest): - name = _resource_name() - for entry in manifest.get("resources", []): - if entry.get("name") == name: - return entry - return {} - - -def _install_system(manifest): - """apt packages cannot be vendored; install this resource's - system_dependencies at cold start.""" - packages = _resource_entry(manifest).get("systemDependencies") or [] - if not packages: - return - import shutil - - if shutil.which("apt-get") is None: - raise PhaseError( - "system", - f"system dependencies {packages} requested but apt-get is " - f"not available in this image; use a debian-based image or " - f"bake them in", - ) - env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") - _log(f"installing system dependencies: {packages}") - update = subprocess.run( - ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env - ) - if update.returncode != 0: - raise PhaseError("system", f"apt-get update failed: {update.stderr[-2000:]}") - result = subprocess.run( - ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], - capture_output=True, - text=True, - env=env, - ) - if result.returncode != 0: - raise PhaseError( - "system", - f"failed to install system dependencies {packages}: " - f"{result.stderr[-2000:]}", - ) - - -def _verify_excluded(manifest): - """packages the build excluded must come from the image. - - a miss means the image was swapped for one without them; install as - a fallback so the worker still comes up, but say so loudly because - it re-adds the cold-start cost the exclusion existed to avoid. - """ - excluded = manifest.get("excludedPackages") or [] - missing = [ - name - for name in excluded - if importlib.util.find_spec(name.replace("-", "_")) is None - ] - if not missing: - return - _log( - f"WARNING: excluded packages {missing} are not in this image; " - f"installing at cold start. use an image that provides them " - f"(e.g. runpod/pytorch) to avoid this cost." - ) - _pip_install(missing, "verify") - - -# ---------------------------------------------------------------- serve - - -def _serve(paths): - """exec the worker runtime with the environment paths leading.""" - module = WORKER_MODULES[_runtime_kind()] - env = dict(os.environ) - existing = env.get("PYTHONPATH", "") - env["PYTHONPATH"] = os.pathsep.join( - paths + ([existing] if existing else []) - ) - if paths: - env["RUNPOD_APP_DIR"] = paths[-1] - os.execve( - sys.executable, - [sys.executable, "-m", module], - env, - ) - - -# ------------------------------------------------------------- live mode - - -def _ensure_runtime_installed(): - """live mode on a custom image: the runtime must be importable. - - baked runtime images make this a no-op. --upgrade matters: the - image may carry an older runpod without the runtimes modules, and - a plain install would no-op against it. - """ - ok, _ = _worker_importable([]) - if ok: - return - spec = os.environ.get("RUNPOD_PACKAGE_SPEC", "runpod") - _log(f"worker runtime not in image, installing {spec}") - packages = [spec, "cloudpickle"] - if _runtime_kind() == "api": - packages.append("uvicorn>=0.30") - _pip_install(packages, "runtime") - ok, err = _worker_importable([]) - if not ok: - raise PhaseError( - "runtime", - f"installed {spec} but the worker runtime is still not " - f"importable: {err}. set RUNPOD_PACKAGE_SPEC to a version " - f"that includes runpod.runtimes.", - ) - - -# ----------------------------------------------------------- error surface - - -def _error_payload(error): - return { - "error_type": "BootstrapError", - "error_message": ( - f"worker bootstrap failed during '{error.phase}': {error.detail}" - ), - } - - -def _queue_error_loop(error): - """answer queue jobs with the bootstrap error.""" - get_url = os.environ.get("RUNPOD_WEBHOOK_GET_JOB", "") - post_url = os.environ.get("RUNPOD_WEBHOOK_POST_OUTPUT", "") - api_key = os.environ.get("RUNPOD_AI_API_KEY", "") - if not get_url or not post_url: - _log("no job webhook env; exiting with error") - sys.exit(1) - - payload = json.dumps({"error": json.dumps(_error_payload(error))}).encode() - - _log("starting error-reporting loop") - while True: - try: - req = urllib.request.Request( - get_url, headers={"Authorization": api_key} - ) - with urllib.request.urlopen(req, timeout=90) as resp: - if resp.status != 200: - time.sleep(2) - continue - job = json.loads(resp.read() or b"{}") - job_id = job.get("id") - if not job_id: - continue - done = urllib.request.Request( - post_url.replace("$ID", job_id), - data=payload, - headers={ - "Authorization": api_key, - "Content-Type": "application/x-www-form-urlencoded", - }, - method="POST", - ) - urllib.request.urlopen(done, timeout=30) - _log(f"reported bootstrap error for job {job_id}") - except Exception as exc: # noqa: BLE001 - loop must survive - _log(f"error loop: {exc}") - time.sleep(5) - - -def _api_error_server(error): - """answer every http request with the bootstrap error.""" - from http.server import BaseHTTPRequestHandler, HTTPServer - - body = json.dumps(_error_payload(error)).encode() - port = int(os.environ.get("PORT", "80")) - - class Handler(BaseHTTPRequestHandler): - def _respond(self): - self.send_response(500) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = _respond - - def log_message(self, *args): - pass - - _log(f"starting error-reporting http server on :{port}") - HTTPServer(("0.0.0.0", port), Handler).serve_forever() - - -def _report_error(error): - _log(f"FAILED {error}") - if _runtime_kind() == "api": - _api_error_server(error) - else: - _queue_error_loop(error) - - -# ------------------------------------------------------------------ main - - -def main(): - try: - if _resource_name(): - app_dir = _locate() - paths = _attach(app_dir) - manifest = _manifest(app_dir) - _verify_excluded(manifest) - _install_system(manifest) - ok, err = _worker_importable(paths) - if not ok: - raise PhaseError( - "attach", - f"worker runtime not importable from the artifact " - f"env or the image: {err}", - ) - _serve(paths) - else: - _ensure_runtime_installed() - _serve([]) - except PhaseError as exc: - _report_error(exc) - - -if __name__ == "__main__": - main() diff --git a/runpod/runtimes/executor.py b/runpod/runtimes/executor.py deleted file mode 100644 index 104704a04..000000000 --- a/runpod/runtimes/executor.py +++ /dev/null @@ -1,291 +0,0 @@ -"""shared FunctionRequest execution engine. - -runs one function from shipped source: installs dependencies, execs the -module, resolves the function, and serializes the result per the -request's serialization_format. every runtime that executes live -requests (task pods, queue workers in dev mode, api servers) drives -this same engine. - -stdlib-only (cloudpickle optional, required only for the cloudpickle -serialization format) so it can ship as part of a single-file bootstrap -onto any python image. -""" - -import base64 -import inspect -import io -import json -import os -import shutil -import subprocess -import sys -import threading -import traceback -from contextlib import redirect_stdout - - -def _load_cloudpickle(install: bool = False): - try: - import cloudpickle - - return cloudpickle - except ImportError: - if not install: - return None - # bare images (custom image= without the runtime baked in) may lack - # cloudpickle; install it on demand the same way dependencies are - if _install(["cloudpickle"], "cloudpickle"): - return None - import cloudpickle - - return cloudpickle - - -def _install(packages, label): - if not packages: - return None - result = subprocess.run( - [sys.executable, "-m", "pip", "install", "-q", *packages], - capture_output=True, - text=True, - ) - if result.returncode != 0: - return f"failed to install {label}: {result.stderr[-2000:]}" - return None - - -_apt_updated = threading.Event() - - -def _install_system(packages): - """install apt packages; requires a debian-family image with root.""" - if not packages: - return None - if shutil.which("apt-get") is None: - return ( - f"system dependencies {packages} requested but apt-get is not " - f"available in this image; use a debian-based image or bake " - f"them in" - ) - env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") - if not _apt_updated.is_set(): - update = subprocess.run( - ["apt-get", "update", "-qq"], capture_output=True, text=True, env=env - ) - if update.returncode != 0: - return f"apt-get update failed: {update.stderr[-2000:]}" - _apt_updated.set() - result = subprocess.run( - ["apt-get", "install", "-y", "-qq", "--no-install-recommends", *packages], - capture_output=True, - text=True, - env=env, - ) - if result.returncode != 0: - return ( - f"failed to install system dependencies {packages}: " - f"{result.stderr[-2000:]}" - ) - return None - - -def _deserialize_args(request): - fmt = request.get("serialization_format", "cloudpickle") - args = request.get("args") or [] - kwargs = request.get("kwargs") or {} - if fmt == "json": - return list(args), dict(kwargs) - cloudpickle = _load_cloudpickle(install=True) - if cloudpickle is None: - raise RuntimeError("cloudpickle not available for argument deserialization") - args = [cloudpickle.loads(base64.b64decode(a)) for a in args] - kwargs = {k: cloudpickle.loads(base64.b64decode(v)) for k, v in kwargs.items()} - return args, kwargs - - -def _serialize_result(result, fmt): - if fmt == "json": - # json-format requests promise json on the wire; failing here - # keeps dev behavior identical to a deployed endpoint - try: - json.dumps(result) - except (TypeError, ValueError) as exc: - raise TypeError( - f"return value must be json-serializable " - f"(this request uses the json wire format): {exc}" - ) from exc - return {"json_result": result} - cloudpickle = _load_cloudpickle(install=True) - if cloudpickle is None: - return {"json_result": result} - return {"result": base64.b64encode(cloudpickle.dumps(result)).decode("utf-8")} - - -def resolve_request(request): - """prepare one FunctionRequest for execution. - - installs dependencies, execs the shipped source, and resolves the - target function. returns ((fn, args, kwargs), None) on success or - (None, response_dict) on failure. - """ - error = _install_system(request.get("system_dependencies")) - if error: - return None, {"success": False, "error": error} - error = _install(request.get("dependencies"), "dependencies") - if error: - return None, {"success": False, "error": error} - - function_name = request.get("function_name") - function_code = request.get("function_code") - if not function_name or not function_code: - return None, { - "success": False, - "error": "function_name and function_code are required", - } - - # exec mirrors deployed-mode module import: the code is the - # user's full module, so __name__ is set like an import would - # (main guards stay inert) and decorated handles resolve to - # their wrapped functions. the source is written to a real - # file first so inspect.getsource works inside the function - # (nested .remote() calls re-extract sibling source) - source_path = _materialize_source(function_code) - namespace = {"__name__": "__runpod_live__", "__file__": source_path} - code_obj = compile(function_code, source_path, "exec") - exec(code_obj, namespace) # noqa: S102 - that is the job - if function_name not in namespace: - return None, { - "success": False, - "error": f"function '{function_name}' not found in provided code", - } - fn = namespace[function_name] - fn = getattr(fn, "_fn", fn) - - args, kwargs = _deserialize_args(request) - return (fn, args, kwargs), None - - -def serialize_chunk(chunk, request): - """wrap one generator chunk in the response envelope. - - the __stream__ marker lets clients tell a stream of chunks apart - from a single aggregated function response. - """ - response = {"success": True, "__stream__": True} - response.update( - _serialize_result( - chunk, request.get("serialization_format", "cloudpickle") - ) - ) - return response - - -def execute_request(request): - """run one FunctionRequest to completion, returning a response dict.""" - stdout_io = io.StringIO() - try: - prepared, error_response = resolve_request(request) - if error_response is not None: - return error_response - fn, args, kwargs = prepared - - # stdout is teed: captured for the job response and written - # through to the real stdout so container logs (and live log - # streaming in dev sessions) carry the function's prints. - # sys.stderr carries the runner's own request logs from other - # threads, which must not leak into job output - with redirect_stdout(_Tee(stdout_io, sys.__stdout__)): - result = fn(*args, **kwargs) - if hasattr(result, "__await__"): - result = _run_awaitable(result) - # generators aggregate so live .remote() matches the - # deployed worker's return_aggregate_stream output - elif inspect.isgenerator(result): - result = list(result) - elif inspect.isasyncgen(result): - result = _run_awaitable(_drain_async_gen(result)) - - response = {"success": True, "stdout": stdout_io.getvalue()} - response.update( - _serialize_result( - result, request.get("serialization_format", "cloudpickle") - ) - ) - return response - except Exception: # noqa: BLE001 - all failures go over the wire - return { - "success": False, - "error": traceback.format_exc(), - "stdout": stdout_io.getvalue(), - } - - -def _materialize_source(function_code): - """persist request source to a stable path for inspect/linecache.""" - import hashlib - import tempfile - - digest = hashlib.sha256(function_code.encode()).hexdigest()[:16] - path = os.path.join( - tempfile.gettempdir(), f"runpod_live_{digest}.py" - ) - if not os.path.exists(path): - with open(path, "w") as f: - f.write(function_code) - return path - - -class _Tee(io.TextIOBase): - """write-through to multiple streams, flushing eagerly so log - followers see lines as they happen.""" - - def __init__(self, *streams): - self._streams = [s for s in streams if s is not None] - - def write(self, s): - for stream in self._streams: - try: - stream.write(s) - stream.flush() - except (ValueError, OSError): - # one sink failing (closed pipe) must not lose the - # write on the other sinks - pass - return len(s) - - def flush(self): - for stream in self._streams: - try: - stream.flush() - except (ValueError, OSError): - # closed sinks are skipped, same as write - pass - - -def _run_awaitable(awaitable): - """drive an awaitable to completion from sync code. - - task pods run this on plain threads, but queue workers in live mode - call execute_request from inside the serverless event loop, where - asyncio.run would raise; a private loop on a helper thread covers - both. - """ - import asyncio - - try: - asyncio.get_running_loop() - except RuntimeError: - return asyncio.run(_await(awaitable)) - - import concurrent.futures - - with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: - return pool.submit(asyncio.run, _await(awaitable)).result() - - -async def _drain_async_gen(agen): - return [chunk async for chunk in agen] - - -async def _await(awaitable): - return await awaitable diff --git a/runpod/runtimes/gpu-base/Dockerfile b/runpod/runtimes/gpu-base/Dockerfile deleted file mode 100644 index 5bc0d2c97..000000000 --- a/runpod/runtimes/gpu-base/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# shared gpu base for the runtime images: python:X.Y-slim plus the -# torch family. this preinstalled set must stay in sync with -# SIZE_PROHIBITIVE_PACKAGES in runpod/apps/build.py: deploys targeting -# builtin gpu images exclude exactly these packages from artifacts and -# expect the image to provide them. torch cuda wheels bundle the cuda -# runtime; the host injects the driver. -ARG PYTHON_VERSION=3.12 -FROM python:${PYTHON_VERSION}-slim - -# nvidia container runtime keys on these to inject the host driver; -# without them torch sees no gpu even on a gpu machine -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 \ - NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# cu128 wheels: default pypi torch is cu130, which needs 580+ drivers -# that much of the fleet does not have yet. cu128 runs on 12.8+ drivers. -# pinned + extra-index (not index-url): the cuda index lacks common -# deps like typing-extensions, and the pin prevents drifting back to -# pypi's cu130 build. -RUN pip install --no-cache-dir \ - "torch==2.9.1+cu128" "torchvision==0.24.1+cu128" "torchaudio==2.9.1+cu128" \ - --extra-index-url https://download.pytorch.org/whl/cu128 diff --git a/runpod/runtimes/queue/Dockerfile b/runpod/runtimes/queue/Dockerfile deleted file mode 100644 index bbaecee68..000000000 --- a/runpod/runtimes/queue/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# queue worker image. BASE_IMAGE is python:X.Y-slim for cpu or -# runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). -ARG BASE_IMAGE=python:3.12-slim -FROM ${BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 \ - RUNPOD_RUNTIME_KIND=queue - -RUN apt-get update && apt-get install -y --no-install-recommends \ - git ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# dependency layer: cached until requirements.txt changes, so code-only -# edits skip the full dependency install -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle \ - && rm /tmp/requirements.txt - -# install the runpod package (worker loop + runtimes) from the build -# context. the context has no .git, so setuptools-scm needs a pretend -# version -COPY . /src -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ - pip install --no-cache-dir --no-deps /src && rm -rf /src - -CMD ["python", "-m", "runpod.runtimes.bootstrap"] diff --git a/runpod/runtimes/queue/__init__.py b/runpod/runtimes/queue/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/runpod/runtimes/queue/worker.py b/runpod/runtimes/queue/worker.py deleted file mode 100644 index 72f2857be..000000000 --- a/runpod/runtimes/queue/worker.py +++ /dev/null @@ -1,192 +0,0 @@ -"""generic queue worker for app endpoints. - -two serving modes, chosen at startup: - -deployed mode (rp deploy): - the build artifact is unpacked at RUNPOD_APP_DIR and - FLASH_RESOURCE_NAME identifies this resource. the worker imports - the user's module, resolves the decorated function from the - manifest, runs its @init hook, and serves jobs by calling the - function body directly. job input is plain kwargs; source never - travels with requests. - -live mode (rp dev): - no artifact. each job carries a FunctionRequest (source + - serialized args) which is executed via the shared engine in - runpod.runtimes.executor. -""" - -import importlib -import inspect -import json -import os -import sys - -import runpod - -APP_DIR = os.environ.get("RUNPOD_APP_DIR", "/app") -MANIFEST_NAME = "runpod_manifest.json" - - -def _resource_name() -> str: - return os.environ.get("FLASH_RESOURCE_NAME") or os.environ.get( - "RUNPOD_RESOURCE_NAME", "" - ) - - -def _is_deployed() -> bool: - return bool(_resource_name()) and os.path.isfile( - os.path.join(APP_DIR, MANIFEST_NAME) - ) - - -def _load_deployed_handle(): - """import the user's module and return the FunctionHandle for this - resource.""" - with open(os.path.join(APP_DIR, MANIFEST_NAME)) as f: - manifest = json.load(f) - - name = _resource_name() - entry = next( - (r for r in manifest.get("resources", []) if r.get("name") == name), - None, - ) - if entry is None: - raise RuntimeError( - f"resource '{name}' not in manifest " - f"(has: {[r.get('name') for r in manifest.get('resources', [])]})" - ) - - if APP_DIR not in sys.path: - sys.path.insert(0, APP_DIR) - module = importlib.import_module(entry["module"]) - - from runpod.apps.handles import FunctionHandle - - for attr in vars(module).values(): - if ( - isinstance(attr, FunctionHandle) - and attr.spec.name == name - ): - return attr - raise RuntimeError( - f"no @app.queue/@app.task handle named '{name}' found in " - f"module '{entry['module']}'" - ) - - -def _job_kwargs(job: dict) -> dict: - body = dict(job.get("input") or {}) - body.pop("__empty", None) - return body - - -def _make_deployed_handler(handle): - """serverless handler that calls the user function directly. - - generator functions become generator handlers so the serverless - core streams partial outputs to /stream as they are yielded. - """ - fn = handle._fn - - if inspect.isasyncgenfunction(fn): - - async def async_gen_handler(job: dict): - async for chunk in fn(**_job_kwargs(job)): - yield chunk - - return async_gen_handler - - if inspect.isgeneratorfunction(fn): - - def gen_handler(job: dict): - yield from fn(**_job_kwargs(job)) - - return gen_handler - - async def handler(job: dict) -> dict: - result = fn(**_job_kwargs(job)) - if inspect.isawaitable(result): - result = await result - return result - - return handler - - -def _run_init(handle) -> None: - """run the resource's @init hook before serving.""" - init_fn = getattr(handle, "_init_fn", None) - if init_fn is None: - return - result = init_fn() - if inspect.isawaitable(result): - import asyncio - - asyncio.run(result) - - -async def _live_handler(job: dict): - """execute a FunctionRequest, streaming when the function is a - generator so dev sessions behave exactly like deployed workers.""" - from runpod.runtimes.executor import ( - execute_request, - resolve_request, - serialize_chunk, - ) - - request = job.get("input") or {} - prepared, error_response = resolve_request(request) - if error_response is not None: - yield error_response - return - fn, args, kwargs = prepared - - if inspect.isasyncgenfunction(fn): - async for chunk in fn(*args, **kwargs): - yield serialize_chunk(chunk, request) - return - if inspect.isgeneratorfunction(fn): - for chunk in fn(*args, **kwargs): - yield serialize_chunk(chunk, request) - return - - yield execute_request(request) - - -def _max_concurrency() -> int: - """jobs one worker may run at once (RUNPOD_MAX_CONCURRENCY, min 1).""" - try: - return max(1, int(os.environ.get("RUNPOD_MAX_CONCURRENCY", "1"))) - except ValueError: - return 1 - - -def _worker_config(handler) -> dict: - config: dict = {"handler": handler} - from runpod.serverless.modules.rp_handler import is_generator - - if is_generator(handler): - # generator jobs stream partials and still finish with a full - # output, so .remote()/result() work alongside .stream() - config["return_aggregate_stream"] = True - concurrency = _max_concurrency() - if concurrency > 1: - config["concurrency_modifier"] = lambda _current: concurrency - return config - - -def main() -> None: - if _is_deployed(): - handle = _load_deployed_handle() - _run_init(handle) - runpod.serverless.start( - _worker_config(_make_deployed_handler(handle)) - ) - else: - # the live handler is a generator so it can stream; aggregation - # keeps .remote() working for plain functions and generators alike - runpod.serverless.start(_worker_config(_live_handler)) - - -if __name__ == "__main__": - main() diff --git a/runpod/runtimes/task/Dockerfile b/runpod/runtimes/task/Dockerfile deleted file mode 100644 index 1e981d9ea..000000000 --- a/runpod/runtimes/task/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# task runner image. BASE_IMAGE is python:X.Y-slim for cpu or -# runpod/gpu-base:pyX.Y-* for gpu (adds the torch family). -ARG BASE_IMAGE=python:3.12-slim -FROM ${BASE_IMAGE} - -ENV DEBIAN_FRONTEND=noninteractive \ - TZ=Etc/UTC \ - PYTHONUNBUFFERED=1 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - git ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - -# dependency layer: cached until requirements.txt changes, so code-only -# edits skip the full dependency install -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt cloudpickle \ - && rm /tmp/requirements.txt - -# install the runpod package: task payloads carry the user's full module -# source, which imports runpod (decorators, nested .remote calls). -# the context has no .git, so setuptools-scm needs a pretend version -COPY . /src -RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD=0.0.0.dev0 \ - pip install --no-cache-dir --no-deps /src && rm -rf /src - -EXPOSE 8080 -CMD ["python", "-m", "runpod.runtimes.task.runner"] diff --git a/runpod/runtimes/task/__init__.py b/runpod/runtimes/task/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/runpod/runtimes/task/runner.py b/runpod/runtimes/task/runner.py deleted file mode 100644 index 0b3e6d9c7..000000000 --- a/runpod/runtimes/task/runner.py +++ /dev/null @@ -1,218 +0,0 @@ -"""single-shot task runner: the process a task pod boots into. - -a minimal http server speaking the FunctionRequest/FunctionResponse -protocol, driving the shared execution engine in -runpod.runtimes.executor. - -endpoints: - GET /ping readiness probe (unauthenticated) - POST /execute run a function, block, return the response - POST /submit start a function in the background, return immediately - GET /result status/result of the submitted job - -auth: every endpoint except /ping requires - Authorization: Bearer $RUNPOD_TASK_TOKEN - -one pod runs one function; the client terminates the pod after -collecting the result. - -ships to bare images as a single file: the sdk concatenates the -executor source above this module, so the import below is satisfied -either way. -""" - -import json -import os -import sys -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -try: - from runpod.runtimes.executor import execute_request -except ImportError: - # single-file bootstrap: the executor source is concatenated above - # this module and its names are already in scope - pass - -PORT = int(os.environ.get("RUNPOD_TASK_PORT", "8080")) -TOKEN = os.environ.get("RUNPOD_TASK_TOKEN", "") - -# watchdog: self-terminate when the client is clearly gone. RUNNING -# jobs are never killed (terminateAfter is the runaway backstop); -# NONE means the client died before submitting, DONE means the result -# sat uncollected. normal flows poll every ~2s, so these never fire -# for a live client. -IDLE_TIMEOUT = float(os.environ.get("RUNPOD_TASK_IDLE_TIMEOUT", "600")) -WATCHDOG_INTERVAL = 15.0 - -# single background job slot for /submit + /result -_job_lock = threading.Lock() -_job_state = {"status": "NONE", "response": None} -_last_contact = {"ts": None} # set at server start -# inline /execute requests in flight; the watchdog must not terminate -# the pod while one runs (long executes outlive the idle timeout) -_inline_executions = {"count": 0} - - -def _touch_contact(): - import time - - _last_contact["ts"] = time.time() - - -def _should_self_terminate(status, last_contact, now, idle_timeout, inline=0): - """the watchdog decision: kill only provably-abandoned pods.""" - if status == "RUNNING" or inline > 0: - return False - if last_contact is None: - return False - return (now - last_contact) > idle_timeout - - -def _terminate_self(): - """terminate this pod via the injected pod-scoped api key. - - every pod carries a RUNPOD_API_KEY scoped to itself; podTerminate - with it removes the pod entirely. exiting the process is the - fallback (stops the workload; terminateAfter finishes the job). - """ - import json as _json - import urllib.request - - pod_id = os.environ.get("RUNPOD_POD_ID") - api_key = os.environ.get("RUNPOD_API_KEY") - if pod_id and api_key: - try: - api_base = os.environ.get( - "RUNPOD_API_BASE_URL", "https://api.runpod.io" - ) - payload = _json.dumps( - { - "query": ( - "mutation podTerminate($input: PodTerminateInput!) " - "{ podTerminate(input: $input) }" - ), - "variables": {"input": {"podId": pod_id}}, - } - ).encode() - request = urllib.request.Request( - f"{api_base}/graphql", - data=payload, - headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}", - }, - ) - urllib.request.urlopen(request, timeout=30) # noqa: S310 - sys.stderr.write("[task-runner] self-terminated (abandoned)\n") - except Exception: # noqa: BLE001 - fall through to process exit - pass - os._exit(0) - - -def _watchdog(): - import time - - while True: - time.sleep(WATCHDOG_INTERVAL) - if _should_self_terminate( - _job_state["status"], - _last_contact["ts"], - time.time(), - IDLE_TIMEOUT, - inline=_inline_executions["count"], - ): - _terminate_self() - - -class Handler(BaseHTTPRequestHandler): - def _send(self, code, payload): - body = json.dumps(payload).encode("utf-8") - self.send_response(code) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def _authed(self): - header = self.headers.get("Authorization", "") - authed = TOKEN and header == f"Bearer {TOKEN}" - if authed: - # any authenticated contact proves the client is alive - _touch_contact() - return authed - - def _read_request(self): - length = int(self.headers.get("Content-Length", "0")) - return json.loads(self.rfile.read(length) or b"{}") - - def do_GET(self): # noqa: N802 - BaseHTTPRequestHandler api - if self.path == "/ping": - self._send(200, {"ready": True}) - return - if self.path == "/result": - if not self._authed(): - self._send(401, {"error": "unauthorized"}) - return - with _job_lock: - self._send( - 200, - { - "status": _job_state["status"], - "response": _job_state["response"], - }, - ) - return - self._send(404, {"error": "not found"}) - - def do_POST(self): # noqa: N802 - BaseHTTPRequestHandler api - if not self._authed(): - self._send(401, {"error": "unauthorized"}) - return - if self.path == "/execute": - with _job_lock: - _inline_executions["count"] += 1 - try: - self._send(200, execute_request(self._read_request())) - finally: - with _job_lock: - _inline_executions["count"] -= 1 - return - if self.path == "/submit": - request = self._read_request() - with _job_lock: - if _job_state["status"] == "RUNNING": - self._send(409, {"error": "a job is already running"}) - return - _job_state["status"] = "RUNNING" - _job_state["response"] = None - - def run(): - response = execute_request(request) - with _job_lock: - _job_state["status"] = "DONE" - _job_state["response"] = response - - threading.Thread(target=run, daemon=True).start() - self._send(200, {"status": "RUNNING"}) - return - self._send(404, {"error": "not found"}) - - def log_message(self, format, *args): # noqa: A002 - stdlib signature - # request logging is noise in container logs (dev sessions - # stream them as the function's output) - pass - - -def main() -> None: - if not TOKEN: - sys.stderr.write("[task-runner] RUNPOD_TASK_TOKEN not set, exiting\n") - sys.exit(1) - _touch_contact() - threading.Thread(target=_watchdog, daemon=True).start() - server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/tests/e2e/examples/README.md b/tests/e2e/examples/README.md index d535d4f4b..6f26d6263 100644 --- a/tests/e2e/examples/README.md +++ b/tests/e2e/examples/README.md @@ -31,10 +31,11 @@ rp login rp secret add ex-demo-secret --value anything # for 09_secrets ``` -`12_custom_image` installs the runpod package at cold start; until the -release with `runpod.runtimes` is on pypi, point the bootstrap at the -branch tarball (plain https, not git+ — slim images have no git): +`12_custom_image` installs the runtime and sdk packages at cold start. +prerelease runs can point both installers at branch archives (plain +https so slim images do not need git): ```bash +export RUNPOD_RUNTIME_PACKAGE_SPEC=https://github.com/runpod-workers/sdk-runtime/archive/refs/heads/main.tar.gz export RUNPOD_PACKAGE_SPEC=https://github.com/runpod/runpod-python/archive/refs/heads/feat/apps-sdk.tar.gz ``` diff --git a/tests/test_apps/test_api_server.py b/tests/test_apps/test_api_server.py deleted file mode 100644 index e3ee2496b..000000000 --- a/tests/test_apps/test_api_server.py +++ /dev/null @@ -1,262 +0,0 @@ -"""tests for the api runtime server.""" - -import json -import textwrap - -import pytest - -pytest.importorskip("fastapi") -from fastapi.testclient import TestClient - -from runpod.apps.app import _clear_registry -from runpod.runtimes.api import server - - -@pytest.fixture(autouse=True) -def clean_registry(): - _clear_registry() - yield - _clear_registry() - - -def _write_project(tmp_path, monkeypatch, module="main_cls"): - (tmp_path / f"{module}.py").write_text( - textwrap.dedent( - """ - import runpod - from runpod import App, init, get, post - - app = App("api-test") - - @app.api(name="inference", cpu="cpu3c-1-2") - class Inference: - @init - def setup(self): - self.model = "loaded" - - @get("/health") - def health(self): - return {"model": self.model} - - @post("/generate") - async def generate(self, body: dict): - return {"echo": body, "model": self.model} - """ - ) - ) - manifest = { - "version": 1, - "app": "api-test", - "resources": [ - { - "kind": "api", - "name": "inference", - "module": module, - "routes": [ - {"method": "GET", "path": "/health", "handler": "health"}, - { - "method": "POST", - "path": "/generate", - "handler": "generate", - }, - ], - } - ], - } - (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) - monkeypatch.setattr(server, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "inference") - - -class TestDeployedClassApi: - def test_routes_and_init(self, tmp_path, monkeypatch): - _write_project(tmp_path, monkeypatch) - app = server.build_app() - with TestClient(app) as client: - # startup ran init before serving - response = client.get("/ping") - assert response.status_code == 200 - assert response.json() == {"status": "healthy"} - - response = client.get("/health") - assert response.json() == {"model": "loaded"} - - response = client.post("/generate", json={"prompt": "hi"}) - assert response.json() == { - "echo": {"prompt": "hi"}, - "model": "loaded", - } - - def test_missing_resource_errors(self, tmp_path, monkeypatch): - _write_project(tmp_path, monkeypatch) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "nope") - with pytest.raises(RuntimeError, match="nope"): - server.build_app() - - -class TestFactoryApi: - def test_asgi_factory_served(self, tmp_path, monkeypatch): - (tmp_path / "main_factory.py").write_text( - textwrap.dedent( - """ - from runpod import App - - app = App("api-test") - - @app.api(name="web", cpu="cpu3c-1-2") - def web(): - from fastapi import FastAPI - - server = FastAPI() - - @server.post("/echo") - async def echo(body: dict): - return body - - return server - """ - ) - ) - manifest = { - "version": 1, - "app": "api-test", - "resources": [ - {"kind": "api", "name": "web", "module": "main_factory"} - ], - } - (tmp_path / "runpod_manifest.json").write_text(json.dumps(manifest)) - monkeypatch.setattr(server, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "web") - - app = server.build_app() - with TestClient(app) as client: - assert client.get("/ping").json() == {"status": "healthy"} - assert client.post("/echo", json={"a": 1}).json() == {"a": 1} - - -class TestLiveApi: - def test_execute_endpoint(self, monkeypatch): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) - app = server.build_app() - with TestClient(app) as client: - assert client.get("/ping").json() == {"status": "healthy"} - response = client.post( - "/execute", - json={ - "input": { - "function_name": "f", - "function_code": "def f(a, b):\n return a * b", - "args": [6, 7], - "kwargs": {}, - "serialization_format": "json", - } - }, - ) - data = response.json() - assert data["success"] is True - assert data["json_result"] == 42 - - -class TestLiveDispatcher: - def test_sync_materializes_routes(self): - from fastapi.testclient import TestClient - - from runpod.runtimes.api.server import _build_live_app - - source = ( - "import runpod\n" - "from runpod.apps.markers import get, init, post\n" - "app = runpod.App('live')\n" - "@app.api(cpu='cpu3c-1-2')\n" - "class Counter:\n" - " @init\n" - " def setup(self):\n" - " self.count = 0\n" - " @post('/bump')\n" - " async def bump(self, body: dict):\n" - " self.count += body.get('by', 1)\n" - " return {'count': self.count}\n" - " @get('/value')\n" - " async def value(self):\n" - " return {'count': self.count}\n" - ) - - client = TestClient(_build_live_app()) - # before sync: core routes only - assert client.get("/ping").status_code == 200 - assert client.post("/bump", json={}).status_code == 404 - - r = client.post( - "/_runpod/sync", json={"source": source, "resource": "Counter"} - ) - assert r.status_code == 200 - - assert client.post("/bump", json={"by": 3}).json() == {"count": 3} - assert client.get("/value").json() == {"count": 3} - # core routes still work after materialization - assert client.get("/ping").status_code == 200 - - def test_sync_installs_dependencies_before_materializing(self): - from unittest.mock import patch - - from fastapi.testclient import TestClient - - from runpod.runtimes.api.server import _build_live_app - - source = ( - "import runpod\n" - "from runpod.apps.markers import get\n" - "app = runpod.App('live-deps')\n" - "@app.api(cpu='cpu3c-1-2', dependencies=['somepkg'])\n" - "class D:\n" - " @get('/ok')\n" - " async def ok(self):\n" - " return {'ok': True}\n" - ) - client = TestClient(_build_live_app()) - with ( - patch( - "runpod.runtimes.executor._install", return_value=None - ) as install, - patch( - "runpod.runtimes.executor._install_system", - return_value=None, - ) as install_system, - ): - r = client.post( - "/_runpod/sync", - json={ - "source": source, - "resource": "D", - "dependencies": ["somepkg"], - "system_dependencies": ["libfoo"], - }, - ) - assert r.status_code == 200 - install.assert_called_once_with(["somepkg"], "dependencies") - install_system.assert_called_once_with(["libfoo"]) - assert client.get("/ok").json() == {"ok": True} - - def test_resync_same_source_keeps_state(self): - from fastapi.testclient import TestClient - - from runpod.runtimes.api.server import _build_live_app - - source = ( - "import runpod\n" - "from runpod.apps.markers import post\n" - "app = runpod.App('live2')\n" - "@app.api(cpu='cpu3c-1-2')\n" - "class S:\n" - " @post('/add')\n" - " async def add(self, body: dict):\n" - " self.total = getattr(self, 'total', 0) + body['x']\n" - " return {'total': self.total}\n" - ) - client = TestClient(_build_live_app()) - client.post("/_runpod/sync", json={"source": source, "resource": "S"}) - assert client.post("/add", json={"x": 2}).json() == {"total": 2} - # identical source: no rebuild, state survives - client.post("/_runpod/sync", json={"source": source, "resource": "S"}) - assert client.post("/add", json={"x": 3}).json() == {"total": 5} diff --git a/tests/test_apps/test_bootstrap.py b/tests/test_apps/test_bootstrap.py deleted file mode 100644 index f2790c8df..000000000 --- a/tests/test_apps/test_bootstrap.py +++ /dev/null @@ -1,459 +0,0 @@ -"""tests for the queue bootstrap and custom-image support.""" - -import base64 -import json - -import pytest - -import runpod -from runpod.apps import App -from runpod.apps.app import _clear_registry -from runpod.apps.dev import _endpoint_input -from runpod.runtimes import bootstrap - - -@pytest.fixture(autouse=True) -def clean_registry(): - _clear_registry() - yield - _clear_registry() - - -class TestLocatePhase: - def test_missing_artifact_fails_with_phase(self, tmp_path, monkeypatch): - monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(tmp_path / "nope.tar.gz")) - monkeypatch.setattr(bootstrap, "ARTIFACT_WAIT_SECONDS", 0) - monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path / "app")) - with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._locate() - assert exc_info.value.phase == "locate" - - def test_extracts_artifact(self, tmp_path, monkeypatch): - import tarfile - - src = tmp_path / "src" - src.mkdir() - (src / "main.py").write_text("x = 1") - artifact = tmp_path / "artifact.tar.gz" - with tarfile.open(artifact, "w:gz") as tar: - tar.add(src / "main.py", arcname="main.py") - - app_dir = tmp_path / "app" - monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) - monkeypatch.setattr(bootstrap, "APP_DIR", str(app_dir)) - assert bootstrap._locate() == str(app_dir) - assert (app_dir / "main.py").read_text() == "x = 1" - - def test_unpack_happens_once(self, tmp_path, monkeypatch): - import tarfile - - src = tmp_path / "src" - src.mkdir() - (src / "main.py").write_text("x = 1") - artifact = tmp_path / "artifact.tar.gz" - with tarfile.open(artifact, "w:gz") as tar: - tar.add(src / "main.py", arcname="main.py") - - app_dir = tmp_path / "app" - monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) - monkeypatch.setattr(bootstrap, "APP_DIR", str(app_dir)) - bootstrap._locate() - # second boot on a warm container: marker short-circuits - artifact.unlink() - assert bootstrap._locate() == str(app_dir) - - def test_prebuilt_dir_skips_unpack(self, tmp_path, monkeypatch): - prebuilt = tmp_path / "prebuilt" - prebuilt.mkdir() - monkeypatch.setattr(bootstrap, "PREBUILT_APP_DIR", str(prebuilt)) - # no artifact anywhere; the host-provided tree wins outright - monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(tmp_path / "no.tar.gz")) - assert bootstrap._locate() == str(prebuilt) - - def test_rejects_traversal(self, tmp_path, monkeypatch): - import io - import tarfile - - artifact = tmp_path / "evil.tar.gz" - with tarfile.open(artifact, "w:gz") as tar: - data = b"evil" - info = tarfile.TarInfo(name="../../etc/evil") - info.size = len(data) - tar.addfile(info, io.BytesIO(data)) - - monkeypatch.setattr(bootstrap, "ARTIFACT_PATH", str(artifact)) - monkeypatch.setattr(bootstrap, "APP_DIR", str(tmp_path / "app")) - with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._locate() - assert exc_info.value.phase == "locate" - - -class TestAttachPhase: - def test_env_dir_leads_path_order(self, tmp_path): - (tmp_path / "env").mkdir() - paths = bootstrap._attach(str(tmp_path)) - assert paths == [str(tmp_path / "env"), str(tmp_path)] - - def test_no_env_dir_falls_back_to_source_only(self, tmp_path): - paths = bootstrap._attach(str(tmp_path)) - assert paths == [str(tmp_path)] - - def test_manifest_read(self, tmp_path): - (tmp_path / "runpod_manifest.json").write_text( - json.dumps({"resources": [{"name": "q1"}]}) - ) - manifest = bootstrap._manifest(str(tmp_path)) - assert manifest["resources"][0]["name"] == "q1" - - def test_manifest_missing_fails_with_phase(self, tmp_path): - with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._manifest(str(tmp_path)) - assert exc_info.value.phase == "attach" - - -class TestSystemPhase: - def test_no_system_deps_no_op(self): - bootstrap._install_system({"resources": [{"name": "q"}]}) - - def test_missing_apt_fails_with_phase(self, monkeypatch): - import shutil - - monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") - monkeypatch.setattr(shutil, "which", lambda _: None) - manifest = { - "resources": [{"name": "q", "systemDependencies": ["ffmpeg"]}] - } - with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._install_system(manifest) - assert exc_info.value.phase == "system" - assert "ffmpeg" in str(exc_info.value) - - def test_installs_resource_system_deps(self, monkeypatch): - import shutil - - monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") - monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - calls = [] - - class R: - returncode = 0 - stderr = "" - - monkeypatch.setattr( - bootstrap.subprocess, - "run", - lambda cmd, **kw: calls.append(cmd) or R(), - ) - manifest = { - "resources": [{"name": "q", "systemDependencies": ["ffmpeg", "sox"]}] - } - bootstrap._install_system(manifest) - assert calls[0][:2] == ["apt-get", "update"] - assert calls[1][:3] == ["apt-get", "install", "-y"] - assert "ffmpeg" in calls[1] and "sox" in calls[1] - - def test_other_resources_deps_ignored(self, monkeypatch): - monkeypatch.setenv("FLASH_RESOURCE_NAME", "q") - manifest = { - "resources": [{"name": "other", "systemDependencies": ["ffmpeg"]}] - } - # q has no system deps; must not attempt anything - bootstrap._install_system(manifest) - - -class TestVerifyPhase: - def test_no_exclusions_no_op(self): - bootstrap._verify_excluded({"resources": []}) - - def test_present_exclusions_no_install(self, monkeypatch): - installed = [] - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: installed.extend(pkgs) - ) - # json is definitely importable; stands in for torch-in-image - bootstrap._verify_excluded({"excludedPackages": ["json"]}) - assert installed == [] - - def test_missing_exclusions_installed(self, monkeypatch): - installed = [] - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: installed.extend(pkgs) - ) - bootstrap._verify_excluded( - {"excludedPackages": ["definitely-not-installed-xyz"]} - ) - assert installed == ["definitely-not-installed-xyz"] - - -class TestCustomImagePayloads: - def test_queue_custom_image_gets_bootstrap(self): - app = App("a") - - @app.queue(name="q", cpu="cpu3c-1-2", image="pytorch/pytorch:latest") - def q(): - pass - - payload = _endpoint_input(app, q.spec) - template = payload["template"] - assert template["imageName"] == "pytorch/pytorch:latest" - assert "bootstrap.py" in template["dockerArgs"] - env = {e["key"]: e["value"] for e in template["env"]} - decoded = base64.b64decode(env["RUNPOD_BOOTSTRAP_B64"]).decode() - assert "def _locate" in decoded - - def test_queue_default_image_no_bootstrap(self): - app = App("a") - - @app.queue(name="q", cpu="cpu3c-1-2") - def q(): - pass - - payload = _endpoint_input(app, q.spec) - template = payload["template"] - from runpod.apps.images import local_python_version - - assert template["imageName"].startswith( - f"runpod/queue:py{local_python_version()}-" - ) - assert template["dockerArgs"] == "" - env = {e["key"]: e["value"] for e in template["env"]} - assert "RUNPOD_BOOTSTRAP_B64" not in env - - def test_api_custom_image_supported(self): - app = App("a") - - @app.api(name="api", cpu="cpu3c-1-2", image="my/server:1") - class Api: - @runpod.post("/x") - def x(self, body: dict): - return body - - payload = _endpoint_input(app, Api.spec) - assert payload["template"]["imageName"] == "my/server:1" - - def test_image_in_manifest(self): - app = App("a") - - @app.queue(name="q", cpu="cpu3c-1-2", image="custom:1") - def q(): - pass - - assert q.spec.to_manifest()["imageName"] == "custom:1" - - -class TestBootstrapIsStdlibOnly: - def test_no_third_party_imports(self): - import ast - from pathlib import Path - - source = ( - Path(bootstrap.__file__).read_text() - ) - tree = ast.parse(source) - # module-level imports only: function-local imports (the runpod - # availability probe in _ensure_runtime) run after installation - imports = set() - for node in tree.body: - if isinstance(node, ast.Import): - imports.update(a.name.split(".")[0] for a in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0: - imports.add(node.module.split(".")[0]) - stdlib = { - "json", "os", "subprocess", "sys", "tarfile", "time", - "urllib", "io", "importlib", "http", - } - assert imports <= stdlib, f"non-stdlib imports: {imports - stdlib}" - - -class TestErrorSurface: - def test_error_payload_shape(self): - error = bootstrap.PhaseError("locate", "artifact missing") - payload = bootstrap._error_payload(error) - assert payload["error_type"] == "BootstrapError" - assert "locate" in payload["error_message"] - assert "artifact missing" in payload["error_message"] - - def test_queue_error_loop_exits_without_webhooks(self, monkeypatch): - monkeypatch.delenv("RUNPOD_WEBHOOK_GET_JOB", raising=False) - monkeypatch.delenv("RUNPOD_WEBHOOK_POST_OUTPUT", raising=False) - with pytest.raises(SystemExit): - bootstrap._queue_error_loop(bootstrap.PhaseError("locate", "x")) - - def test_queue_error_loop_reports_jobs(self, monkeypatch): - import contextlib - import io - - monkeypatch.setenv("RUNPOD_WEBHOOK_GET_JOB", "http://jobs/get") - monkeypatch.setenv("RUNPOD_WEBHOOK_POST_OUTPUT", "http://jobs/out/$ID") - monkeypatch.setenv("RUNPOD_AI_API_KEY", "k") - - posted = [] - - @contextlib.contextmanager - def _get_response(): - response = io.BytesIO(json.dumps({"id": "job1"}).encode()) - response.status = 200 - yield response - - def fake_urlopen(req, timeout=None): - url = req.full_url - if url.endswith("/get"): - return _get_response() - posted.append((url, req.data)) - # first report done: break out of the infinite loop - raise KeyboardInterrupt - - monkeypatch.setattr( - bootstrap.urllib.request, "urlopen", fake_urlopen - ) - error = bootstrap.PhaseError("attach", "manifest gone") - with pytest.raises(KeyboardInterrupt): - bootstrap._queue_error_loop(error) - - assert posted - url, body = posted[0] - assert url.endswith("/out/job1") - assert b"BootstrapError" in body - - def test_report_error_routes_by_kind(self, monkeypatch): - calls = [] - monkeypatch.setattr( - bootstrap, "_api_error_server", lambda e: calls.append("api") - ) - monkeypatch.setattr( - bootstrap, "_queue_error_loop", lambda e: calls.append("queue") - ) - error = bootstrap.PhaseError("locate", "x") - - monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") - bootstrap._report_error(error) - monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "queue") - bootstrap._report_error(error) - assert calls == ["api", "queue"] - - -class TestLiveRuntimeInstall: - def test_importable_runtime_no_op(self, monkeypatch): - monkeypatch.setattr( - bootstrap, "_worker_importable", lambda paths: (True, "") - ) - installs = [] - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) - ) - bootstrap._ensure_runtime_installed() - assert installs == [] - - def test_installs_package_spec(self, monkeypatch): - probes = iter([(False, "no module"), (True, "")]) - monkeypatch.setattr( - bootstrap, "_worker_importable", lambda paths: next(probes) - ) - installs = [] - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) - ) - monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==1.99.0") - bootstrap._ensure_runtime_installed() - assert installs == [["runpod==1.99.0", "cloudpickle"]] - - def test_api_kind_adds_uvicorn(self, monkeypatch): - probes = iter([(False, "no module"), (True, "")]) - monkeypatch.setattr( - bootstrap, "_worker_importable", lambda paths: next(probes) - ) - installs = [] - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: installs.append(pkgs) - ) - monkeypatch.delenv("RUNPOD_PACKAGE_SPEC", raising=False) - monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") - bootstrap._ensure_runtime_installed() - assert installs == [["runpod", "cloudpickle", "uvicorn>=0.30"]] - - def test_install_failure_raises_phase(self, monkeypatch): - monkeypatch.setattr( - bootstrap, "_worker_importable", lambda paths: (False, "still broken") - ) - monkeypatch.setattr( - bootstrap, "_pip_install", lambda pkgs, phase: None - ) - with pytest.raises(bootstrap.PhaseError) as exc_info: - bootstrap._ensure_runtime_installed() - assert exc_info.value.phase == "runtime" - - -class TestServe: - def test_serve_execs_worker_module(self, monkeypatch): - execs = [] - monkeypatch.setattr( - bootstrap.os, - "execve", - lambda exe, argv, env: execs.append((exe, argv, env)), - ) - monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "queue") - monkeypatch.setenv("PYTHONPATH", "/existing") - bootstrap._serve(["/env", "/app"]) - exe, argv, env = execs[0] - assert argv[-1] == bootstrap.WORKER_MODULES["queue"] - assert env["PYTHONPATH"].split(":")[:2] == ["/env", "/app"] - assert "/existing" in env["PYTHONPATH"] - assert env["RUNPOD_APP_DIR"] == "/app" - - def test_serve_live_mode_no_paths(self, monkeypatch): - execs = [] - monkeypatch.setattr( - bootstrap.os, - "execve", - lambda exe, argv, env: execs.append((exe, argv, env)), - ) - monkeypatch.delenv("PYTHONPATH", raising=False) - monkeypatch.setenv("RUNPOD_RUNTIME_KIND", "api") - bootstrap._serve([]) - _, argv, env = execs[0] - assert argv[-1] == bootstrap.WORKER_MODULES["api"] - assert "RUNPOD_APP_DIR" not in env - - -class TestMain: - def test_deployed_path_serves(self, tmp_path, monkeypatch): - app_dir = str(tmp_path) - (tmp_path / bootstrap.MANIFEST_NAME).write_text( - json.dumps({"resources": [{"name": "chat"}]}) - ) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - monkeypatch.setattr(bootstrap, "_locate", lambda: app_dir) - monkeypatch.setattr( - bootstrap, "_worker_importable", lambda paths: (True, "") - ) - served = [] - monkeypatch.setattr(bootstrap, "_serve", lambda paths: served.append(paths)) - bootstrap.main() - assert served and served[0][-1] == app_dir - - def test_phase_error_reports(self, monkeypatch): - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - monkeypatch.setattr( - bootstrap, - "_locate", - lambda: (_ for _ in ()).throw( - bootstrap.PhaseError("locate", "gone") - ), - ) - reported = [] - monkeypatch.setattr( - bootstrap, "_report_error", lambda e: reported.append(e) - ) - bootstrap.main() - assert reported and reported[0].phase == "locate" - - def test_live_path(self, monkeypatch): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) - monkeypatch.setattr( - bootstrap, "_ensure_runtime_installed", lambda: None - ) - served = [] - monkeypatch.setattr(bootstrap, "_serve", lambda paths: served.append(paths)) - bootstrap.main() - assert served == [[]] diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index 4fbe8f13e..eebcf9588 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -11,7 +11,7 @@ BuildError, collect_requirements, requirement_name, - runtime_requirement, + runpod_requirement, split_exclusions, vendor, ) @@ -104,10 +104,10 @@ def test_exclusion_set_is_size_based(self): } -class TestRuntimeRequirement: +class TestRunpodRequirement: def test_default_is_published_package(self, tmp_path, monkeypatch): monkeypatch.delenv("RUNPOD_PACKAGE_SPEC", raising=False) - assert runtime_requirement(tmp_path) == "runpod" + assert runpod_requirement(tmp_path) == "runpod" def test_running_package_overlays_env(self, tmp_path): from runpod.apps.build import sync_running_package @@ -119,8 +119,7 @@ def test_running_package_overlays_env(self, tmp_path): sync_running_package(env_dir) - # the client's own tree (which has the runtimes) wins - assert (env_dir / "runpod" / "runtimes" / "bootstrap.py").is_file() + # the client's sdk tree wins assert (env_dir / "runpod" / "apps" / "build.py").is_file() assert not any( p.name == "__pycache__" @@ -130,7 +129,7 @@ def test_running_package_overlays_env(self, tmp_path): def test_version_pin_passes_through(self, tmp_path, monkeypatch): monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==1.8.0") - assert runtime_requirement(tmp_path) == "runpod==1.8.0" + assert runpod_requirement(tmp_path) == "runpod==1.8.0" def test_url_spec_builds_wheel(self, tmp_path, monkeypatch): monkeypatch.setenv( @@ -149,7 +148,7 @@ class R: return R() with patch("runpod.apps.build.subprocess.run", side_effect=fake_run): - assert runtime_requirement(tmp_path) == str(wheel) + assert runpod_requirement(tmp_path) == str(wheel) def _fake_popen(captured, *, returncode=0, stdout="", stderr=""): diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 4ad35661e..6f5a723c3 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -111,6 +111,39 @@ def x(self, body: dict): assert payload["type"] == "LB" assert payload["scalerType"] == "REQUEST_COUNT" + def test_custom_image_starts_external_runtime(self, monkeypatch): + monkeypatch.setenv( + "RUNPOD_RUNTIME_PACKAGE_SPEC", "runpod-sdk-runtime==1.2.3" + ) + app = App("a") + + @app.queue(name="q", cpu="cpu3c-1-2", image="my/image:1") + def q(): + pass + + payload = _endpoint_input(app, q.spec) + template = payload["template"] + env = {entry["key"]: entry["value"] for entry in template["env"]} + assert template["imageName"] == "my/image:1" + assert "runpod_sdk_runtime.bootstrap" in template["dockerArgs"] + assert env["RUNPOD_RUNTIME_KIND"] == "queue" + assert env["RUNPOD_RUNTIME_PACKAGE_SPEC"] == ( + "runpod-sdk-runtime==1.2.3" + ) + + def test_custom_api_uses_shared_bootstrap(self): + app = App("a") + + @app.api(name="api", cpu="cpu3c-1-2", image="my/api:1") + class Api: + @runpod.post("/x") + def x(self, body: dict): + return body + + payload = _endpoint_input(app, Api.spec) + assert payload["type"] == "LB" + assert "runpod_sdk_runtime.bootstrap" in payload["template"]["dockerArgs"] + def test_datacenter_pins_locations(self): app = App("a") diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index a6dd03889..f091c825c 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -446,62 +446,6 @@ def test_whole_module_ships(self, tmp_path): finally: sys.modules.pop("shipmod", None) - def test_execute_request_unwraps_handles(self): - # a shipped module defines decorated handles; the runner must - # call the wrapped function, not the handle - from runpod.runtimes.executor import execute_request - - code = ( - "import runpod\n" - "app = runpod.App('t')\n" - "@app.queue()\n" - "def double(x):\n" - " return x * 2\n" - ) - response = execute_request( - { - "function_name": "double", - "function_code": code, - "args": [], - "kwargs": {"x": "AAA"}, - "serialization_format": "json", - } - ) - assert response["success"], response.get("error") - assert response["json_result"] == "AAAAAA" - - def test_nested_source_extraction_from_shipped_module(self): - # inside a live worker, a function calling sibling.remote() - # re-extracts that sibling's source via inspect; the shipped - # module must be inspectable after exec - from runpod.runtimes.executor import execute_request - - code = ( - "import runpod\n" - "app = runpod.App('t2')\n" - "@app.queue()\n" - "def sibling(x):\n" - " return x\n" - "@app.queue()\n" - "def caller():\n" - " from runpod.apps.serialization import get_function_source\n" - " return get_function_source(sibling._fn)\n" - ) - response = execute_request( - { - "function_name": "caller", - "function_code": code, - "args": [], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"], response.get("error") - # the whole module re-ships (decorators intact), matching what - # the first hop sent - assert "def sibling" in response["json_result"] - assert "@app.queue()" in response["json_result"] - def test_missing_module_on_deserialize_is_actionable(self): import base64 diff --git a/tests/test_apps/test_executor.py b/tests/test_apps/test_executor.py deleted file mode 100644 index 3fa76c729..000000000 --- a/tests/test_apps/test_executor.py +++ /dev/null @@ -1,195 +0,0 @@ -"""unit tests for the shared FunctionRequest execution engine.""" - -import pytest - -from runpod.runtimes import executor - - -def test_execute_roundtrip_json(): - response = executor.execute_request( - { - "function_name": "mul", - "function_code": "def mul(a, b):\n return a * b", - "args": [6, 7], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"] is True - assert response["json_result"] == 42 - - -def test_json_result_rejects_non_json_return(): - with pytest.raises(TypeError, match="json-serializable"): - executor._serialize_result(object(), "json") - - -def test_serialize_chunk_marks_stream(): - chunk = executor.serialize_chunk("tok", {"serialization_format": "json"}) - assert chunk == {"success": True, "__stream__": True, "json_result": "tok"} - - -def test_resolve_request_returns_callable(): - prepared, error = executor.resolve_request( - { - "function_name": "double", - "function_code": "def double(x):\n return x * 2", - "kwargs": {}, - "args": [], - "serialization_format": "json", - } - ) - assert error is None - fn, args, kwargs = prepared - assert fn(4) == 8 - - -def test_resolve_request_missing_function(): - prepared, error = executor.resolve_request( - {"function_name": "nope", "function_code": "x = 1"} - ) - assert prepared is None - assert error["success"] is False - assert "not found" in error["error"] - - -def test_generator_results_aggregate(): - response = executor.execute_request( - { - "function_name": "gen", - "function_code": "def gen(n):\n yield from range(n)", - "args": [], - "kwargs": {"n": 3}, - "serialization_format": "json", - } - ) - assert response["success"] is True - assert response["json_result"] == [0, 1, 2] - - -class TestSystemDependencies: - def test_missing_apt_reports_error(self, monkeypatch): - import shutil - - monkeypatch.setattr(shutil, "which", lambda _: None) - response = executor.execute_request( - { - "function_name": "f", - "function_code": "def f():\n return 1", - "system_dependencies": ["ffmpeg"], - "serialization_format": "json", - } - ) - assert response["success"] is False - assert "apt-get is not available" in response["error"] - - def test_system_deps_installed_before_execution(self, monkeypatch): - import shutil - - monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(executor, "_apt_updated", executor.threading.Event()) - calls = [] - - class R: - returncode = 0 - stderr = "" - - monkeypatch.setattr( - executor.subprocess, - "run", - lambda cmd, **kw: calls.append(cmd) or R(), - ) - response = executor.execute_request( - { - "function_name": "f", - "function_code": "def f():\n return 40 + 2", - "system_dependencies": ["ffmpeg"], - "serialization_format": "json", - } - ) - assert response["success"] is True - assert response["json_result"] == 42 - assert calls[0][:2] == ["apt-get", "update"] - assert "ffmpeg" in calls[1] - - def test_apt_update_runs_once(self, monkeypatch): - import shutil - - monkeypatch.setattr(shutil, "which", lambda _: "/usr/bin/apt-get") - monkeypatch.setattr(executor, "_apt_updated", executor.threading.Event()) - calls = [] - - class R: - returncode = 0 - stderr = "" - - monkeypatch.setattr( - executor.subprocess, - "run", - lambda cmd, **kw: calls.append(cmd) or R(), - ) - executor._install_system(["ffmpeg"]) - executor._install_system(["sox"]) - updates = [c for c in calls if c[:2] == ["apt-get", "update"]] - assert len(updates) == 1 - - -class TestStdoutTee: - def test_prints_reach_real_stdout_and_response(self, capfd): - response = executor.execute_request( - { - "function_name": "speak", - "function_code": "def speak():\n print('live line')\n return 1\n", - "args": [], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"] - # captured for the job response - assert "live line" in response["stdout"] - # and written through to the container's stdout for log streams - assert "live line" in capfd.readouterr().out - - -class TestCloudpickleLoading: - def test_available_returns_module(self): - module = executor._load_cloudpickle() - assert module is not None - assert hasattr(module, "dumps") - - def test_missing_without_install_returns_none(self, monkeypatch): - import builtins - - real_import = builtins.__import__ - - def no_cloudpickle(name, *args, **kwargs): - if name == "cloudpickle": - raise ImportError("not installed") - return real_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", no_cloudpickle) - assert executor._load_cloudpickle(install=False) is None - - -class TestInstallHelpers: - def test_install_empty_is_noop(self): - assert executor._install([], "nothing") is None - - def test_install_failure_returns_message(self, monkeypatch): - from unittest.mock import MagicMock - - result = MagicMock(returncode=1, stderr="resolver exploded") - monkeypatch.setattr( - executor.subprocess, "run", MagicMock(return_value=result) - ) - message = executor._install(["ghost-package"], "deps") - assert "resolver exploded" in message - - def test_install_system_requires_apt(self, monkeypatch): - monkeypatch.setattr(executor.shutil, "which", lambda _: None) - message = executor._install_system(["ffmpeg"]) - assert "apt-get" in message - - def test_install_system_empty_is_noop(self): - assert executor._install_system([]) is None diff --git a/tests/test_apps/test_queue_worker.py b/tests/test_apps/test_queue_worker.py deleted file mode 100644 index 8ac14f8e9..000000000 --- a/tests/test_apps/test_queue_worker.py +++ /dev/null @@ -1,282 +0,0 @@ -"""unit tests for the generic queue worker runtime.""" - -import inspect -import json -from unittest.mock import MagicMock, patch - -import pytest - -from runpod.runtimes.queue import worker - - -class TestModeDetection: - def test_resource_name_flash_env(self, monkeypatch): - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - assert worker._resource_name() == "chat" - - def test_resource_name_runpod_env(self, monkeypatch): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.setenv("RUNPOD_RESOURCE_NAME", "embed") - assert worker._resource_name() == "embed" - - def test_resource_name_empty(self, monkeypatch): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) - assert worker._resource_name() == "" - - def test_is_deployed_requires_name_and_manifest(self, monkeypatch, tmp_path): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) - monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) - assert not worker._is_deployed() - - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - assert not worker._is_deployed() - - (tmp_path / worker.MANIFEST_NAME).write_text("{}") - assert worker._is_deployed() - - -class TestDeployedHandle: - def _write_manifest(self, tmp_path, resources): - (tmp_path / worker.MANIFEST_NAME).write_text( - json.dumps({"resources": resources}) - ) - - def test_load_deployed_handle(self, monkeypatch, tmp_path): - module_src = ( - "import runpod\n" - "app = runpod.App('t')\n" - "@app.queue(name='chat', gpu='4090')\n" - "def chat(prompt: str):\n" - " return {'echo': prompt}\n" - ) - (tmp_path / "user_mod_a.py").write_text(module_src) - self._write_manifest( - tmp_path, [{"name": "chat", "module": "user_mod_a"}] - ) - monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - - handle = worker._load_deployed_handle() - assert handle.spec.name == "chat" - - def test_load_deployed_handle_missing_resource(self, monkeypatch, tmp_path): - self._write_manifest(tmp_path, [{"name": "other", "module": "x"}]) - monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - - with pytest.raises(RuntimeError, match="not in manifest"): - worker._load_deployed_handle() - - def test_load_deployed_handle_missing_handle(self, monkeypatch, tmp_path): - (tmp_path / "user_mod_b.py").write_text("x = 1\n") - self._write_manifest( - tmp_path, [{"name": "chat", "module": "user_mod_b"}] - ) - monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "chat") - - with pytest.raises(RuntimeError, match="no @app.queue"): - worker._load_deployed_handle() - - -class TestDeployedHandler: - def _handle(self, fn): - handle = MagicMock() - handle._fn = fn - return handle - - async def test_sync_function(self): - handler = worker._make_deployed_handler( - self._handle(lambda a, b: {"sum": a + b}) - ) - result = await handler({"input": {"a": 1, "b": 2}}) - assert result == {"sum": 3} - - async def test_async_function(self): - async def fn(x): - return x * 2 - - handler = worker._make_deployed_handler(self._handle(fn)) - assert await handler({"input": {"x": 5}}) == 10 - - async def test_empty_marker_stripped(self): - handler = worker._make_deployed_handler( - self._handle(lambda: "ok") - ) - assert await handler({"input": {"__empty": True}}) == "ok" - - async def test_no_input(self): - handler = worker._make_deployed_handler( - self._handle(lambda: "ok") - ) - assert await handler({}) == "ok" - - async def test_sync_generator(self): - def fn(n): - for i in range(n): - yield i - - handler = worker._make_deployed_handler(self._handle(fn)) - assert inspect.isgeneratorfunction(handler) - assert list(handler({"input": {"n": 3}})) == [0, 1, 2] - - async def test_async_generator(self): - async def fn(n): - for i in range(n): - yield i - - handler = worker._make_deployed_handler(self._handle(fn)) - assert inspect.isasyncgenfunction(handler) - chunks = [c async for c in handler({"input": {"n": 2}})] - assert chunks == [0, 1] - - -class TestInitHook: - def test_no_init(self): - handle = MagicMock(spec=[]) - worker._run_init(handle) # no _init_fn attr: no-op - - def test_sync_init(self): - calls = [] - handle = MagicMock() - handle._init_fn = lambda: calls.append(1) - worker._run_init(handle) - assert calls == [1] - - def test_async_init(self): - calls = [] - - async def init(): - calls.append(1) - - handle = MagicMock() - handle._init_fn = init - worker._run_init(handle) - assert calls == [1] - - -class TestConcurrency: - def test_default(self, monkeypatch): - monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) - assert worker._max_concurrency() == 1 - - def test_env_value(self, monkeypatch): - monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "8") - assert worker._max_concurrency() == 8 - - def test_invalid_value(self, monkeypatch): - monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "banana") - assert worker._max_concurrency() == 1 - - def test_floor_of_one(self, monkeypatch): - monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "0") - assert worker._max_concurrency() == 1 - - def test_worker_config_single(self, monkeypatch): - monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) - config = worker._worker_config(lambda job: job) - assert "concurrency_modifier" not in config - - def test_worker_config_concurrent(self, monkeypatch): - monkeypatch.setenv("RUNPOD_MAX_CONCURRENCY", "4") - config = worker._worker_config(lambda job: job) - assert config["concurrency_modifier"](1) == 4 - - def test_worker_config_plain_handler(self, monkeypatch): - monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) - config = worker._worker_config(lambda job: job) - assert "return_aggregate_stream" not in config - - def test_worker_config_generator_handler(self, monkeypatch): - monkeypatch.delenv("RUNPOD_MAX_CONCURRENCY", raising=False) - - def gen_handler(job): - yield job - - config = worker._worker_config(gen_handler) - assert config["return_aggregate_stream"] is True - - -class TestMain: - def test_live_mode(self, monkeypatch): - monkeypatch.delenv("FLASH_RESOURCE_NAME", raising=False) - monkeypatch.delenv("RUNPOD_RESOURCE_NAME", raising=False) - with patch("runpod.serverless.start") as start: - worker.main() - config = start.call_args[0][0] - assert config["handler"] is worker._live_handler - - def test_deployed_mode(self, monkeypatch, tmp_path): - module_src = ( - "import runpod\n" - "app = runpod.App('t')\n" - "@app.queue(name='greet', gpu='4090')\n" - "def greet(name: str):\n" - " return f'hi {name}'\n" - ) - (tmp_path / "user_mod_c.py").write_text(module_src) - (tmp_path / worker.MANIFEST_NAME).write_text( - json.dumps({"resources": [{"name": "greet", "module": "user_mod_c"}]}) - ) - monkeypatch.setattr(worker, "APP_DIR", str(tmp_path)) - monkeypatch.setenv("FLASH_RESOURCE_NAME", "greet") - - with patch("runpod.serverless.start") as start: - worker.main() - handler = start.call_args[0][0]["handler"] - assert inspect.iscoroutinefunction(handler) - - async def test_live_handler_plain_function(self): - def fn(x): - return {"ok": x} - - with ( - patch( - "runpod.runtimes.executor.resolve_request", - return_value=((fn, [1], {}), None), - ), - patch( - "runpod.runtimes.executor.execute_request", - return_value={"success": True, "json_result": {"ok": 1}}, - ) as execute, - ): - chunks = [c async for c in worker._live_handler({"input": {"foo": 1}})] - assert chunks == [{"success": True, "json_result": {"ok": 1}}] - execute.assert_called_once_with({"foo": 1}) - - async def test_live_handler_resolve_error(self): - with patch( - "runpod.runtimes.executor.resolve_request", - return_value=(None, {"success": False, "error": "boom"}), - ): - chunks = [c async for c in worker._live_handler({"input": {}})] - assert chunks == [{"success": False, "error": "boom"}] - - async def test_live_handler_streams_generator(self): - def fn(n): - for i in range(n): - yield i - - with patch( - "runpod.runtimes.executor.resolve_request", - return_value=((fn, [], {"n": 2}), None), - ): - chunks = [c async for c in worker._live_handler({"input": {}})] - assert all(c["__stream__"] for c in chunks) - assert all(c["success"] for c in chunks) - assert len(chunks) == 2 - - async def test_live_handler_streams_async_generator(self): - async def fn(n): - for i in range(n): - yield i - - with patch( - "runpod.runtimes.executor.resolve_request", - return_value=((fn, [], {"n": 3}), None), - ): - chunks = [c async for c in worker._live_handler({"input": {}})] - assert len(chunks) == 3 - assert all(c["__stream__"] for c in chunks) diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py index 8339609ca..e39d16741 100644 --- a/tests/test_apps/test_shim.py +++ b/tests/test_apps/test_shim.py @@ -1,54 +1,72 @@ -"""tests for the shell launcher shim.""" +"""tests for the custom-image runtime launcher.""" -import base64 +import os import subprocess +import sys -from runpod.apps.shim import shell_launcher +import pytest + +from runpod.apps.shim import runtime_launcher def test_no_inner_single_quotes(): - # the host lexes dockerArgs; an inner single quote would break the - # outer quoting - cmd = shell_launcher("VAR", "/dest.py") - assert cmd.startswith("sh -c '") - assert cmd.endswith("'") - inner = cmd[len("sh -c '") : -1] - assert "'" not in inner + command = runtime_launcher("queue") + assert command.startswith("sh -c '") + assert command.endswith("'") + assert "'" not in command[len("sh -c '") : -1] def test_posix_sh_not_bash(): - cmd = shell_launcher("VAR", "/dest.py") - assert cmd.startswith("sh -c "), "must not require bash" + assert runtime_launcher("queue").startswith("sh -c ") -def test_decodes_and_execs_locally(tmp_path): - """run the inner script under sh with a payload that writes a marker.""" +def test_starts_an_installed_runtime(tmp_path): marker = tmp_path / "marker" - payload = f"open({str(marker)!r}, 'w').write('ran')" - b64 = base64.b64encode(payload.encode()).decode() - - dest = tmp_path / "injected.py" - cmd = shell_launcher("TESTVAR", str(dest)) - inner = cmd[len("sh -c '") : -1] + package = tmp_path / "runpod_sdk_runtime" / "task" + package.mkdir(parents=True) + (package.parent / "__init__.py").write_text("") + (package / "__init__.py").write_text("") + (package / "runner.py").write_text( + f"from pathlib import Path\nPath({str(marker)!r}).write_text('ran')\n" + ) + command = runtime_launcher("task") + inner = command[len("sh -c '") : -1] result = subprocess.run( ["sh", "-c", inner], - env={"TESTVAR": b64, "PATH": "/usr/bin:/bin:/usr/local/bin"}, + env={ + "PATH": f"{os.path.dirname(sys.executable)}:/usr/bin:/bin", + "PYTHONPATH": str(tmp_path), + }, capture_output=True, text=True, timeout=30, + check=False, ) + assert result.returncode == 0, result.stderr assert marker.read_text() == "ran" def test_probes_beyond_path(): - cmd = shell_launcher("VAR", "/dest.py") - assert "/opt/conda/bin/python" in cmd - assert "/opt/venv/bin/python" in cmd + command = runtime_launcher("api") + assert "/opt/conda/bin/python" in command + assert "/opt/venv/bin/python" in command def test_pythonless_image_fails_loudly(): - cmd = shell_launcher("VAR", "/dest.py") - assert "FATAL" in cmd - assert "must include python3" in cmd + command = runtime_launcher("queue") + assert "FATAL" in command + assert "must include python3" in command + + +def test_supports_runtime_and_sdk_package_overrides(): + command = runtime_launcher("queue") + assert "RUNPOD_RUNTIME_PACKAGE_SPEC" in command + assert "RUNPOD_PACKAGE_SPEC" in command + assert "runpod-sdk-runtime" in command + + +def test_rejects_unknown_runtime_kind(): + with pytest.raises(ValueError, match="unknown runtime kind"): + runtime_launcher("other") diff --git a/tests/test_apps/test_task_runner_http.py b/tests/test_apps/test_task_runner_http.py deleted file mode 100644 index 645b2d0f3..000000000 --- a/tests/test_apps/test_task_runner_http.py +++ /dev/null @@ -1,238 +0,0 @@ -"""integration tests for the task runner http server. - -boots the real ThreadingHTTPServer on a random port and exercises the -protocol over actual sockets. engine behavior (dependency install, -serialization, stdout capture) is covered in test_executor.py. -""" - -import base64 -import json -import threading -import urllib.error -import urllib.request -from http.server import ThreadingHTTPServer - -import cloudpickle -import pytest - -from runpod.runtimes.task import runner as task_runner -from runpod.runtimes.task.runner import Handler - -# http.server's per-request sockets are collected lazily; the unraisable -# checker flags them as ResourceWarnings non-deterministically -pytestmark = pytest.mark.filterwarnings( - "ignore::pytest.PytestUnraisableExceptionWarning" -) - -TOKEN = "test-token" - - -@pytest.fixture() -def server(monkeypatch): - monkeypatch.setattr(task_runner, "TOKEN", TOKEN) - monkeypatch.setattr( - task_runner, "_job_state", {"status": "NONE", "response": None} - ) - httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - httpd.daemon_threads = True - thread = threading.Thread(target=httpd.serve_forever, daemon=True) - thread.start() - yield f"http://127.0.0.1:{httpd.server_port}" - httpd.shutdown() - thread.join(timeout=5) - httpd.server_close() - - -def _request(url, method="GET", body=None, token=TOKEN): - req = urllib.request.Request(url, method=method) - if token: - req.add_header("Authorization", f"Bearer {token}") - data = None - if body is not None: - data = json.dumps(body).encode() - req.add_header("Content-Type", "application/json") - with urllib.request.urlopen(req, data=data, timeout=10) as resp: - return resp.status, json.loads(resp.read()) - - -def _b64(value): - return base64.b64encode(cloudpickle.dumps(value)).decode() - - -def _unb64(value): - return cloudpickle.loads(base64.b64decode(value)) - - -def test_ping_unauthenticated(server): - status, body = _request(f"{server}/ping", token=None) - assert status == 200 - assert body == {"ready": True} - - -def test_execute_requires_auth(server): - with pytest.raises(urllib.error.HTTPError) as exc_info: - _request(f"{server}/execute", method="POST", body={}, token="wrong") - assert exc_info.value.code == 401 - - -def test_execute_roundtrip(server): - status, body = _request( - f"{server}/execute", - method="POST", - body={ - "function_name": "mul", - "function_code": "def mul(a, b):\n return a * b", - "args": [_b64(6), _b64(7)], - "kwargs": {}, - }, - ) - assert status == 200 - assert body["success"] is True - assert _unb64(body["result"]) == 42 - - -def test_submit_and_result(server): - status, body = _request( - f"{server}/submit", - method="POST", - body={ - "function_name": "quick", - "function_code": "def quick():\n return 'done'", - "args": [], - "kwargs": {}, - "serialization_format": "json", - }, - ) - assert status == 200 - assert body == {"status": "RUNNING"} - - import time - - deadline = time.monotonic() + 10 - while time.monotonic() < deadline: - status, body = _request(f"{server}/result") - if body["status"] == "DONE": - break - time.sleep(0.1) - - assert body["status"] == "DONE" - assert body["response"]["success"] is True - assert body["response"]["json_result"] == "done" - - -def test_unknown_path_404(server): - with pytest.raises(urllib.error.HTTPError) as exc_info: - _request(f"{server}/nope") - assert exc_info.value.code == 404 - - -def test_single_file_source_is_self_contained(): - """the shipped runner must run without the runpod package installed.""" - import subprocess - import sys - - from runpod.apps.tasks import _runner_source - - source = _runner_source() - assert "from runpod.runtimes.executor import" in source - # compile in an empty namespace with runpod imports failing - probe = ( - "import sys\n" - "sys.modules['runpod'] = None\n" # forces the ImportError branch - "sys.argv = ['runner']\n" - + source.replace('if __name__ == "__main__":', "if False:") - ) - result = subprocess.run( - [sys.executable, "-c", probe], capture_output=True, text=True - ) - assert result.returncode == 0, result.stderr - - -class TestWatchdog: - def test_running_job_never_killed(self): - from runpod.runtimes.task.runner import _should_self_terminate - - assert not _should_self_terminate("RUNNING", 0, 10_000, 600) - - def test_abandoned_before_submit(self): - from runpod.runtimes.task.runner import _should_self_terminate - - assert _should_self_terminate("NONE", 0, 601, 600) - - def test_uncollected_result(self): - from runpod.runtimes.task.runner import _should_self_terminate - - assert _should_self_terminate("DONE", 0, 601, 600) - - def test_live_client_keeps_pod(self): - from runpod.runtimes.task.runner import _should_self_terminate - - # polls every ~2s: last contact is always recent - assert not _should_self_terminate("DONE", 599, 600, 600) - - def test_no_contact_recorded_yet(self): - from runpod.runtimes.task.runner import _should_self_terminate - - assert not _should_self_terminate("NONE", None, 10_000, 600) - - def test_authed_requests_touch_contact(self, server, monkeypatch): - monkeypatch.setattr(task_runner, "_last_contact", {"ts": None}) - _request(f"{server}/result") - assert task_runner._last_contact["ts"] is not None - - -class TestSelfTermination: - def test_terminate_self_calls_graphql_then_exits(self, monkeypatch): - import contextlib - import io as _io - - calls = [] - - @contextlib.contextmanager - def _response(): - yield _io.BytesIO(b"{}") - - def fake_urlopen(req, timeout=None): - calls.append((req.full_url, req.data)) - return _response() - - exits = [] - monkeypatch.setenv("RUNPOD_POD_ID", "pod-1") - monkeypatch.setenv("RUNPOD_API_KEY", "pod-scoped-key") - monkeypatch.setattr( - "urllib.request.urlopen", fake_urlopen - ) - monkeypatch.setattr( - task_runner.os, "_exit", lambda code: exits.append(code) - ) - task_runner._terminate_self() - - assert exits == [0] - url, body = calls[0] - assert url.endswith("/graphql") - assert b"podTerminate" in body - assert b"pod-1" in body - - def test_terminate_self_exits_without_credentials(self, monkeypatch): - exits = [] - monkeypatch.delenv("RUNPOD_POD_ID", raising=False) - monkeypatch.delenv("RUNPOD_API_KEY", raising=False) - monkeypatch.setattr( - task_runner.os, "_exit", lambda code: exits.append(code) - ) - task_runner._terminate_self() - assert exits == [0] - - def test_terminate_self_survives_api_failure(self, monkeypatch): - def fail(req, timeout=None): - raise OSError("network down") - - exits = [] - monkeypatch.setenv("RUNPOD_POD_ID", "pod-1") - monkeypatch.setenv("RUNPOD_API_KEY", "k") - monkeypatch.setattr("urllib.request.urlopen", fail) - monkeypatch.setattr( - task_runner.os, "_exit", lambda code: exits.append(code) - ) - task_runner._terminate_self() - assert exits == [0] diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 9dc58a7ec..c3a2b3800 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -11,9 +11,8 @@ from runpod.apps.app import _clear_registry from runpod.apps.errors import RemoteExecutionError from runpod.apps.spec import ResourceKind, ResourceSpec -from runpod.runtimes.executor import execute_request -from runpod.apps.tasks import _pod_input, unwrap_task_response from runpod.apps.targets import PodTarget +from runpod.apps.tasks import _pod_input, unwrap_task_response # the sync bridge finalizes coroutines on a background loop thread; # asyncmock coroutines observed there trip the unraisable checker @@ -38,84 +37,6 @@ def _unb64(value): return cloudpickle.loads(base64.b64decode(value)) -class TestExecuteRequest: - def test_simple_function(self): - response = execute_request( - { - "function_name": "add", - "function_code": "def add(a, b):\n return a + b", - "args": [_b64(2), _b64(3)], - "kwargs": {}, - } - ) - assert response["success"] is True - assert _unb64(response["result"]) == 5 - - def test_json_format(self): - response = execute_request( - { - "function_name": "add", - "function_code": "def add(a, b):\n return a + b", - "args": [2, 3], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"] is True - assert response["json_result"] == 5 - - def test_async_function(self): - response = execute_request( - { - "function_name": "hello", - "function_code": ( - "async def hello(name):\n return f'hi {name}'" - ), - "args": [], - "kwargs": {"name": _b64("world")}, - } - ) - assert response["success"] is True - assert _unb64(response["result"]) == "hi world" - - def test_stdout_captured(self): - response = execute_request( - { - "function_name": "loud", - "function_code": "def loud():\n print('noise')\n return 1", - "args": [], - "kwargs": {}, - "serialization_format": "json", - } - ) - assert response["success"] is True - assert "noise" in response["stdout"] - - def test_exception_reported(self): - response = execute_request( - { - "function_name": "boom", - "function_code": "def boom():\n raise ValueError('bad')", - "args": [], - "kwargs": {}, - } - ) - assert response["success"] is False - assert "ValueError" in response["error"] - - def test_missing_function(self): - response = execute_request( - { - "function_name": "nope", - "function_code": "x = 1", - "args": [], - "kwargs": {}, - } - ) - assert response["success"] is False - assert "not found" in response["error"] - - class TestPodInput: def _spec(self, **kw): defaults = dict(kind=ResourceKind.TASK, name="t") @@ -135,8 +56,7 @@ def test_cpu_pod_input(self): assert pod["terminateAfter"] env = {e["key"]: e["value"] for e in pod["env"]} assert env["RUNPOD_TASK_TOKEN"] == "tok" - # runtime images have the runner baked in; no env payload - assert "RUNPOD_TASK_RUNNER_B64" not in env + assert "RUNPOD_RUNTIME_PACKAGE_SPEC" not in env assert "dockerArgs" not in pod def test_gpu_pod_input(self): @@ -153,22 +73,20 @@ def test_gpu_pod_input(self): == f"runpod/task-gpu:py{local_python_version()}-latest" ) - def test_custom_image_bootstraps_runner(self): + def test_custom_image_starts_runtime_package(self, monkeypatch): + monkeypatch.setenv("RUNPOD_RUNTIME_PACKAGE_SPEC", "runpod-sdk-runtime==1.2.3") + monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==2.0.0") pod = _pod_input( self._spec(cpu=["cpu3c-1-2"], image="my/image:1", volume="vol-1"), "tok", "t", ) assert pod["imageName"] == "my/image:1" - # volumes resolve at TaskExecution.start (placement solve), - # not in the static pod input assert "networkVolumeId" not in pod - # custom images get the env-injection fallback env = {e["key"]: e["value"] for e in pod["env"]} - assert "RUNPOD_TASK_RUNNER_B64" in env - decoded = base64.b64decode(env["RUNPOD_TASK_RUNNER_B64"]).decode() - assert "def execute_request" in decoded - assert "task_runner.py" in pod["dockerArgs"] + assert env["RUNPOD_RUNTIME_PACKAGE_SPEC"] == "runpod-sdk-runtime==1.2.3" + assert env["RUNPOD_PACKAGE_SPEC"] == "runpod==2.0.0" + assert "runpod_sdk_runtime.task.runner" in pod["dockerArgs"] def test_datacenter_forwarded(self): pod = _pod_input( From 938eb6f1edab34af18c7a8b41c3a88ce81e3011b Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:15:48 -0700 Subject: [PATCH 096/120] fix: support sdk source archives in runtime launcher --- runpod/apps/shim.py | 1 + tests/test_apps/test_shim.py | 1 + 2 files changed, 2 insertions(+) diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py index 8c71a54da..dd61c9546 100644 --- a/runpod/apps/shim.py +++ b/runpod/apps/shim.py @@ -48,6 +48,7 @@ def runtime_launcher(kind: str) -> str: f'echo "[shim] FATAL: no python interpreter found in this image. "' f'"custom images must include python3." >&2; ' f"exit 1; fi; " + f'export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD="${{SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD:-0.0.0.dev0}}"; ' f'RUNTIME_SPEC="${{RUNPOD_RUNTIME_PACKAGE_SPEC:-runpod-sdk-runtime}}"; ' f'if [ -n "${{RUNPOD_RUNTIME_PACKAGE_SPEC:-}}" ] || ' f'! "$PY" -c "import runpod_sdk_runtime" >/dev/null 2>&1; then ' diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py index e39d16741..e961befd3 100644 --- a/tests/test_apps/test_shim.py +++ b/tests/test_apps/test_shim.py @@ -64,6 +64,7 @@ def test_supports_runtime_and_sdk_package_overrides(): command = runtime_launcher("queue") assert "RUNPOD_RUNTIME_PACKAGE_SPEC" in command assert "RUNPOD_PACKAGE_SPEC" in command + assert "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD" in command assert "runpod-sdk-runtime" in command From f8377e323f698842837641b339f85a41895d5e1f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:42:35 -0700 Subject: [PATCH 097/120] feat: group apps commands under flash --- README.md | 4 +- docs/cli/references/command_line_interface.md | 19 +++++++ docs/cli/start_here.md | 2 +- examples/apps/README.md | 4 +- examples/apps/gpu_inference.py | 2 +- examples/apps/hello_world.py | 2 +- examples/apps/streaming.py | 2 +- examples/apps/train_and_eval.py | 2 +- examples/apps/web_service.py | 2 +- runpod/apps/app.py | 2 +- runpod/apps/context.py | 2 +- runpod/apps/dev.py | 4 +- runpod/apps/discovery.py | 2 +- runpod/apps/entrypoint.py | 2 +- runpod/apps/errors.py | 2 +- runpod/apps/init.py | 10 ++-- runpod/apps/manage.py | 2 +- runpod/apps/schedule.py | 2 +- runpod/apps/spec.py | 2 +- runpod/rp_cli/main.py | 45 ++++++++++------ tests/e2e/examples/01_hello_world.py | 2 +- tests/e2e/examples/02_invocation_styles.py | 2 +- tests/e2e/examples/03_gpu.py | 2 +- tests/e2e/examples/04_dependencies.py | 2 +- tests/e2e/examples/05_api_service.py | 2 +- tests/e2e/examples/06_tasks.py | 2 +- tests/e2e/examples/07_pipelines.py | 2 +- tests/e2e/examples/08_volumes.py | 2 +- tests/e2e/examples/09_secrets.py | 2 +- tests/e2e/examples/10_cached_models.py | 2 +- tests/e2e/examples/11_train_eval.py | 2 +- tests/e2e/examples/12_custom_image.py | 2 +- tests/e2e/examples/13_streaming.py | 2 +- tests/e2e/examples/README.md | 4 +- tests/e2e/examples/run_all.sh | 16 +++--- tests/e2e/matrix/run.py | 2 +- tests/test_apps/test_manage.py | 2 +- tests/test_cli/test_deploy_command.py | 30 +++++------ tests/test_cli/test_init_command.py | 10 ++-- tests/test_cli/test_management_commands.py | 54 +++++++++++++------ 40 files changed, 157 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 47efd626a..132596e05 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ def main(): ``` ```bash -rp dev main.py # live dev session: edit, re-run, logs stream back -rp deploy # deploy production endpoints +rp flash dev main.py # live dev session: edit, re-run, logs stream back +rp flash deploy # deploy production endpoints ``` Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for runnable examples and [docs.runpod.io](https://docs.runpod.io) for the full guide. diff --git a/docs/cli/references/command_line_interface.md b/docs/cli/references/command_line_interface.md index e3b5232e7..f855bbc6e 100644 --- a/docs/cli/references/command_line_interface.md +++ b/docs/cli/references/command_line_interface.md @@ -6,6 +6,14 @@ Note: This CLI is not the same as runpodctl and provides a different set of feat # Auth rp login +# Flash apps +rp flash init my-app +rp flash dev main.py +rp flash deploy +rp flash app list +rp flash env list --app my-app +rp flash undeploy --app my-app + # SSH rp ssh add # add a key to your account rp ssh list # list account keys @@ -31,3 +39,14 @@ rp login --api-key YOUR_KEY # store a key directly ``` Credentials are stored in `~/.runpod/config.toml`. + +### Flash apps + +```bash +rp flash init my-app # scaffold a project +rp flash dev main.py # run a live development session +rp flash deploy # deploy the current project +rp flash app list # list deployed apps +rp flash env list --app my-app # list an app's environments +rp flash undeploy --app my-app # delete an environment's endpoints +``` diff --git a/docs/cli/start_here.md b/docs/cli/start_here.md index 99902938b..0269595db 100644 --- a/docs/cli/start_here.md +++ b/docs/cli/start_here.md @@ -18,4 +18,4 @@ Once a key is added, open a terminal on any pod with `rp ssh ` (or the e ## Apps -An app is a collection of Python functions that run on Runpod. Scaffold one with `rp init`, iterate on it live with `rp dev main.py`, and ship it with `rp deploy`. See the [README](../../README.md) for the full workflow. +An app is a collection of Python functions that run on Runpod. Scaffold one with `rp flash init`, iterate on it live with `rp flash dev main.py`, and ship it with `rp flash deploy`. See the [README](../../README.md) for the full workflow. diff --git a/examples/apps/README.md b/examples/apps/README.md index b8a381300..49e9802c4 100644 --- a/examples/apps/README.md +++ b/examples/apps/README.md @@ -1,10 +1,10 @@ # app examples each file is a self-contained app showing one way to use the sdk. -run any of them live with `rp dev`: +run any of them live with `rp flash dev`: ```bash -rp dev examples/apps/hello_world.py +rp flash dev examples/apps/hello_world.py ``` | example | shows | diff --git a/examples/apps/gpu_inference.py b/examples/apps/gpu_inference.py index 7eaa9f093..56eebdcaa 100644 --- a/examples/apps/gpu_inference.py +++ b/examples/apps/gpu_inference.py @@ -5,7 +5,7 @@ listed on the decorator are ready before your function runs, so the worker needs no custom image. - rp dev examples/apps/gpu_inference.py + rp flash dev examples/apps/gpu_inference.py """ import runpod diff --git a/examples/apps/hello_world.py b/examples/apps/hello_world.py index 530371389..87bfe9f7b 100644 --- a/examples/apps/hello_world.py +++ b/examples/apps/hello_world.py @@ -6,7 +6,7 @@ Try it: - rp dev examples/apps/hello_world.py + rp flash dev examples/apps/hello_world.py Edit the greeting while the session is running, then press enter to re-run — the worker picks up your new code automatically. diff --git a/examples/apps/streaming.py b/examples/apps/streaming.py index 1f682d959..b75ffc81a 100644 --- a/examples/apps/streaming.py +++ b/examples/apps/streaming.py @@ -7,7 +7,7 @@ Try it: - rp dev examples/apps/streaming.py + rp flash dev examples/apps/streaming.py """ import time diff --git a/examples/apps/train_and_eval.py b/examples/apps/train_and_eval.py index 8459973a4..b06e75a1f 100644 --- a/examples/apps/train_and_eval.py +++ b/examples/apps/train_and_eval.py @@ -7,7 +7,7 @@ volume is created on first use, and both tasks are automatically placed in its datacenter. - rp dev examples/apps/train_and_eval.py + rp flash dev examples/apps/train_and_eval.py """ import runpod diff --git a/examples/apps/web_service.py b/examples/apps/web_service.py index ec1dbb365..50ef74e42 100644 --- a/examples/apps/web_service.py +++ b/examples/apps/web_service.py @@ -5,7 +5,7 @@ it takes traffic. Workers keep state between requests — this counter lives in memory on the worker. - rp dev examples/apps/web_service.py + rp flash dev examples/apps/web_service.py Deployed services get a stable URL; in dev you call routes through the class itself, as shown in the entrypoint. diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 009e6a10e..45e28a70a 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -8,7 +8,7 @@ @app.queue(name="transcribe", gpu=GpuType.NVIDIA_GEFORCE_RTX_4090) async def transcribe(audio_url: str): ... -`rp deploy` imports project modules, collects App instances from the +`rp flash deploy` imports project modules, collects App instances from the module-level registry, and ships each app's resources. resolution of deployed resources is fully server-side (sentinel headers), so apps hold no persistent local state. diff --git a/runpod/apps/context.py b/runpod/apps/context.py index 198df12e7..3bcffc5a4 100644 --- a/runpod/apps/context.py +++ b/runpod/apps/context.py @@ -14,7 +14,7 @@ class Context(Enum): """dev machine, plain `python main.py`.""" DEV = "dev" - """inside an `rp dev` session (ephemeral live provisioning).""" + """inside an `rp flash dev` session (ephemeral live provisioning).""" WORKER = "worker" """inside a runpod serverless endpoint or pod.""" diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index da3d2d0ec..42c1d098c 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -1,4 +1,4 @@ -"""dev session: ephemeral live endpoints for `rp dev`. +"""dev session: ephemeral live endpoints for `rp flash dev`. lifecycle: get-or-create endpoints named dev-{app}-{resource} on the generic worker images (adopting leftovers from a killed session), run @@ -188,7 +188,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: class DevSession: - """owns the live endpoints for one `rp dev` invocation. + """owns the live endpoints for one `rp flash dev` invocation. lifecycle: adopt-or-create by name on start, reconcile on refresh (config updates + a generation bump that recreates workers so every diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 4db1c907c..4a1f0724d 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -1,4 +1,4 @@ -"""module discovery for `rp deploy` and `rp dev`. +"""module discovery for `rp flash deploy` and `rp flash dev`. imports target modules under __name__ != "__main__" (so main guards never run) and collects App instances from the registry. a per-module diff --git a/runpod/apps/entrypoint.py b/runpod/apps/entrypoint.py index e920308c3..029285d5d 100644 --- a/runpod/apps/entrypoint.py +++ b/runpod/apps/entrypoint.py @@ -4,7 +4,7 @@ async def main(): result = transcribe.remote("https://...") -the decorated function is registered so `rp dev ` can find and +the decorated function is registered so `rp flash dev ` can find and run it (sync or async). importing the module never executes it. """ diff --git a/runpod/apps/errors.py b/runpod/apps/errors.py index acd3dd0a1..3044182fd 100644 --- a/runpod/apps/errors.py +++ b/runpod/apps/errors.py @@ -13,7 +13,7 @@ def __init__(self, app_name: str, resource_name: str): self.resource_name = resource_name super().__init__( f"'{resource_name}' in app '{app_name}' is not deployed. " - f"run `rp deploy` first, or use .local() to run it here." + f"run `rp flash deploy` first, or use .local() to run it here." ) diff --git a/runpod/apps/init.py b/runpod/apps/init.py index b7a59edce..0bb0e9cd1 100644 --- a/runpod/apps/init.py +++ b/runpod/apps/init.py @@ -1,4 +1,4 @@ -"""project scaffolding for `rp init`. +"""project scaffolding for `rp flash init`. writes a minimal app project: a main module with one queue function and a local entrypoint, a requirements file, and a .runpodignore. existing @@ -12,11 +12,11 @@ Run it live: - rp dev main.py + rp flash dev main.py Deploy it: - rp deploy + rp flash deploy """ import runpod @@ -27,7 +27,7 @@ @app.queue(cpu="cpu3c-1-2") def hello(name: str): - # this print streams back to your terminal during rp dev + # this print streams back to your terminal during rp flash dev print(f"running in the cloud, greeting {{name}}") return f"hello {{name}}!" @@ -39,7 +39,7 @@ def main(): ''' REQUIREMENTS_TEMPLATE = """# packages your functions need on the workers -# (also installed locally for rp dev) +# (also installed locally for rp flash dev) """ RUNPODIGNORE_TEMPLATE = """# excluded from the deploy artifact diff --git a/runpod/apps/manage.py b/runpod/apps/manage.py index 9c260e1fb..7c7145e72 100644 --- a/runpod/apps/manage.py +++ b/runpod/apps/manage.py @@ -19,7 +19,7 @@ class AppNotFound(AppError): def __init__(self, app_name: str): super().__init__( - f"no app named '{app_name}' found. run `rp deploy` to create one." + f"no app named '{app_name}' found. run `rp flash deploy` to create one." ) diff --git a/runpod/apps/schedule.py b/runpod/apps/schedule.py index c55d4bfc5..b1efff491 100644 --- a/runpod/apps/schedule.py +++ b/runpod/apps/schedule.py @@ -1,6 +1,6 @@ """the @schedule decorator: records a cron expression on a handle. -schedules only take effect through `rp deploy` (the schedule must live +schedules only take effect through `rp flash deploy` (the schedule must live server-side). recurring schedule execution is gated on backend support; deploying a scheduled resource raises ScheduleNotSupported until it lands. """ diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index e87e9b036..df5958869 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -1,7 +1,7 @@ """resource specifications produced by app decorators. a ResourceSpec is the declarative description of one deployable resource. -it is what the manifest serializes, what `rp deploy` ships to the backend, +it is what the manifest serializes, what `rp flash deploy` ships to the backend, and what dev-session provisioning consumes. it holds no live state. """ diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 001c2bf1a..9026d5380 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -1,10 +1,9 @@ """the `rp` / `runpod` cli. -a single plain-click command tree: app commands (deploy, dev, logs, -login) and the pod/ssh/exec/config groups all hang off one root, so -help, errors, and exit codes behave identically at every level. rich -is used only for runtime output (progress, lifecycle lines), never for -help rendering. +flash app commands share one namespace while account, registry, secret, +logs, pod, and ssh commands remain on the root. help, errors, and exit +codes behave identically at every level. rich is used only for runtime +output, never for help rendering. """ import asyncio @@ -17,19 +16,30 @@ # commands whose output would be polluted by (or that manage) the # update notice -_UPDATE_CHECK_EXCLUDED = frozenset({"dev", "update"}) +_UPDATE_CHECK_EXCLUDED = frozenset({"flash", "update"}) +_FLASH_UPDATE_CHECK_EXCLUDED = frozenset({"dev"}) @click.group() @click.pass_context def cli(ctx): - """Runpod CLI: deploy and manage apps, endpoints, and pods.""" + """Runpod CLI: manage Flash apps, endpoints, and pods.""" if ctx.invoked_subcommand not in _UPDATE_CHECK_EXCLUDED: from runpod.rp_cli.update import start_background_check start_background_check() +@cli.group() +@click.pass_context +def flash(ctx): + """Build, develop, and manage Flash apps.""" + if ctx.invoked_subcommand not in _FLASH_UPDATE_CHECK_EXCLUDED: + from runpod.rp_cli.update import start_background_check + + start_background_check() + + def _fail(message: str) -> None: raise click.ClickException(message) @@ -48,7 +58,7 @@ def _app_source(found, project_root: Path) -> str: # ------------------------------------------------------------------ init -@cli.command() +@flash.command() @click.argument("project_name", required=False) @click.option("--force", "-f", is_flag=True, help="Overwrite existing files.") def init(project_name, force): @@ -80,7 +90,7 @@ def init(project_name, force): if project_dir != Path.cwd(): ui.console.print(f" [dim]cd {project_name}[/dim]") ui.console.print(" [dim]rp login[/dim]") - ui.console.print(" [dim]rp dev main.py[/dim]") + ui.console.print(" [dim]rp flash dev main.py[/dim]") ui.console.print() @@ -134,7 +144,7 @@ def update(version_opt): # ---------------------------------------------------------------- deploy -@cli.command() +@flash.command() @click.argument("target", required=False, type=click.Path(path_type=Path)) @click.option("--env", "-e", "env", default=None, help="Target environment name.") @click.option( @@ -283,7 +293,7 @@ def deploy(target, env, python_version, exclude, build_only): # ------------------------------------------------------------------- dev -@cli.command() +@flash.command() @click.argument("module", type=click.Path(exists=True, path_type=Path)) @click.option( "--once", @@ -567,7 +577,7 @@ def _fmt_ts(value) -> str: return text[:10] if len(text) >= 10 else text -@cli.group() +@flash.group() def app(): """Manage deployed apps.""" @@ -584,7 +594,9 @@ def app_list(): raise click.ClickException(str(exc)) from exc if not apps: - ui.console.print("\n no apps deployed. run [white]rp deploy[/white]\n") + ui.console.print( + "\n no apps deployed. run [white]rp flash deploy[/white]\n" + ) return width = max(len(a["name"]) for a in apps) @@ -820,7 +832,7 @@ async def _delete(): ui.success(f"deleted secret [white]{name}[/white]") -@cli.group() +@flash.group() def env(): """Manage app environments.""" @@ -856,7 +868,8 @@ def env_list(app_name): envs = entry.get("flashEnvironments") or [] if not envs: ui.console.print( - f"\n no environments in [white]{name}[/white]. run [white]rp deploy[/white]\n" + f"\n no environments in [white]{name}[/white]. " + f"run [white]rp flash deploy[/white]\n" ) return ui.console.print() @@ -958,7 +971,7 @@ def env_delete(env_name, app_name, yes): ) -@cli.command() +@flash.command() @click.option("--app", "-a", "app_name", default=None, help="App name.") @click.option("--env", "-e", "env_name", default="default", show_default=True) @click.option("--yes", "-y", is_flag=True, help="Skip the confirmation prompt.") diff --git a/tests/e2e/examples/01_hello_world.py b/tests/e2e/examples/01_hello_world.py index e4c16a79f..55794a3e7 100644 --- a/tests/e2e/examples/01_hello_world.py +++ b/tests/e2e/examples/01_hello_world.py @@ -1,6 +1,6 @@ """the smallest possible app: one queue function, one remote call. - rp dev tests/e2e/examples/01_hello_world.py --once + rp flash dev tests/e2e/examples/01_hello_world.py --once """ import runpod diff --git a/tests/e2e/examples/02_invocation_styles.py b/tests/e2e/examples/02_invocation_styles.py index 43a73ef20..5f936f9e6 100644 --- a/tests/e2e/examples/02_invocation_styles.py +++ b/tests/e2e/examples/02_invocation_styles.py @@ -1,6 +1,6 @@ """every way to call a function: remote, async, spawn, local. - rp dev tests/e2e/examples/02_invocation_styles.py --once + rp flash dev tests/e2e/examples/02_invocation_styles.py --once """ import asyncio diff --git a/tests/e2e/examples/03_gpu.py b/tests/e2e/examples/03_gpu.py index 5999f9c9f..32311926d 100644 --- a/tests/e2e/examples/03_gpu.py +++ b/tests/e2e/examples/03_gpu.py @@ -1,6 +1,6 @@ """gpu selection: shorthand names resolve to real devices. - rp dev tests/e2e/examples/03_gpu.py --once + rp flash dev tests/e2e/examples/03_gpu.py --once """ import runpod diff --git a/tests/e2e/examples/04_dependencies.py b/tests/e2e/examples/04_dependencies.py index 054ee3139..da9b73ae4 100644 --- a/tests/e2e/examples/04_dependencies.py +++ b/tests/e2e/examples/04_dependencies.py @@ -1,6 +1,6 @@ """pip and apt dependencies, available on the worker without a custom image. - rp dev tests/e2e/examples/04_dependencies.py --once + rp flash dev tests/e2e/examples/04_dependencies.py --once """ import runpod diff --git a/tests/e2e/examples/05_api_service.py b/tests/e2e/examples/05_api_service.py index be8d0d899..164858b68 100644 --- a/tests/e2e/examples/05_api_service.py +++ b/tests/e2e/examples/05_api_service.py @@ -1,6 +1,6 @@ """a load-balanced http service from a class with route markers. - rp dev tests/e2e/examples/05_api_service.py --once + rp flash dev tests/e2e/examples/05_api_service.py --once """ import runpod diff --git a/tests/e2e/examples/06_tasks.py b/tests/e2e/examples/06_tasks.py index 5f96962fe..daee97156 100644 --- a/tests/e2e/examples/06_tasks.py +++ b/tests/e2e/examples/06_tasks.py @@ -1,6 +1,6 @@ """tasks: one ephemeral pod per call, terminated when done. - rp dev tests/e2e/examples/06_tasks.py --once + rp flash dev tests/e2e/examples/06_tasks.py --once """ import runpod diff --git a/tests/e2e/examples/07_pipelines.py b/tests/e2e/examples/07_pipelines.py index a797699a8..f01d61f83 100644 --- a/tests/e2e/examples/07_pipelines.py +++ b/tests/e2e/examples/07_pipelines.py @@ -1,6 +1,6 @@ """functions calling functions: workers invoke sibling resources remotely. - rp dev tests/e2e/examples/07_pipelines.py --once + rp flash dev tests/e2e/examples/07_pipelines.py --once """ import runpod diff --git a/tests/e2e/examples/08_volumes.py b/tests/e2e/examples/08_volumes.py index 834bb976f..cdbe2ab86 100644 --- a/tests/e2e/examples/08_volumes.py +++ b/tests/e2e/examples/08_volumes.py @@ -3,7 +3,7 @@ the volume is created on first use; placement co-locates both tasks in the volume's datacenter automatically. - rp dev tests/e2e/examples/08_volumes.py --once + rp flash dev tests/e2e/examples/08_volumes.py --once """ import runpod diff --git a/tests/e2e/examples/09_secrets.py b/tests/e2e/examples/09_secrets.py index f0bc688dd..5b5a80700 100644 --- a/tests/e2e/examples/09_secrets.py +++ b/tests/e2e/examples/09_secrets.py @@ -3,7 +3,7 @@ requires the secret to exist first: rp secret add ex-demo-secret --value "s3cret" - rp dev tests/e2e/examples/09_secrets.py --once + rp flash dev tests/e2e/examples/09_secrets.py --once """ import runpod diff --git a/tests/e2e/examples/10_cached_models.py b/tests/e2e/examples/10_cached_models.py index a406e9140..b99188a33 100644 --- a/tests/e2e/examples/10_cached_models.py +++ b/tests/e2e/examples/10_cached_models.py @@ -1,6 +1,6 @@ """platform-cached model weights: staged on the host before the worker starts. - rp dev tests/e2e/examples/10_cached_models.py --once + rp flash dev tests/e2e/examples/10_cached_models.py --once """ import runpod diff --git a/tests/e2e/examples/11_train_eval.py b/tests/e2e/examples/11_train_eval.py index 0470bb4e4..16759c70d 100644 --- a/tests/e2e/examples/11_train_eval.py +++ b/tests/e2e/examples/11_train_eval.py @@ -1,6 +1,6 @@ """a real workflow: train on one gpu pod, save to a volume, eval on another. - rp dev tests/e2e/examples/11_train_eval.py --once + rp flash dev tests/e2e/examples/11_train_eval.py --once """ import runpod diff --git a/tests/e2e/examples/12_custom_image.py b/tests/e2e/examples/12_custom_image.py index 6bedae0ea..6aef20263 100644 --- a/tests/e2e/examples/12_custom_image.py +++ b/tests/e2e/examples/12_custom_image.py @@ -1,7 +1,7 @@ """a custom container image: the sdk bootstraps its runtime onto any image with python3. - rp dev tests/e2e/examples/12_custom_image.py --once + rp flash dev tests/e2e/examples/12_custom_image.py --once """ import runpod diff --git a/tests/e2e/examples/13_streaming.py b/tests/e2e/examples/13_streaming.py index cf317ce14..cf86ffc73 100644 --- a/tests/e2e/examples/13_streaming.py +++ b/tests/e2e/examples/13_streaming.py @@ -1,6 +1,6 @@ """streaming permutations: sync/async generators, every consumption style. - rp dev tests/e2e/examples/13_streaming.py --once + rp flash dev tests/e2e/examples/13_streaming.py --once """ import asyncio diff --git a/tests/e2e/examples/README.md b/tests/e2e/examples/README.md index 6f26d6263..00b60d907 100644 --- a/tests/e2e/examples/README.md +++ b/tests/e2e/examples/README.md @@ -8,8 +8,8 @@ three modes: | mode | command | what it exercises | | ------ | -------------------- | --------------------------------------------- | -| dev | `rp dev --once` | ephemeral endpoints, entrypoint, teardown | -| deploy | `rp deploy ` | persistent endpoints for the app | +| dev | `rp flash dev --once` | ephemeral endpoints, entrypoint, teardown | +| deploy | `rp flash deploy ` | persistent endpoints for the app | | invoke | `python3 ` | `main()` against the deployed endpoints | `run_all.sh` runs every example through all three (the deploy phase does diff --git a/tests/e2e/examples/run_all.sh b/tests/e2e/examples/run_all.sh index 619991555..daf9bd184 100755 --- a/tests/e2e/examples/run_all.sh +++ b/tests/e2e/examples/run_all.sh @@ -5,8 +5,8 @@ # @runpod.local_entrypoint and an `if __name__ == "__main__"` block, so # the same main() runs under all three: # -# dev rp dev --once ephemeral endpoints, run entrypoint, tear down -# deploy rp deploy persistent endpoints for the app +# dev rp flash dev --once ephemeral endpoints, run entrypoint, tear down +# deploy rp flash deploy persistent endpoints for the app # invoke python3 run main() against the deployed endpoints # # the deploy phase runs deploy -> invoke -> undeploy per file, so no @@ -21,8 +21,8 @@ # ./run_all.sh 01 05 08 # subset by prefix # # env toggles: -# SKIP_DEV=1 skip the rp dev phase -# SKIP_DEPLOY=1 skip the rp deploy/python3 phase +# SKIP_DEV=1 skip the rp flash dev phase +# SKIP_DEPLOY=1 skip the rp flash deploy/python3 phase # KEEP=1 leave deployed endpoints up (skip undeploy) set -uo pipefail @@ -85,7 +85,7 @@ echo # ---------------------------------------------------------------- dev if [ -z "${SKIP_DEV:-}" ]; then for f in "${files[@]}"; do - run_step dev "$(basename "$f")" rp dev "$f" --once + run_step dev "$(basename "$f")" rp flash dev "$f" --once done fi @@ -95,11 +95,11 @@ if [ -z "${SKIP_DEPLOY:-}" ]; then name="$(basename "$f")" app="$(app_name_of "$f")" - if run_step deploy "$name" rp deploy "$f"; then + if run_step deploy "$name" rp flash deploy "$f"; then run_step invoke "$name" python3 "$f" if [ -z "${KEEP:-}" ] && [ -n "$app" ]; then - echo ">>> cleanup: rp undeploy -a $app -y" - rp undeploy -a "$app" -y || echo "!!! cleanup failed for $app" >&2 + echo ">>> cleanup: rp flash undeploy -a $app -y" + rp flash undeploy -a "$app" -y || echo "!!! cleanup failed for $app" >&2 echo fi fi diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py index e5f93b7bb..0c9bf69f6 100644 --- a/tests/e2e/matrix/run.py +++ b/tests/e2e/matrix/run.py @@ -1,4 +1,4 @@ -"""drive the deployed e2e matrix. run AFTER `rp deploy tests/e2e/matrix`. +"""drive the deployed e2e matrix. run AFTER `rp flash deploy tests/e2e/matrix`. RUNPOD_RUNTIME_TAG=dev python tests/e2e/matrix/run.py [--only q-basic,...] diff --git a/tests/test_apps/test_manage.py b/tests/test_apps/test_manage.py index 637505a51..71133a640 100644 --- a/tests/test_apps/test_manage.py +++ b/tests/test_apps/test_manage.py @@ -38,7 +38,7 @@ def test_list_sorted_by_name(self): assert [a["name"] for a in apps] == ["alpha", "zeta"] def test_get_app_not_found(self): - with pytest.raises(AppNotFound, match="rp deploy"): + with pytest.raises(AppNotFound, match="rp flash deploy"): asyncio.run(get_app("ghost", api=_api())) def test_get_environment_not_found(self): diff --git a/tests/test_cli/test_deploy_command.py b/tests/test_cli/test_deploy_command.py index 8a0196566..6a6a17359 100644 --- a/tests/test_cli/test_deploy_command.py +++ b/tests/test_cli/test_deploy_command.py @@ -1,4 +1,4 @@ -"""tests for rp deploy and rp dev command wiring.""" +"""tests for rp flash deploy and rp flash dev command wiring.""" from unittest.mock import AsyncMock, MagicMock, patch @@ -66,7 +66,7 @@ def test_deploys_discovered_app(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) deploy_app = AsyncMock(return_value=_result()) with patch("runpod.apps.deploy.deploy_app", deploy_app): - result = _runner().invoke(cli, ["deploy"]) + result = _runner().invoke(cli, ["flash", "deploy"]) assert result.exit_code == 0, result.output assert "demo/default" in result.output assert "ep1" in result.output @@ -77,7 +77,7 @@ def test_env_flag(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) deploy_app = AsyncMock(return_value=_result()) with patch("runpod.apps.deploy.deploy_app", deploy_app): - result = _runner().invoke(cli, ["deploy", "--env", "prod"]) + result = _runner().invoke(cli, ["flash", "deploy", "--env", "prod"]) assert result.exit_code == 0 assert deploy_app.call_args[1]["env_name"] == "prod" assert "demo/prod" in result.output @@ -89,21 +89,21 @@ def test_multi_app_plan(self, tmp_path, monkeypatch): side_effect=[_result("demo"), _result("other", {"embed": "ep2"})] ) with patch("runpod.apps.deploy.deploy_app", deploy_app): - result = _runner().invoke(cli, ["deploy"]) + result = _runner().invoke(cli, ["flash", "deploy"]) assert result.exit_code == 0, result.output assert "2 apps" in result.output assert deploy_app.await_count == 2 def test_missing_target(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["deploy", "missing_dir"]) + result = _runner().invoke(cli, ["flash", "deploy", "missing_dir"]) assert result.exit_code == 1 assert "does not exist" in result.output def test_no_apps_found(self, tmp_path, monkeypatch): (tmp_path / "main.py").write_text("x = 1\n") monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["deploy"]) + result = _runner().invoke(cli, ["flash", "deploy"]) assert result.exit_code == 1 assert "no runpod.App found" in result.output @@ -112,7 +112,7 @@ def test_engine_error_is_clean(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) deploy_app = AsyncMock(side_effect=RuntimeError("upload failed")) with patch("runpod.apps.deploy.deploy_app", deploy_app): - result = _runner().invoke(cli, ["deploy"]) + result = _runner().invoke(cli, ["flash", "deploy"]) assert result.exit_code == 1 assert "upload failed" in result.output assert "Traceback" not in result.output @@ -124,7 +124,7 @@ def test_build_only(self, tmp_path, monkeypatch): artifact.write_bytes(b"x" * 100) build = MagicMock(return_value=artifact) with patch("runpod.apps.deploy.build_artifact", build): - result = _runner().invoke(cli, ["deploy", "--build-only"]) + result = _runner().invoke(cli, ["flash", "deploy", "--build-only"]) assert result.exit_code == 0, result.output assert "demo-artifact.tar.gz" in result.output assert build.call_args[1]["output"] == artifact @@ -135,7 +135,7 @@ def test_exclude_passthrough(self, tmp_path, monkeypatch): deploy_app = AsyncMock(return_value=_result()) with patch("runpod.apps.deploy.deploy_app", deploy_app): result = _runner().invoke( - cli, ["deploy", "--exclude", "torch,numpy"] + cli, ["flash", "deploy", "--exclude", "torch,numpy"] ) assert result.exit_code == 0 assert deploy_app.call_args[1]["exclude"] == ["torch", "numpy"] @@ -151,7 +151,7 @@ def test_python_version_fallback_warns(self, tmp_path, monkeypatch): side_effect=RuntimeError("3.9 unsupported"), ), ): - result = _runner().invoke(cli, ["deploy"]) + result = _runner().invoke(cli, ["flash", "deploy"]) assert result.exit_code == 0 assert "no runtime image" in result.output @@ -173,7 +173,7 @@ def make_session(apps, events=None): return session with patch("runpod.apps.dev.DevSession", side_effect=make_session): - result = _runner().invoke(cli, ["dev", str(module), "--once"]) + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) assert result.exit_code == 0, result.output session.start.assert_awaited_once() session.stop.assert_awaited_once() @@ -197,7 +197,7 @@ def make_session(apps, events=None): return session with patch("runpod.apps.dev.DevSession", side_effect=make_session): - result = _runner().invoke(cli, ["dev", str(module), "--once"]) + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) assert result.exit_code == 1 session.stop.assert_awaited_once() @@ -205,7 +205,7 @@ def test_module_without_entrypoint(self, tmp_path, monkeypatch): module = tmp_path / "main.py" module.write_text(APP_SOURCE) monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["dev", str(module), "--once"]) + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) assert result.exit_code == 1 assert "local_entrypoint" in result.output @@ -213,11 +213,11 @@ def test_module_without_app(self, tmp_path, monkeypatch): module = tmp_path / "main.py" module.write_text("x = 1\n") monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["dev", str(module), "--once"]) + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) assert result.exit_code == 1 assert "no runpod.App" in result.output def test_missing_module(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["dev", "missing.py", "--once"]) + result = _runner().invoke(cli, ["flash", "dev", "missing.py", "--once"]) assert result.exit_code == 2 diff --git a/tests/test_cli/test_init_command.py b/tests/test_cli/test_init_command.py index 48d969eaa..4bad4c4b3 100644 --- a/tests/test_cli/test_init_command.py +++ b/tests/test_cli/test_init_command.py @@ -1,4 +1,4 @@ -"""rp init: project scaffolding.""" +"""rp flash init: project scaffolding.""" from pathlib import Path @@ -52,14 +52,14 @@ class TestInitCommand: def test_init_new_project(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner = CliRunner() - result = runner.invoke(cli, ["init", "demo"]) + result = runner.invoke(cli, ["flash", "init", "demo"]) assert result.exit_code == 0, result.output assert (tmp_path / "demo" / "main.py").exists() def test_init_current_directory(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner = CliRunner() - result = runner.invoke(cli, ["init", "."]) + result = runner.invoke(cli, ["flash", "init", "."]) assert result.exit_code == 0, result.output assert (tmp_path / "main.py").exists() assert tmp_path.name in (tmp_path / "main.py").read_text() @@ -68,7 +68,7 @@ def test_init_conflicts_fail_without_force(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "main.py").write_text("keep me") runner = CliRunner() - result = runner.invoke(cli, ["init", "."]) + result = runner.invoke(cli, ["flash", "init", "."]) assert result.exit_code != 0 assert "main.py" in result.output assert (tmp_path / "main.py").read_text() == "keep me" @@ -77,6 +77,6 @@ def test_init_force_overwrites(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "main.py").write_text("old") runner = CliRunner() - result = runner.invoke(cli, ["init", ".", "--force"]) + result = runner.invoke(cli, ["flash", "init", ".", "--force"]) assert result.exit_code == 0, result.output assert "App(" in (tmp_path / "main.py").read_text() diff --git a/tests/test_cli/test_management_commands.py b/tests/test_cli/test_management_commands.py index 131aceff9..b81b9a744 100644 --- a/tests/test_cli/test_management_commands.py +++ b/tests/test_cli/test_management_commands.py @@ -12,10 +12,34 @@ def _runner(): return CliRunner() +class TestCommandLayout: + def test_flash_owns_app_commands(self): + flash = cli.commands["flash"] + assert set(flash.commands) == { + "app", + "deploy", + "dev", + "env", + "init", + "undeploy", + } + + def test_app_commands_are_not_root_commands(self): + assert not { + "app", + "deploy", + "dev", + "env", + "init", + "undeploy", + } & set(cli.commands) + assert {"registry", "secret"} <= set(cli.commands) + + class TestAppCommands: def test_list_empty(self): with patch("runpod.apps.manage.list_apps", AsyncMock(return_value=[])): - result = _runner().invoke(cli, ["app", "list"]) + result = _runner().invoke(cli, ["flash", "app", "list"]) assert result.exit_code == 0 assert "no apps deployed" in result.output @@ -30,7 +54,7 @@ def test_list(self): with patch( "runpod.apps.manage.list_apps", AsyncMock(return_value=apps) ): - result = _runner().invoke(cli, ["app", "list"]) + result = _runner().invoke(cli, ["flash", "app", "list"]) assert result.exit_code == 0 assert "demo" in result.output assert "default, prod" in result.output @@ -45,7 +69,7 @@ def test_get(self): with patch( "runpod.apps.manage.get_app", AsyncMock(return_value=entry) ): - result = _runner().invoke(cli, ["app", "get", "demo"]) + result = _runner().invoke(cli, ["flash", "app", "get", "demo"]) assert result.exit_code == 0 assert "build1234567" in result.output @@ -54,7 +78,7 @@ def test_get_error(self): "runpod.apps.manage.get_app", AsyncMock(side_effect=RuntimeError("no app named 'demo'")), ): - result = _runner().invoke(cli, ["app", "get", "demo"]) + result = _runner().invoke(cli, ["flash", "app", "get", "demo"]) assert result.exit_code == 1 assert "no app named" in result.output @@ -63,11 +87,11 @@ def test_delete_confirmed(self): with patch( "runpod.apps.manage.delete_app", AsyncMock(return_value=outcome) ): - result = _runner().invoke(cli, ["app", "delete", "demo", "--yes"]) + result = _runner().invoke(cli, ["flash", "app", "delete", "demo", "--yes"]) assert result.exit_code == 0 def test_delete_aborts_without_confirmation(self): - result = _runner().invoke(cli, ["app", "delete", "demo"], input="n\n") + result = _runner().invoke(cli, ["flash", "app", "delete", "demo"], input="n\n") assert result.exit_code == 1 def test_delete_failures_keep_app(self): @@ -75,7 +99,7 @@ def test_delete_failures_keep_app(self): with patch( "runpod.apps.manage.delete_app", AsyncMock(return_value=outcome) ): - result = _runner().invoke(cli, ["app", "delete", "demo", "--yes"]) + result = _runner().invoke(cli, ["flash", "app", "delete", "demo", "--yes"]) assert result.exit_code == 1 assert "undeploy incomplete" in result.output @@ -95,7 +119,7 @@ def test_list(self): "runpod.apps.manage.get_app", AsyncMock(return_value=entry) ): result = _runner().invoke( - cli, ["env", "list", "--app", "demo"] + cli, ["flash", "env", "list", "--app", "demo"] ) assert result.exit_code == 0 assert "default" in result.output @@ -105,7 +129,7 @@ def test_list_empty(self): "runpod.apps.manage.get_app", AsyncMock(return_value={"flashEnvironments": []}), ): - result = _runner().invoke(cli, ["env", "list", "--app", "demo"]) + result = _runner().invoke(cli, ["flash", "env", "list", "--app", "demo"]) assert result.exit_code == 0 assert "no environments" in result.output @@ -120,7 +144,7 @@ def test_get(self): AsyncMock(return_value=entry), ): result = _runner().invoke( - cli, ["env", "get", "prod", "--app", "demo"] + cli, ["flash", "env", "get", "prod", "--app", "demo"] ) assert result.exit_code == 0 assert "demo/prod" in result.output @@ -137,7 +161,7 @@ def test_add(self): ), ): result = _runner().invoke( - cli, ["env", "add", "staging", "--app", "demo"] + cli, ["flash", "env", "add", "staging", "--app", "demo"] ) assert result.exit_code == 0 client.create_environment.assert_awaited_once_with("app1", "staging") @@ -149,13 +173,13 @@ def test_delete(self): AsyncMock(return_value=outcome), ): result = _runner().invoke( - cli, ["env", "delete", "prod", "--app", "demo", "--yes"] + cli, ["flash", "env", "delete", "prod", "--app", "demo", "--yes"] ) assert result.exit_code == 0 def test_resolve_app_name_fails_without_unique_app(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) - result = _runner().invoke(cli, ["env", "list"]) + result = _runner().invoke(cli, ["flash", "env", "list"]) assert result.exit_code == 1 assert "--app" in result.output @@ -168,7 +192,7 @@ def test_undeploy(self): AsyncMock(return_value=outcome), ): result = _runner().invoke( - cli, ["undeploy", "--app", "demo", "--yes"] + cli, ["flash", "undeploy", "--app", "demo", "--yes"] ) assert result.exit_code == 0 assert "3 endpoints removed" in result.output @@ -180,7 +204,7 @@ def test_undeploy_failures(self): AsyncMock(return_value=outcome), ): result = _runner().invoke( - cli, ["undeploy", "--app", "demo", "--yes"] + cli, ["flash", "undeploy", "--app", "demo", "--yes"] ) assert result.exit_code == 1 From eef269e76e3e07267cd84c123b3adbf261450df9 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:20:19 -0700 Subject: [PATCH 098/120] fix: stabilize apps production execution --- runpod/apps/deploy.py | 4 +++ runpod/apps/dev.py | 8 +++++- runpod/apps/targets.py | 51 ++++++++++++++++++++++++++++++---- runpod/apps/tasks.py | 44 +++++++++++++++++++++++++++-- tests/e2e/matrix/main.py | 18 ++++++++++-- tests/e2e/matrix/run.py | 11 +++++++- tests/test_apps/test_deploy.py | 21 ++++++++++++++ tests/test_apps/test_dev.py | 4 +++ tests/test_apps/test_retry.py | 30 ++++++++++++++++++++ tests/test_apps/test_tasks.py | 41 +++++++++++++++++++++++++++ 10 files changed, 219 insertions(+), 13 deletions(-) diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 30d9723ff..9140fb79f 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -245,6 +245,9 @@ def _deployed_endpoint_input( if spec.max_concurrency > 1: template_env["RUNPOD_MAX_CONCURRENCY"] = str(spec.max_concurrency) + if spec.kind is ResourceKind.API: + template_env["PORT"] = "80" + template_env["PORT_HEALTH"] = "80" payload: Dict[str, Any] = { "name": spec.name, @@ -276,6 +279,7 @@ def _deployed_endpoint_input( if spec.kind is ResourceKind.API: payload["type"] = "LB" + payload["template"]["ports"] = "80/http" if spec.datacenter: payload["locations"] = ",".join(spec.datacenter) if spec.is_cpu: diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 42c1d098c..f3c36026a 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -118,6 +118,11 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: the endpoint and cascades on deleteEndpoint. """ + resource_env = _render_env(spec.env) + if spec.kind is ResourceKind.API: + resource_env["PORT"] = "80" + resource_env["PORT_HEALTH"] = "80" + payload: Dict = { "name": dev_endpoint_name(app.name, spec.name), "workersMin": spec.workers[0], @@ -152,7 +157,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: if spec.max_concurrency > 1 else [] ), - *({"key": k, "value": v} for k, v in _render_env(spec.env).items()), + *({"key": k, "value": v} for k, v in resource_env.items()), ], }, } @@ -171,6 +176,7 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: payload["template"]["dockerArgs"] = runtime_launcher(spec.kind.value) if spec.kind is ResourceKind.API: payload["type"] = "LB" + payload["template"]["ports"] = "80/http" if spec.datacenter: payload["locations"] = ",".join(spec.datacenter) if spec.is_cpu: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 4f18391e2..fbe626597 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -196,6 +196,7 @@ async def request( # than 429) are never retried; the request itself is wrong. RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504, 520, 522, 524}) RETRY_ATTEMPTS = 4 +JOB_PROPAGATION_ATTEMPTS = 7 RETRY_BASE_DELAY = 0.5 @@ -208,13 +209,15 @@ async def _request_json( payload: Any = None, app_name: str = "", resource_name: str = "", + retry_not_found: bool = False, ) -> Dict[str, Any]: """http json call with exponential backoff on transient failures.""" import asyncio client_timeout = aiohttp.ClientTimeout(total=timeout) last_exc: Optional[Exception] = None - for attempt in range(RETRY_ATTEMPTS): + attempts = JOB_PROPAGATION_ATTEMPTS if retry_not_found else RETRY_ATTEMPTS + for attempt in range(attempts): if attempt: await asyncio.sleep(RETRY_BASE_DELAY * (2 ** (attempt - 1))) try: @@ -223,7 +226,10 @@ async def _request_json( method, url, json=payload, headers=headers ) as resp: if resp.status == 404: - raise EndpointNotFound(app_name, resource_name) + last_exc = EndpointNotFound(app_name, resource_name) + if retry_not_found and attempt < attempts - 1: + continue + raise last_exc if resp.status in RETRYABLE_STATUSES: last_exc = aiohttp.ClientResponseError( resp.request_info, @@ -266,9 +272,23 @@ async def _post_json( async def _get_json( - url: str, headers: Dict[str, str], timeout: float + url: str, + headers: Dict[str, str], + timeout: float, + *, + app_name: str = "", + resource_name: str = "", + retry_not_found: bool = False, ) -> Dict[str, Any]: - return await _request_json("GET", url, headers, timeout) + return await _request_json( + "GET", + url, + headers, + timeout, + app_name=app_name, + resource_name=resource_name, + retry_not_found=retry_not_found, + ) async def _wait_terminal( @@ -277,6 +297,9 @@ async def _wait_terminal( headers: Dict[str, str], timeout: float, on_status: Optional[Callable[[Dict[str, Any]], None]] = None, + *, + app_name: str = "", + resource_name: str = "", ) -> Dict[str, Any]: """poll /status until the job reaches a terminal state. @@ -305,7 +328,14 @@ async def _wait_terminal( ) await asyncio.sleep(interval) interval = min(interval * 1.5, max_interval) - data = await _get_json(f"{base_url}/status/{job_id}", headers, 30.0) + data = await _get_json( + f"{base_url}/status/{job_id}", + headers, + 30.0, + app_name=app_name, + resource_name=resource_name, + retry_not_found=True, + ) if on_status is not None: on_status(data) return data @@ -352,6 +382,9 @@ async def _call( payload=payload, app_name=self._app_name, resource_name=self._resource_name, + retry_not_found=path.startswith( + ("status/", "stream/", "cancel/", "retry/") + ), ) async def run(self, payload: Dict[str, Any]) -> Dict[str, Any]: @@ -403,7 +436,13 @@ async def wait( on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Dict[str, Any]: return await _wait_terminal( - self.base_url, data, self._headers(), timeout, on_status + self.base_url, + data, + self._headers(), + timeout, + on_status, + app_name=self._app_name, + resource_name=self._resource_name, ) async def http( diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 3d221c7fe..934efa9b6 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -167,12 +167,50 @@ async def start(self) -> None: ) if self.spec.volume: pod = await self._attach_volume(pod) - result = await self.api.deploy_task_pod( - pod, is_cpu=self.spec.is_cpu - ) + result = await self._deploy_pod(pod) self.pod_id = result["id"] log.info("task pod %s deployed for %s", self.pod_id, self.spec.name) + async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: + from ..error import QueryError + + try: + return await self.api.deploy_task_pod( + pod, is_cpu=self.spec.is_cpu + ) + except QueryError as exc: + if not self._is_capacity_error(exc): + raise + capacity_error = exc + + datacenters = pod.get("dataCenterIds") or [] + if self.spec.volume or len(datacenters) < 2: + raise capacity_error + + for datacenter in datacenters: + candidate = dict(pod, dataCenterIds=[datacenter]) + try: + return await self.api.deploy_task_pod( + candidate, is_cpu=self.spec.is_cpu + ) + except QueryError as exc: + if not self._is_capacity_error(exc): + raise + raise capacity_error + + @staticmethod + def _is_capacity_error(exc: Exception) -> bool: + message = str(exc).lower() + return any( + phrase in message + for phrase in ( + "resources to deploy your pod", + "insufficient capacity", + "no available machine", + "out of stock", + ) + ) + async def _attach_volume(self, pod: Dict[str, Any]) -> Dict[str, Any]: """resolve the task's volume and pin the pod to its datacenter.""" from .volume import VolumeError, VolumeResolver, volume_list diff --git a/tests/e2e/matrix/main.py b/tests/e2e/matrix/main.py index 00ccaf622..801fdddf8 100644 --- a/tests/e2e/matrix/main.py +++ b/tests/e2e/matrix/main.py @@ -1,7 +1,7 @@ """e2e matrix app: every resource kind, cross-calls, deps, custom images. -deployed by tests/e2e/matrix/run.py; each resource exercises a distinct -permutation of the surface. +deploy with `rp flash deploy tests/e2e/matrix --python-version 3.12`. +each resource exercises a distinct permutation of the surface. """ import runpod @@ -130,3 +130,17 @@ def gen(): yield word + "\n" return StreamingResponse(gen(), media_type="text/plain") + + +@app.api( + name="a-custom", + cpu="cpu3c-1-2", + workers=(1, 1), + image="python:3.12-slim", +) +class CustomSvc: + @runpod.get("/runtime") + def runtime(self): + import platform + + return {"python": platform.python_version()} diff --git a/tests/e2e/matrix/run.py b/tests/e2e/matrix/run.py index 0c9bf69f6..02b731f62 100644 --- a/tests/e2e/matrix/run.py +++ b/tests/e2e/matrix/run.py @@ -1,5 +1,6 @@ -"""drive the deployed e2e matrix. run AFTER `rp flash deploy tests/e2e/matrix`. +"""drive the deployed e2e matrix after deploying it. + rp flash deploy tests/e2e/matrix --python-version 3.12 RUNPOD_RUNTIME_TAG=dev python tests/e2e/matrix/run.py [--only q-basic,...] each check is independent; failures are collected, not fatal, so one @@ -13,6 +14,7 @@ import traceback from main import ( # noqa: E402 + CustomSvc, Svc, q_basic, q_caller, @@ -86,6 +88,12 @@ async def check_api_streaming_response(): return {"body": r} +async def check_custom_api(): + r = await CustomSvc.get.aio("/runtime") + assert r["python"].startswith("3.12"), r + return r + + async def check_spawn(): job = await q_basic.spawn.aio(4) r = await job.result.aio() @@ -158,6 +166,7 @@ async def check_task_stream_rejected(): "t-mul": check_t_mul, "t-gpu": check_t_gpu, "api": check_api, + "api-custom": check_custom_api, "spawn": check_spawn, "job-ops": check_job_ops, "stream": check_stream, diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index 365d412be..4de46f4d6 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -13,6 +13,7 @@ from runpod.apps.app import _clear_registry from runpod.apps.deploy import ( DeployResult, + _deployed_endpoint_input, build_manifest, deploy_app, package_project, @@ -116,6 +117,26 @@ def t(): build_manifest(app, tmp_path) +class TestEndpointInput: + def test_custom_api_declares_runtime_port(self): + app = App("api-app") + + @app.api(name="api", cpu="cpu3c-1-2", image="python:3.12-slim") + class Api: + @runpod.get("/value") + def value(self): + return {"value": 1} + + payload = _deployed_endpoint_input( + app, Api.spec, "env-1", "build-1", "3.12" + ) + assert payload["type"] == "LB" + assert payload["template"]["ports"] == "80/http" + env = {entry["key"]: entry["value"] for entry in payload["template"]["env"]} + assert env["PORT"] == "80" + assert env["PORT_HEALTH"] == "80" + + class TestPackaging: def test_tarball_contains_source_and_manifest(self, tmp_path): _write_project(tmp_path) diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 6f5a723c3..4fb593e63 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -110,6 +110,10 @@ def x(self, body: dict): payload = _endpoint_input(app, Api.spec) assert payload["type"] == "LB" assert payload["scalerType"] == "REQUEST_COUNT" + assert payload["template"]["ports"] == "80/http" + env = {entry["key"]: entry["value"] for entry in payload["template"]["env"]} + assert env["PORT"] == "80" + assert env["PORT_HEALTH"] == "80" def test_custom_image_starts_external_runtime(self, monkeypatch): monkeypatch.setenv( diff --git a/tests/test_apps/test_retry.py b/tests/test_apps/test_retry.py index a973cb3cb..f2a3ea549 100644 --- a/tests/test_apps/test_retry.py +++ b/tests/test_apps/test_retry.py @@ -10,9 +10,11 @@ from runpod.apps.errors import EndpointNotFound from runpod.apps.targets import ( + JOB_PROPAGATION_ATTEMPTS, RETRY_ATTEMPTS, RETRYABLE_STATUSES, _post_json, + _request_json, ) @@ -82,6 +84,34 @@ async def test_404_not_retried_maps_to_endpoint_not_found(flaky_server): assert flaky_server.state["hits"] == 1 +async def test_job_404_retried_during_propagation(flaky_server): + flaky_server.state["script"] = [404, 404] + with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): + data = await _request_json( + "GET", + flaky_server.url, + {}, + timeout=10, + retry_not_found=True, + ) + assert data["ok"] is True + assert flaky_server.state["hits"] == 3 + + +async def test_persistent_job_404_exhausts_retries(flaky_server): + flaky_server.state["script"] = [404] * JOB_PROPAGATION_ATTEMPTS + with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): + with pytest.raises(EndpointNotFound): + await _request_json( + "GET", + flaky_server.url, + {}, + timeout=10, + retry_not_found=True, + ) + assert flaky_server.state["hits"] == JOB_PROPAGATION_ATTEMPTS + + async def test_client_4xx_not_retried(flaky_server): flaky_server.state["script"] = [400] with pytest.raises(aiohttp.ClientResponseError) as exc_info: diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index c3a2b3800..55ef3eac8 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -322,6 +322,47 @@ async def test_start_deploys_pod(self): assert execution.pod_id == "pod-9" assert api.deploy_task_pod.call_args[1]["is_cpu"] is True + async def test_start_retries_datacenters_after_capacity_error(self): + from runpod.apps.tasks import TaskExecution + from runpod.error import QueryError + + api = MagicMock() + capacity = "This machine does not have the resources to deploy your pod" + api.deploy_task_pod = AsyncMock( + side_effect=[ + QueryError(capacity), + QueryError(capacity), + {"id": "pod-9"}, + ] + ) + execution = TaskExecution( + self._spec( + gpu=["NVIDIA GeForce RTX 4090"], + cpu=None, + datacenter=["US-IL-1", "EU-RO-1"], + ), + api=api, + ) + await execution.start() + + calls = api.deploy_task_pod.await_args_list + assert calls[0].args[0]["dataCenterIds"] == ["US-IL-1", "EU-RO-1"] + assert calls[1].args[0]["dataCenterIds"] == ["US-IL-1"] + assert calls[2].args[0]["dataCenterIds"] == ["EU-RO-1"] + assert execution.pod_id == "pod-9" + + async def test_start_does_not_retry_non_capacity_error(self): + from runpod.apps.tasks import TaskExecution + from runpod.error import QueryError + + api = MagicMock() + api.deploy_task_pod = AsyncMock(side_effect=QueryError("invalid image")) + execution = TaskExecution(self._spec(), api=api) + + with pytest.raises(QueryError, match="invalid image"): + await execution.start() + api.deploy_task_pod.assert_awaited_once() + async def test_start_resolves_registry_auth(self): from runpod.apps.tasks import TaskExecution From f3da6e84fef41d4d25e9d6c507ecf1d075c7f055 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:57:16 -0700 Subject: [PATCH 099/120] feat: add sync and async sandbox SDK Implements CON-1523, CON-1524, CON-1525, and CON-1526. Based on REST migration PR #584. Keep this draft local until that PR is merged. Includes managed ownership, bounded startup handling, closeable SSE streams, and cancellation-safe synchronous and asynchronous resource cleanup. --- runpod/__init__.py | 16 +- runpod/api/rest.py | 38 ++- runpod/api/sandboxes.py | 386 +++++++++++++++++++++++++ runpod/sandbox/__init__.py | 31 ++ runpod/sandbox/asyncio.py | 565 +++++++++++++++++++++++++++++++++++++ runpod/sandbox/models.py | 136 +++++++++ runpod/sandbox/sync.py | 438 ++++++++++++++++++++++++++++ tests/test_sandbox.py | 458 ++++++++++++++++++++++++++++++ 8 files changed, 2052 insertions(+), 16 deletions(-) create mode 100644 runpod/api/sandboxes.py create mode 100644 runpod/sandbox/__init__.py create mode 100644 runpod/sandbox/asyncio.py create mode 100644 runpod/sandbox/models.py create mode 100644 runpod/sandbox/sync.py create mode 100644 tests/test_sandbox.py diff --git a/runpod/__init__.py b/runpod/__init__.py index 6d24180ae..812fc866a 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -1,4 +1,4 @@ -""" Allows runpod to be imported as a module. """ +"""Allows runpod to be imported as a module.""" import logging import os @@ -29,19 +29,20 @@ set_credentials, ) from .endpoint import AsyncioEndpoint, AsyncioJob, Endpoint +from .sandbox import AsyncioSandbox, Sandbox from .serverless.modules.rp_logger import RunPodLogger from .version import __version__ __all__ = [ # API functions "create_container_registry_auth", - "create_endpoint", + "create_endpoint", "create_pod", "create_template", "delete_container_registry_auth", "get_endpoints", "get_gpu", - "get_gpus", + "get_gpus", "get_pod", "get_pods", "get_user", @@ -53,12 +54,15 @@ "update_user_settings", # Config functions "check_credentials", - "get_credentials", + "get_credentials", "set_credentials", # Endpoint classes "AsyncioEndpoint", "AsyncioJob", "Endpoint", + # Sandbox classes + "AsyncioSandbox", + "Sandbox", # Serverless module "serverless", # Logger class @@ -68,8 +72,8 @@ # Module variables "SSH_KEY_PATH", "profile", - "api_key", - "endpoint_url_base" + "api_key", + "endpoint_url_base", ] # ------------------------------- Config Paths ------------------------------- # diff --git a/runpod/api/rest.py b/runpod/api/rest.py index 987eb1439..fced24296 100644 --- a/runpod/api/rest.py +++ b/runpod/api/rest.py @@ -15,7 +15,9 @@ def _resolve_api_key(api_key: Optional[str]) -> str: - from runpod import api_key as global_api_key # pylint: disable=import-outside-toplevel,cyclic-import + from runpod import ( + api_key as global_api_key, + ) # pylint: disable=import-outside-toplevel,cyclic-import effective_api_key = api_key or global_api_key if not effective_api_key: @@ -23,8 +25,10 @@ def _resolve_api_key(api_key: Optional[str]) -> str: return effective_api_key -def _build_url(path: str) -> str: - api_url_base = os.environ.get("RUNPOD_API_BASE_URL", "https://api.runpod.io") +def _build_url(path: str, base_url: Optional[str] = None) -> str: + api_url_base = base_url or os.environ.get( + "RUNPOD_API_BASE_URL", "https://api.runpod.io" + ) return f"{api_url_base.rstrip('/')}/{path.lstrip('/')}" @@ -45,30 +49,44 @@ def _response_json(response: requests.Response) -> dict[str, Any]: return payload if isinstance(payload, dict) else {} -def _raise_for_error( - response: requests.Response, method: str, path: str +def _raise_for_status( + status_code: int, + payload: Mapping[str, Any], + text: str, + method: str, + path: str, ) -> None: - if response.status_code == HTTP_STATUS_UNAUTHORIZED: + """Map HTTP error details consistently across REST transports.""" + if status_code == HTTP_STATUS_UNAUTHORIZED: raise error.AuthenticationError( "Unauthorized request, please check your API key." ) - if response.status_code < HTTP_STATUS_BAD_REQUEST: + if status_code < HTTP_STATUS_BAD_REQUEST: return - payload = _response_json(response) message = payload.get("detail") or payload.get("title") if not message: - message = response.text or f"Request failed with status {response.status_code}" + message = text or f"Request failed with status {status_code}" raise error.QueryError( str(message), f"{method.upper()} {path}", - status_code=response.status_code, + status_code=status_code, errors=payload.get("errors"), ) +def _raise_for_error(response: requests.Response, method: str, path: str) -> None: + if response.status_code < HTTP_STATUS_BAD_REQUEST: + return + if response.status_code == HTTP_STATUS_UNAUTHORIZED: + _raise_for_status(response.status_code, {}, "", method, path) + _raise_for_status( + response.status_code, _response_json(response), response.text, method, path + ) + + def run_rest_request( method: str, path: str, diff --git a/runpod/api/sandboxes.py b/runpod/api/sandboxes.py new file mode 100644 index 000000000..74dac1fbd --- /dev/null +++ b/runpod/api/sandboxes.py @@ -0,0 +1,386 @@ +"""Asynchronous REST and streaming transport for CPU sandboxes.""" + +from __future__ import annotations + +import asyncio +import codecs +import json +import math +import re +from collections.abc import AsyncIterator, Mapping, Sequence +from typing import Any +from urllib.parse import quote, urlencode + +import aiohttp +from yarl import URL + +from runpod import error +from runpod.api.rest import ( + _build_headers, + _build_url, + _raise_for_status, + _resolve_api_key, +) + +_SANDBOX_PATH = "/v2/sandboxes" +_LINE_END = re.compile(r"\r\n|\r|\n") + + +def _sandbox_path(sandbox_id: str) -> str: + return f"{_SANDBOX_PATH}/{quote(sandbox_id, safe='')}" + + +async def _raise_response_error( + response: aiohttp.ClientResponse, method: str, path: str +) -> None: + if response.status == 401: + _raise_for_status(response.status, {}, "", method, path) + if 200 <= response.status < 300: + return + raw = await response.read() + try: + payload = json.loads(raw) + except (ValueError, UnicodeError): + payload = {} + if not isinstance(payload, dict): + payload = {} + try: + text = raw.decode(response.charset or "utf-8", errors="replace") + except LookupError: + text = raw.decode("utf-8", errors="replace") + _raise_for_status(response.status, payload, text, method, path) + # Redirects are deliberately not followed, including on read operations. + raise error.QueryError( + text or f"Unexpected HTTP status {response.status}", + f"{method} {path}", + status_code=response.status, + errors=payload.get("errors"), + ) + + +async def _sse_lines(content: aiohttp.StreamReader) -> AsyncIterator[str]: + """Decode only the current line, allowing arbitrary UTF-8/chunk boundaries.""" + decoder = codecs.getincrementaldecoder("utf-8-sig")() + pending = "" + skip_lf = False + async for chunk in content.iter_chunked(8192): + text = decoder.decode(chunk) + if not text: + continue + if skip_lf: + if text.startswith("\n"): + text = text[1:] + skip_lf = False + text = pending + text + start = 0 + for match in _LINE_END.finditer(text): + yield text[start : match.start()] + start = match.end() + skip_lf = text.endswith("\r") + pending = text[start:] + pending += decoder.decode(b"", final=True) + if pending: + yield pending + + +class AsyncSandboxLogStream: + """One SSE connection; use an async context or ``aclose`` on early exit. + + ``open`` performs the HTTP handshake without waiting for a log event. + Exhaustion, cancellation, parsing errors and server timeout frames close + the response. No reconnect is attempted, even when a cursor is supplied. + """ + + def __init__( + self, + api: AsyncSandboxAPI, + path: str, + params: list[tuple[str, str]], + last_event_id: str | None, + ) -> None: + self._api = api + self._path = path + self._params = params + self._last_event_id = last_event_id + self._response: aiohttp.ClientResponse | None = None + self._open_task: asyncio.Task[None] | None = None + self._iterator: AsyncIterator[dict[str, Any]] | None = None + self._closed = False + + async def _perform_open(self) -> None: + try: + if self._closed: + raise RuntimeError("Sandbox log stream is closed") + headers = self._api._headers() + headers["Accept"] = "text/event-stream" + if self._last_event_id is not None: + headers["Last-Event-ID"] = self._last_event_id + self._response = await self._api._get_session().request( + "GET", + self._api._url(self._path, self._params), + headers=headers, + timeout=aiohttp.ClientTimeout( + total=None, + connect=self._api.request_timeout, + sock_connect=self._api.request_timeout, + sock_read=None, + ), + allow_redirects=False, + ) + await _raise_response_error(self._response, "GET", self._path) + if self._response.content_type != "text/event-stream": + raise error.QueryError( + "Expected a text/event-stream sandbox log response", + f"GET {self._path}", + status_code=self._response.status, + ) + self._iterator = self._events(self._response) + except BaseException: + self._dispose() + raise + + async def open(self) -> AsyncSandboxLogStream: + """Open once, with a bounded handshake but no total stream timeout.""" + if self._closed: + raise RuntimeError("Sandbox log stream is closed") + if self._open_task is None: + self._open_task = asyncio.create_task( + asyncio.wait_for( + self._perform_open(), timeout=self._api.request_timeout + ) + ) + try: + await self._open_task + except BaseException: + await self.aclose() + raise + return self + + def _dispose(self) -> None: + self._closed = True + if self._response is not None: + self._response.close() + self._response = None + self._iterator = None + self._api._streams.discard(self) + + async def aclose(self) -> None: + """Release this stream, including a handshake still in progress.""" + self._dispose() + if self._open_task is not None and not self._open_task.done(): + self._open_task.cancel() + await asyncio.gather(self._open_task, return_exceptions=True) + + async def __aenter__(self) -> AsyncSandboxLogStream: + return await self.open() + + async def __aexit__(self, exc_type, exc_value, traceback) -> None: + await self.aclose() + + def __aiter__(self) -> AsyncSandboxLogStream: + return self + + async def __anext__(self) -> dict[str, Any]: + if self._closed: + raise StopAsyncIteration + try: + await self.open() + assert self._iterator is not None + return await anext(self._iterator) + except BaseException: + await self.aclose() + raise + + async def _events( + self, response: aiohttp.ClientResponse + ) -> AsyncIterator[dict[str, Any]]: + data: list[str] = [] + event_type = "" + event_id: str | None = None + async for line in _sse_lines(response.content): + if not line: + if event_type == "timeout": + return + if event_type not in ("", "message"): + raise error.QueryError( + f"Sandbox log event {event_type!r}: " + "\n".join(data), + f"GET {self._path}", + ) + if data: + payload = json.loads("\n".join(data)) + if not isinstance(payload, dict): + raise ValueError("Sandbox log event must be a JSON object") + yield { + "source": payload["source"], + "line": payload["line"], + "ts": payload["ts"], + "id": event_id, + } + data = [] + event_type = "" + continue + if line.startswith(":"): + continue + field, separator, value = line.partition(":") + if separator and value.startswith(" "): + value = value[1:] + if field == "data": + data.append(value) + elif field == "event": + event_type = value + elif field == "id" and "\x00" not in value: + event_id = value + # SSE retry directives are intentionally ignored: no reconnects. + # Per SSE, EOF does not dispatch an event lacking its blank-line boundary. + + +class AsyncSandboxAPI: + """Lazy, owned aiohttp transport. Each operation makes one HTTP request. + + ``close`` releases streams and sockets, but a later request can open a new + session. Use the instance from one event loop while its session is active. + Readiness and retry policies belong to the higher-level sandbox handles. + """ + + def __init__( + self, + api_key: str | None = None, + base_url: str | None = None, + request_timeout: float = 30, + ) -> None: + if not math.isfinite(request_timeout) or request_timeout <= 0: + raise ValueError("request_timeout must be a finite positive number") + self.api_key = api_key + self.base_url = base_url + self.request_timeout = request_timeout + self._session: aiohttp.ClientSession | None = None + self._streams: set[AsyncSandboxLogStream] = set() + + def _headers(self) -> dict[str, str]: + return _build_headers(_resolve_api_key(self.api_key)) + + def _url(self, path: str, params: list[tuple[str, str]] | None = None) -> URL: + url = _build_url(path, self.base_url) + if params: + url += "?" + urlencode(params, safe="=") + # Preserve quoted ID segments (including encoded slashes) verbatim. + return URL(url, encoded=True) + + def _get_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=self.request_timeout), + cookie_jar=aiohttp.DummyCookieJar(), + ) + # aiohttp otherwise transparently replays disconnected GET/DELETE + # requests. It currently exposes no public switch for this policy. + self._session._retry_connection = False + return self._session + + async def _request( + self, + method: str, + path: str, + *, + params: list[tuple[str, str]] | None = None, + body: Mapping[str, Any] | None = None, + ) -> dict[str, Any] | None: + headers = self._headers() + async with self._get_session().request( + method, + self._url(path, params), + headers=headers, + json=body, + allow_redirects=False, + ) as response: + await _raise_response_error(response, method, path) + if response.status == 204: + return None + payload = await response.json(content_type=None) + if not isinstance(payload, dict): + raise ValueError("Sandbox API response must be a JSON object") + return payload + + async def create(self, body: Mapping[str, Any]) -> dict[str, Any]: + payload = await self._request( + "POST", + _SANDBOX_PATH, + body={key: value for key, value in body.items() if value is not None}, + ) + if payload is None: + raise ValueError("Sandbox creation returned no snapshot") + return payload + + async def list( + self, + *, + state: str | None = None, + labels: Mapping[str, str] | None = None, + ) -> list[dict[str, Any]]: + params = [] + if state is not None: + params.append(("state", state)) + if labels is not None: + params.extend(("labels", f"{key}={value}") for key, value in labels.items()) + payload = await self._request("GET", _SANDBOX_PATH, params=params) + if payload is None: + raise ValueError("Sandbox listing returned no response") + sandboxes = payload["sandboxes"] + if not isinstance(sandboxes, list) or any( + not isinstance(item, dict) for item in sandboxes + ): + raise ValueError("Sandbox listing must contain an array of objects") + return sandboxes + + async def get(self, sandbox_id: str) -> dict[str, Any]: + payload = await self._request("GET", _sandbox_path(sandbox_id)) + if payload is None: + raise ValueError("Sandbox lookup returned no snapshot") + return payload + + async def terminate(self, sandbox_id: str) -> None: + await self._request("DELETE", _sandbox_path(sandbox_id)) + + async def exec(self, sandbox_id: str, command: Sequence[str]) -> dict[str, Any]: + payload = await self._request( + "POST", f"{_sandbox_path(sandbox_id)}/exec", body={"command": list(command)} + ) + if payload is None: + raise ValueError("Sandbox execution returned no result") + return payload + + def logs( + self, + sandbox_id: str, + *, + source: str | None = None, + tail: int | None = None, + since: str | None = None, + last_event_id: str | None = None, + ) -> AsyncSandboxLogStream: + params = [ + (key, str(value)) + for key, value in (("source", source), ("tail", tail), ("since", since)) + if value is not None + ] + stream = AsyncSandboxLogStream( + self, f"{_sandbox_path(sandbox_id)}/logs", params, last_event_id + ) + self._streams.add(stream) + return stream + + async def close(self) -> None: + """Close all active streams and the owned session; allow later reuse.""" + streams = tuple(self._streams) + session, self._session = self._session, None + # Dispose every response before the first await, including on cancellation. + for stream in streams: + stream._dispose() + if stream._open_task is not None and not stream._open_task.done(): + stream._open_task.cancel() + try: + for stream in streams: + await stream.aclose() + finally: + if session is not None: + await session.close() diff --git a/runpod/sandbox/__init__.py b/runpod/sandbox/__init__.py new file mode 100644 index 000000000..9bcb930d5 --- /dev/null +++ b/runpod/sandbox/__init__.py @@ -0,0 +1,31 @@ +"""Sync and async sandbox lifecycles, execution results, and log streams.""" + +from .asyncio import AsyncioSandbox, AsyncSandboxLogStream +from .models import ( + ExecResult, + LogEvent, + LogSource, + SandboxCompute, + SandboxExecutionError, + SandboxInfo, + SandboxStartupTimeout, + SandboxState, + SandboxStateError, +) +from .sync import Sandbox, SandboxLogs + +__all__ = [ + "AsyncioSandbox", + "AsyncSandboxLogStream", + "Sandbox", + "SandboxLogs", + "ExecResult", + "LogEvent", + "LogSource", + "SandboxCompute", + "SandboxExecutionError", + "SandboxInfo", + "SandboxStartupTimeout", + "SandboxState", + "SandboxStateError", +] diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py new file mode 100644 index 000000000..849c9494d --- /dev/null +++ b/runpod/sandbox/asyncio.py @@ -0,0 +1,565 @@ +"""Asynchronous sandbox handles, lifecycle management, and typed log streams.""" + +import asyncio +import math +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import replace +from datetime import datetime +from types import TracebackType +from typing import Any, Optional, TypeVar + +from runpod.api.sandboxes import AsyncSandboxAPI +from runpod.error import QueryError +from runpod.sandbox.models import ( + ExecResult, + LogEvent, + LogSource, + SandboxCompute, + SandboxExecutionError, + SandboxInfo, + SandboxStartupTimeout, + SandboxState, + SandboxStateError, +) + +_T = TypeVar("_T") + + +def _timeout(value: float, name: str, *, allow_zero: bool = False) -> float: + if not math.isfinite(value) or value < 0 or (value == 0 and not allow_zero): + raise ValueError( + f"{name} must be finite and {'nonnegative' if allow_zero else 'positive'}" + ) + return value + + +async def _cleanup( + operation: Awaitable[None], + timeout: float, + original: Optional[BaseException] = None, +) -> None: + """Finish bounded cleanup despite cancellation, keeping the original primary.""" + task = asyncio.create_task(asyncio.wait_for(operation, timeout)) + interrupted = original + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as error: + if interrupted is None: + interrupted = error + except BaseException: + break + try: + task.result() + except BaseException as error: + if interrupted is not None: + raise interrupted from error + raise + if interrupted is not None: + raise interrupted + + +class AsyncioSandbox: + """A cached sandbox handle; construction never performs network I/O. + + Supply exactly one of ``image_name`` or ``template_id``. Unspecified + resource and lifetime options are left to the server's account policy. + Created handles own their remote sandbox in a context; ``get`` and ``list`` + return borrowed handles whose context only closes local resources. + """ + + def __init__( + self, + *, + image_name: Optional[str] = None, + template_id: Optional[str] = None, + name: Optional[str] = None, + cpu_flavor_id: Optional[str] = None, + vcpu_count: Optional[int] = None, + memory_in_gb: Optional[int] = None, + data_center_id: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + idle_timeout_seconds: Optional[int] = None, + max_lifetime_seconds: Optional[int] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> None: + if (image_name is None) == (template_id is None): + raise ValueError("Supply exactly one of image_name or template_id") + options = { + "imageName": image_name, + "templateId": template_id, + "name": name, + "cpuFlavorId": cpu_flavor_id, + "vcpuCount": vcpu_count, + "memoryInGb": memory_in_gb, + "dataCenterId": data_center_id, + "env": dict(env) if env is not None else None, + "idleTimeoutSeconds": idle_timeout_seconds, + "maxLifetimeSeconds": max_lifetime_seconds, + "labels": dict(labels) if labels is not None else None, + } + self._initialize(api_key, base_url, request_timeout, startup_timeout) + self._create_body = { + key: value for key, value in options.items() if value is not None + } + + def _initialize( + self, + api_key: Optional[str], + base_url: Optional[str], + request_timeout: float, + startup_timeout: float, + ) -> None: + self._request_timeout = _timeout(request_timeout, "request_timeout") + self._startup_timeout = _timeout( + startup_timeout, "startup_timeout", allow_zero=True + ) + self._api = AsyncSandboxAPI(api_key, base_url, request_timeout) + self._info: Optional[SandboxInfo] = None + self._sandbox_id: Optional[str] = None + self._create_body: Optional[dict[str, Any]] = None + self._owned = False + self._entering = False + self._creating = False + self._streams: set[AsyncSandboxLogStream] = set() + + @classmethod + async def create( + cls, + *, + image_name: Optional[str] = None, + template_id: Optional[str] = None, + name: Optional[str] = None, + cpu_flavor_id: Optional[str] = None, + vcpu_count: Optional[int] = None, + memory_in_gb: Optional[int] = None, + data_center_id: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + idle_timeout_seconds: Optional[int] = None, + max_lifetime_seconds: Optional[int] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> "AsyncioSandbox": + """Create and return an owned handle, without claiming container readiness.""" + sandbox = cls( + image_name=image_name, + template_id=template_id, + name=name, + cpu_flavor_id=cpu_flavor_id, + vcpu_count=vcpu_count, + memory_in_gb=memory_in_gb, + data_center_id=data_center_id, + env=env, + idle_timeout_seconds=idle_timeout_seconds, + max_lifetime_seconds=max_lifetime_seconds, + labels=labels, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, + ) + await sandbox._create() + return sandbox + + @classmethod + async def get( + cls, + sandbox_id: str, + *, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> "AsyncioSandbox": + """Fetch a borrowed handle; leaving its context does not terminate it.""" + sandbox = cls.__new__(cls) + sandbox._initialize(api_key, base_url, request_timeout, startup_timeout) + sandbox._sandbox_id = sandbox_id + try: + await sandbox.refresh() + except BaseException as error: + await _cleanup(sandbox.close(), request_timeout, error) + return sandbox + + @classmethod + async def list( + cls, + *, + state: Optional[SandboxState] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> list["AsyncioSandbox"]: + """Return borrowed snapshots with independent, not-yet-open sessions.""" + _timeout(request_timeout, "request_timeout") + _timeout(startup_timeout, "startup_timeout", allow_zero=True) + api = AsyncSandboxAPI(api_key, base_url, request_timeout) + try: + rows = await api.list(state=state, labels=labels) + snapshots = [SandboxInfo.from_dict(row) for row in rows] + except BaseException as error: + await _cleanup(api.close(), request_timeout, error) + raise + else: + await _cleanup(api.close(), request_timeout) + handles = [] + for info in snapshots: + sandbox = cls.__new__(cls) + sandbox._initialize(api_key, base_url, request_timeout, startup_timeout) + sandbox._info = info + sandbox._sandbox_id = info.id + handles.append(sandbox) + return handles + + @property + def info(self) -> SandboxInfo: + """The latest server snapshot; raises before creation, never fetches.""" + if self._info is None: + raise RuntimeError("Sandbox has not been created or fetched") + return self._info + + @property + def id(self) -> str: + """The cached sandbox identifier.""" + return self.info.id + + @property + def state(self) -> SandboxState: + """The cached lifecycle state, not a container-readiness guarantee.""" + return self.info.state + + @property + def compute(self) -> Optional[SandboxCompute]: + """The cached allocation, if one currently exists.""" + return self.info.compute + + @property + def expires_at(self) -> datetime: + """The cached maximum-lifetime deadline.""" + return self.info.expires_at + + async def _create(self) -> None: + if self._creating: + raise RuntimeError("Sandbox creation is already in progress") + if self._sandbox_id is not None: + return + if self._create_body is None: + raise RuntimeError("A borrowed sandbox cannot create a new resource") + self._creating = True + + async def create_remote() -> None: + data = await self._api.create(self._create_body) + sandbox_id = data["id"] + if not isinstance(sandbox_id, str) or not sandbox_id: + raise ValueError("Create response has no valid sandbox id") + self._sandbox_id = sandbox_id + self._owned = True + self._info = SandboxInfo.from_dict(data) + + task = asyncio.create_task(create_remote()) + try: + await asyncio.shield(task) + except BaseException as original: + + async def recover_and_release() -> None: + recovery_error = None + try: + if not task.done(): + await asyncio.wait_for(task, self._request_timeout) + else: + task.result() + except BaseException as error: + if error is not original: + recovery_error = error + try: + if self._sandbox_id is not None: + await self.terminate() + else: + await self.close() + except BaseException as error: + if recovery_error is not None: + raise error from recovery_error + raise + if recovery_error is not None: + raise recovery_error + + await _cleanup(recover_and_release(), 2 * self._request_timeout, original) + finally: + self._creating = False + + async def __aenter__(self) -> "AsyncioSandbox": + """Create if needed and enter a single, non-reentrant ownership scope.""" + if self._entering: + raise RuntimeError( + "Sandbox contexts cannot be nested or entered concurrently" + ) + self._entering = True + try: + await self._create() + return self + except BaseException: + self._entering = False + raise + + async def __aexit__( + self, + exc_type: Optional[type[BaseException]], + exc: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + """Release the owned resource, shielding bounded cleanup from cancellation.""" + try: + operation = self.terminate() if self._owned else self.close() + await _cleanup(operation, self._request_timeout, exc) + finally: + self._entering = False + + async def refresh(self) -> SandboxInfo: + """Fetch a fresh snapshot, including after local close or termination.""" + if self._sandbox_id is None: + raise RuntimeError("Sandbox has not been created or fetched") + self._info = SandboxInfo.from_dict(await self._api.get(self._sandbox_id)) + return self._info + + async def _ready_operation( + self, + operation: Callable[[], Awaitable[_T]], + startup_timeout: Optional[float], + *, + handshake: bool = False, + ) -> _T: + timeout = ( + self._startup_timeout + if startup_timeout is None + else _timeout(startup_timeout, "startup_timeout", allow_zero=True) + ) + sandbox_id = self.id + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while True: + try: + if handshake and timeout > 0: + try: + return await asyncio.wait_for( + operation(), deadline - loop.time() + ) + except asyncio.TimeoutError as error: + if loop.time() >= deadline: + raise SandboxStartupTimeout(sandbox_id, timeout) from error + raise + return await operation() + except QueryError as error: + if error.status_code != 409 or timeout == 0: + raise + remaining = deadline - loop.time() + if remaining <= 0: + raise SandboxStartupTimeout(sandbox_id, timeout) from error + try: + info = await asyncio.wait_for(self.refresh(), remaining) + except asyncio.TimeoutError as refresh_error: + if loop.time() >= deadline: + raise SandboxStartupTimeout( + sandbox_id, timeout + ) from refresh_error + raise + if info.state in ("FAILED", "TERMINATED"): + raise SandboxStateError(info) from error + remaining = deadline - loop.time() + if remaining <= 0: + raise SandboxStartupTimeout(sandbox_id, timeout) from error + await asyncio.sleep(min(0.25, remaining)) + if loop.time() >= deadline: + raise SandboxStartupTimeout(sandbox_id, timeout) from error + + async def exec( + self, + command: Sequence[str], + *, + check: bool = False, + startup_timeout: Optional[float] = None, + ) -> ExecResult: + """Execute argv, retrying only explicit 409 startup rejections. + + The startup deadline limits retries, not execution of an accepted + command. Each request retains its request timeout. Transport errors, + timeouts, and other HTTP errors are never retried because the command + might already have executed. ``check=True`` preserves output on failure. + """ + if isinstance(command, (str, bytes)) or not isinstance(command, Sequence): + raise TypeError( + "command must be a nonempty sequence of strings, not a string" + ) + if not command or any(not isinstance(argument, str) for argument in command): + raise ValueError("command must be a nonempty sequence of strings") + argv = tuple(command) + data = await self._ready_operation( + lambda: self._api.exec(self.id, argv), startup_timeout + ) + result = ExecResult(output=data["output"], error=data.get("error")) + if check and result.error is not None: + raise SandboxExecutionError(self.id, result) + return result + + def logs( + self, + *, + source: Optional[LogSource] = None, + tail: Optional[int] = None, + since: Optional[str] = None, + last_event_id: Optional[str] = None, + startup_timeout: Optional[float] = None, + ) -> "AsyncSandboxLogStream": + """Return a lazy, closeable stream of main-process or lifecycle logs. + + This does not include exec output. Only the initial HTTP handshake is + retried on 409; disconnects and errors after opening are never replayed. + Use an async context or ``aclose`` when stopping iteration early. + """ + sandbox_id = self.id + stream = AsyncSandboxLogStream( + self, sandbox_id, source, tail, since, last_event_id, startup_timeout + ) + self._streams.add(stream) + return stream + + async def terminate(self) -> None: + """Delete remotely and always close locally, even if deletion fails. + + The readable remote record remains. Only successful deletion changes + the cached state; timestamps are left untouched until ``refresh``. + """ + try: + if self._sandbox_id is None: + raise RuntimeError("Sandbox has not been created or fetched") + await self._api.terminate(self._sandbox_id) + if self._info is not None: + self._info = replace(self._info, state="TERMINATED", compute=None) + except BaseException as error: + await _cleanup(self.close(), self._request_timeout, error) + else: + await _cleanup(self.close(), self._request_timeout) + + async def close(self) -> None: + """Close streams and the local session without deleting; later reuse is allowed.""" + error = None + for stream in tuple(self._streams): + try: + await stream.aclose() + except BaseException as stream_error: + if error is None: + error = stream_error + try: + await self._api.close() + except BaseException as close_error: + if error is not None: + raise error from close_error + raise + if error is not None: + raise error + + +class AsyncSandboxLogStream: + """A handle-owned typed log stream with a separately awaitable handshake.""" + + def __init__( + self, + sandbox: AsyncioSandbox, + sandbox_id: str, + source: Optional[LogSource], + tail: Optional[int], + since: Optional[str], + last_event_id: Optional[str], + startup_timeout: Optional[float], + ) -> None: + self._sandbox = sandbox + self._sandbox_id = sandbox_id + self._options = dict( + source=source, tail=tail, since=since, last_event_id=last_event_id + ) + self._startup_timeout = startup_timeout + self._stream: Any = None + self._opened = False + self._closed = False + self._opening: Optional[asyncio.Task[None]] = None + + async def _open_once(self) -> None: + stream = self._sandbox._api.logs(self._sandbox_id, **self._options) + self._stream = stream + try: + await stream.open() + except BaseException as error: + self._stream = None + await _cleanup(stream.aclose(), self._sandbox._request_timeout, error) + + async def open(self) -> "AsyncSandboxLogStream": + """Complete the HTTP handshake without waiting for the first log event.""" + if self._closed: + raise RuntimeError("Log stream is closed") + if not self._opened: + if self._opening is None: + self._opening = asyncio.create_task( + self._sandbox._ready_operation( + self._open_once, self._startup_timeout, handshake=True + ) + ) + try: + await self._opening + if self._closed: + raise RuntimeError("Log stream was closed while opening") + self._opened = True + except BaseException as error: + await _cleanup(self.aclose(), self._sandbox._request_timeout, error) + return self + + def __aiter__(self) -> "AsyncSandboxLogStream": + return self + + async def __anext__(self) -> LogEvent: + if self._closed: + raise StopAsyncIteration + try: + await self.open() + return LogEvent.from_dict(await self._stream.__anext__()) + except StopAsyncIteration: + await self.aclose() + raise + except BaseException as error: + await _cleanup(self.aclose(), self._sandbox._request_timeout, error) + raise + + async def aclose(self) -> None: + """Release the response and detach from the sandbox; idempotent.""" + if self._closed: + return + self._closed = True + self._sandbox._streams.discard(self) + if self._opening is not None and not self._opening.done(): + self._opening.cancel() + try: + await self._opening + except BaseException: + # The open caller receives its failure; close only owns disposal. + pass + stream, self._stream = self._stream, None + if stream is not None: + await stream.aclose() + + async def __aenter__(self) -> "AsyncSandboxLogStream": + return await self.open() + + async def __aexit__( + self, + exc_type: Optional[type[BaseException]], + exc: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + await _cleanup(self.aclose(), self._sandbox._request_timeout, exc) diff --git a/runpod/sandbox/models.py b/runpod/sandbox/models.py new file mode 100644 index 000000000..712989e13 --- /dev/null +++ b/runpod/sandbox/models.py @@ -0,0 +1,136 @@ +"""Typed snapshots and results returned by the sandbox domain.""" + +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Literal, Mapping, Optional + +from runpod.error import RunPodError + +SandboxState = Literal["CREATING", "RUNNING", "TERMINATED", "FAILED"] +LogSource = Literal["container", "system"] + + +def _timestamp(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +@dataclass(frozen=True) +class SandboxCompute: + vcpu_count: float + memory_in_gb: float + container_disk_in_gb: int + cost_per_hr: float + + +@dataclass(frozen=True) +class SandboxInfo: + """A server snapshot. Reading fields does not fetch or refresh anything.""" + + id: str + name: str + state: SandboxState + idle_timeout_seconds: int + max_lifetime_seconds: int + last_activity_at: datetime + idle_expires_at: datetime + expires_at: datetime + created_at: datetime + updated_at: datetime + template_id: Optional[str] + image_name: Optional[str] + cpu_flavor_id: Optional[str] + termination_reason: Optional[str] + terminated_at: Optional[datetime] + labels: Mapping[str, str] + compute: Optional[SandboxCompute] + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "SandboxInfo": + compute = data.get("compute") + terminated_at = data.get("terminatedAt") + return cls( + id=data["id"], + name=data["name"], + state=data["state"], + idle_timeout_seconds=data["idleTimeoutSeconds"], + max_lifetime_seconds=data["maxLifetimeSeconds"], + last_activity_at=_timestamp(data["lastActivityAt"]), + idle_expires_at=_timestamp(data["idleExpiresAt"]), + expires_at=_timestamp(data["expiresAt"]), + created_at=_timestamp(data["createdAt"]), + updated_at=_timestamp(data["updatedAt"]), + template_id=data.get("templateId"), + image_name=data.get("imageName"), + cpu_flavor_id=data.get("cpuFlavorId"), + termination_reason=data.get("terminationReason"), + terminated_at=_timestamp(terminated_at) if terminated_at else None, + labels=dict(data.get("labels") or {}), + compute=( + SandboxCompute( + vcpu_count=compute["vcpuCount"], + memory_in_gb=compute["memoryInGb"], + container_disk_in_gb=compute["containerDiskInGb"], + cost_per_hr=compute["costPerHr"], + ) + if compute is not None + else None + ), + ) + + +@dataclass(frozen=True) +class ExecResult: + """Combined command output and the host's optional failure description.""" + + output: str + error: Optional[str] = None + + +@dataclass(frozen=True) +class LogEvent: + """One log record; preserve the opaque SSE id for subsequent resume requests.""" + + source: LogSource + line: str + timestamp: datetime + id: Optional[str] = None + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "LogEvent": + return cls( + source=data["source"], + line=data["line"], + timestamp=_timestamp(data["ts"]), + id=data.get("id"), + ) + + +class SandboxExecutionError(RunPodError): + """A command failed with check=True; its partial output remains available.""" + + def __init__(self, sandbox_id: str, result: ExecResult): + super().__init__(result.error) + self.sandbox_id = sandbox_id + self.result = result + + +class SandboxStateError(RunPodError): + """An operation cannot proceed because the sandbox reached a terminal state.""" + + def __init__(self, info: SandboxInfo): + super().__init__( + f"Sandbox {info.id} is {info.state}" + + (f": {info.termination_reason}" if info.termination_reason else "") + ) + self.info = info + + +class SandboxStartupTimeout(TimeoutError): + """The sandbox did not accept the operation within the startup deadline.""" + + def __init__(self, sandbox_id: str, timeout: float): + super().__init__( + f"Sandbox {sandbox_id} did not become ready within {timeout:g}s" + ) + self.sandbox_id = sandbox_id + self.timeout = timeout diff --git a/runpod/sandbox/sync.py b/runpod/sandbox/sync.py new file mode 100644 index 000000000..b5f9db64e --- /dev/null +++ b/runpod/sandbox/sync.py @@ -0,0 +1,438 @@ +"""Blocking sandbox facade backed by one stable asyncio loop per active handle.""" + +import asyncio +import threading +from concurrent.futures import Future +from datetime import datetime +from typing import Any, Coroutine, Iterator, Mapping, Optional, Sequence, TypeVar + +from .asyncio import AsyncioSandbox +from .models import ( + ExecResult, + LogEvent, + LogSource, + SandboxCompute, + SandboxInfo, + SandboxState, +) + +_T = TypeVar("_T") + + +class _LoopRunner: + """Keep aiohttp sessions on their original loop, including across log reads.""" + + def __init__(self) -> None: + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._thread: Optional[threading.Thread] = None + self._lock = threading.Lock() + + @property + def started(self) -> bool: + return self._thread is not None + + def _start(self) -> asyncio.AbstractEventLoop: + with self._lock: + if self._loop is None: + ready = threading.Event() + + def serve() -> None: + loop = asyncio.new_event_loop() + self._loop = loop + ready.set() + try: + loop.run_forever() + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + loop.close() + + self._thread = threading.Thread( + target=serve, name="runpod-sandbox", daemon=True + ) + self._thread.start() + ready.wait() + assert self._loop is not None + return self._loop + + def run(self, coroutine: Coroutine[Any, Any, _T]) -> _T: + if threading.current_thread() is self._thread: + coroutine.close() + raise RuntimeError("Cannot call the sync sandbox API from its own loop") + loop = self._start() + future: Future[_T] = Future() + finished = threading.Event() + task: Optional[asyncio.Task[tuple[Optional[_T], Optional[BaseException]]]] = ( + None + ) + invoked = False + + async def invoke() -> tuple[Optional[_T], Optional[BaseException]]: + nonlocal invoked + invoked = True + try: + return await coroutine, None + except BaseException as error: + # KeyboardInterrupt/SystemExit must reach the calling thread, + # not escape Task execution and stop the background loop. + return None, error + + def complete(done: asyncio.Task) -> None: + try: + result, error = done.result() + if error is not None: + future.set_exception(error) + else: + future.set_result(result) + except BaseException as error: + future.set_exception(error) + finally: + if not invoked: + coroutine.close() + finished.set() + + def submit() -> None: + nonlocal task + task = loop.create_task(invoke()) + task.add_done_callback(complete) + + def cancel() -> None: + if task is not None: + task.cancel() + + loop.call_soon_threadsafe(submit) + try: + return future.result() + except BaseException: + if not future.done(): + # Cancelling a concurrent Future marks it done before the async + # task's finally blocks finish. Wait for the actual task before + # stopping its loop, or Ctrl-C can strand a newly created sandbox. + loop.call_soon_threadsafe(cancel) + while not finished.is_set(): + try: + finished.wait() + except KeyboardInterrupt: + # Cleanup is bounded by the async domain's timeouts. + continue + raise + + def stop(self) -> None: + with self._lock: + loop, thread = self._loop, self._thread + if loop is None or thread is None: + return + + async def drain() -> None: + current = asyncio.current_task() + tasks = [task for task in asyncio.all_tasks() if task is not current] + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + asyncio.run_coroutine_threadsafe(drain(), loop).result() + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join() + self._loop = None + self._thread = None + + +class SandboxLogs(Iterator[LogEvent]): + """Closeable log iterator. Use a with block when stopping consumption early.""" + + def __init__(self, stream: Any, runner: _LoopRunner) -> None: + self._stream = stream + self._runner = runner + self._closed = False + + def __enter__(self) -> "SandboxLogs": + if self._closed: + raise RuntimeError("Log stream is closed") + self._runner.run(self._stream.__aenter__()) + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + self._closed = True + if self._runner.started: + self._runner.run(self._stream.__aexit__(exc_type, exc, traceback)) + return False + + def __iter__(self) -> "SandboxLogs": + return self + + def __next__(self) -> LogEvent: + if self._closed: + raise StopIteration + try: + return self._runner.run(self._stream.__anext__()) + except StopAsyncIteration: + self._closed = True + raise StopIteration from None + except BaseException: + self.close() + raise + + def close(self) -> None: + if not self._closed: + self._closed = True + if self._runner.started: + self._runner.run(self._stream.aclose()) + + +class Sandbox: + """An isolated CPU sandbox with blocking methods. + + ``with Sandbox(image_name=...)`` creates on entry and terminates on exit. + ``create`` leaves the lifetime under your control; call ``terminate`` when + finished, or ``close`` to release only local connections. Handles retrieved + with ``get`` or ``list`` never terminate compute on context exit. + + A stable background event loop runs the async implementation. No thread or + connection is started by the constructor. Prefer AsyncioSandbox in async + applications so blocking calls do not pause the application's event loop. + """ + + def __init__( + self, + *, + image_name: Optional[str] = None, + template_id: Optional[str] = None, + name: Optional[str] = None, + cpu_flavor_id: Optional[str] = None, + vcpu_count: Optional[int] = None, + memory_in_gb: Optional[int] = None, + data_center_id: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + idle_timeout_seconds: Optional[int] = None, + max_lifetime_seconds: Optional[int] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> None: + self._runner = _LoopRunner() + self._entered = False + self._sandbox = AsyncioSandbox( + image_name=image_name, + template_id=template_id, + name=name, + cpu_flavor_id=cpu_flavor_id, + vcpu_count=vcpu_count, + memory_in_gb=memory_in_gb, + data_center_id=data_center_id, + env=env, + idle_timeout_seconds=idle_timeout_seconds, + max_lifetime_seconds=max_lifetime_seconds, + labels=labels, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, + ) + + @classmethod + def _from_async( + cls, sandbox: AsyncioSandbox, runner: Optional[_LoopRunner] = None + ) -> "Sandbox": + instance = cls.__new__(cls) + instance._sandbox = sandbox + instance._runner = runner if runner is not None else _LoopRunner() + instance._entered = False + return instance + + @classmethod + def create( + cls, + *, + image_name: Optional[str] = None, + template_id: Optional[str] = None, + name: Optional[str] = None, + cpu_flavor_id: Optional[str] = None, + vcpu_count: Optional[int] = None, + memory_in_gb: Optional[int] = None, + data_center_id: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + idle_timeout_seconds: Optional[int] = None, + max_lifetime_seconds: Optional[int] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> "Sandbox": + """Create immediately; the returned snapshot may still be CREATING.""" + runner = _LoopRunner() + try: + sandbox = runner.run( + AsyncioSandbox.create( + image_name=image_name, + template_id=template_id, + name=name, + cpu_flavor_id=cpu_flavor_id, + vcpu_count=vcpu_count, + memory_in_gb=memory_in_gb, + data_center_id=data_center_id, + env=env, + idle_timeout_seconds=idle_timeout_seconds, + max_lifetime_seconds=max_lifetime_seconds, + labels=labels, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, + ) + ) + except BaseException: + runner.stop() + raise + return cls._from_async(sandbox, runner) + + @classmethod + def get( + cls, + sandbox_id: str, + *, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> "Sandbox": + """Fetch a borrowed handle; closing its context does not terminate it.""" + runner = _LoopRunner() + try: + sandbox = runner.run( + AsyncioSandbox.get( + sandbox_id, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, + ) + ) + except BaseException: + runner.stop() + raise + return cls._from_async(sandbox, runner) + + @classmethod + def list( + cls, + *, + state: Optional[SandboxState] = None, + labels: Optional[Mapping[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + request_timeout: float = 30, + startup_timeout: float = 60, + ) -> list["Sandbox"]: + """List borrowed handles. Filters combine with AND on the server.""" + runner = _LoopRunner() + try: + sandboxes = runner.run( + AsyncioSandbox.list( + state=state, + labels=labels, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, + ) + ) + finally: + runner.stop() + return [cls._from_async(sandbox) for sandbox in sandboxes] + + @property + def info(self) -> SandboxInfo: + return self._sandbox.info + + @property + def id(self) -> str: + return self._sandbox.id + + @property + def state(self) -> SandboxState: + return self._sandbox.state + + @property + def compute(self) -> Optional[SandboxCompute]: + return self._sandbox.compute + + @property + def expires_at(self) -> datetime: + return self._sandbox.expires_at + + def refresh(self) -> SandboxInfo: + """Fetch current server metadata and replace the local snapshot.""" + return self._runner.run(self._sandbox.refresh()) + + def exec( + self, + command: Sequence[str], + *, + check: bool = False, + startup_timeout: Optional[float] = None, + ) -> ExecResult: + """Execute argv, waiting only for explicit startup rejections.""" + return self._runner.run( + self._sandbox.exec(command, check=check, startup_timeout=startup_timeout) + ) + + def logs( + self, + *, + source: Optional[LogSource] = None, + tail: Optional[int] = None, + since: Optional[str] = None, + last_event_id: Optional[str] = None, + startup_timeout: Optional[float] = None, + ) -> SandboxLogs: + """Stream container/system logs, not exec output; close on early exit.""" + return SandboxLogs( + self._sandbox.logs( + source=source, + tail=tail, + since=since, + last_event_id=last_event_id, + startup_timeout=startup_timeout, + ), + self._runner, + ) + + def terminate(self) -> None: + """Release remote compute and local connections; safe to repeat.""" + try: + self._runner.run(self._sandbox.terminate()) + finally: + self._runner.stop() + + def close(self) -> None: + """Release local connections and the loop, without terminating compute.""" + if self._runner.started: + try: + self._runner.run(self._sandbox.close()) + finally: + self._runner.stop() + + def __enter__(self) -> "Sandbox": + if self._entered: + raise RuntimeError("Sandbox context is already entered") + try: + self._runner.run(self._sandbox.__aenter__()) + except BaseException: + self._runner.stop() + raise + self._entered = True + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + try: + self._runner.run(self._sandbox.__aexit__(exc_type, exc, traceback)) + return False + finally: + self._entered = False + self._runner.stop() diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py new file mode 100644 index 000000000..a326e0f3b --- /dev/null +++ b/tests/test_sandbox.py @@ -0,0 +1,458 @@ +"""Sandbox lifecycle and streaming regressions over an actual HTTP connection.""" + +import asyncio +import json +import subprocess +import sys +import threading +from collections import deque +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, unquote, urlsplit + +import pytest + +from runpod import AsyncioSandbox, Sandbox +from runpod.error import QueryError +from runpod.sandbox import ( + SandboxExecutionError, + SandboxStartupTimeout, + SandboxStateError, +) + + +class SandboxService: + """A controllable REST peer for lifecycle races, not a mocked SDK transport.""" + + def __init__(self): + self.records = {} + self.requests = [] + self.executed = [] + self.exec_statuses = deque() + self.log_statuses = deque() + self.delete_status = 204 + self.create_started = threading.Event() + self.allow_create = threading.Event() + self.allow_create.set() + self.log_started = threading.Event() + self.allow_logs = threading.Event() + self.allow_logs.set() + self.log_disconnected = threading.Event() + self.hold_logs = False + self.stopping = threading.Event() + # Mix line endings, split a UTF-8 codepoint, preserve an opaque SSE id, + # and include both a multiline data event and a non-JSON timeout frame. + self.log_bytes = ( + ": heartbeat\r\n\r\nid: opaque/42\r\nevent: message\r\n" + 'data: {"source":"container",\r\n' + 'data: "line":"caf\u00e9","ts":"2026-09-14T12:00:00Z"}\r\n\r\n' + 'id: opaque/43\rdata: {"source":"system","line":"started",' + '"ts":"2026-09-14T12:00:01Z"}\r\r' + "event: timeout\ndata: max stream duration reached\n\n" + ).encode() + + def create_record(self, body): + sandbox_id = f"sandbox-{len(self.records) + 1}" + now = "2026-09-14T12:00:00Z" + record = { + "id": sandbox_id, + "name": body.get("name", sandbox_id), + "state": "RUNNING", + "imageName": body.get("imageName"), + "templateId": body.get("templateId"), + "idleTimeoutSeconds": body.get("idleTimeoutSeconds", 300), + "maxLifetimeSeconds": body.get("maxLifetimeSeconds", 900), + "lastActivityAt": now, + "idleExpiresAt": "2026-09-14T12:05:00Z", + "expiresAt": "2026-09-14T12:15:00Z", + "createdAt": now, + "updatedAt": now, + "labels": body.get("labels", {}), + "compute": { + "vcpuCount": body.get("vcpuCount", 2), + "memoryInGb": body.get("memoryInGb", 4), + "containerDiskInGb": 10, + "costPerHr": 0.026, + }, + } + self.records[sandbox_id] = record + return record + + +@contextmanager +def sandbox_peer(): + service = SandboxService() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *_args): + pass + + def reply(self, status, data=None): + payload = json.dumps(data).encode() if data is not None else b"" + self.send_response(status) + self.send_header( + "Content-Type", + "application/problem+json" if status >= 400 else "application/json", + ) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + try: + self.wfile.write(payload) + except (BrokenPipeError, ConnectionResetError): + pass + + def handle_request(self): + parsed = urlsplit(self.path) + segments = [unquote(part) for part in parsed.path.split("/") if part] + content_length = int(self.headers.get("Content-Length", "0")) + body = ( + json.loads(self.rfile.read(content_length)) if content_length else None + ) + service.requests.append( + (self.command, parsed.path, body, dict(self.headers)) + ) + if self.headers.get("Authorization") != "Bearer sandbox-test-key": + self.reply(401, {"detail": "invalid key"}) + return + if segments == ["v2", "sandboxes"]: + if self.command == "POST": + record = service.create_record(body) + service.create_started.set() + service.allow_create.wait(5) + self.reply(201, record) + else: + query = parse_qs(parsed.query) + state = query.get("state", [None])[0] + records = [ + record + for record in service.records.values() + if ( + record["state"] == state + if state + else record["state"] != "TERMINATED" + ) + ] + for label in query.get("labels", []): + key, value = label.split("=", 1) + records = [ + record + for record in records + if record["labels"].get(key) == value + ] + self.reply(200, {"sandboxes": records}) + return + if ( + len(segments) < 3 + or segments[:2] != ["v2", "sandboxes"] + or segments[2] not in service.records + ): + self.reply(404, {"detail": "missing sandbox"}) + return + record = service.records[segments[2]] + if len(segments) == 3: + if self.command == "DELETE": + if service.delete_status == 204: + record.update( + state="TERMINATED", + terminatedAt="2026-09-14T12:01:00Z", + terminationReason="USER", + compute=None, + ) + self.reply(204) + else: + self.reply( + service.delete_status, {"detail": "cleanup unavailable"} + ) + else: + self.reply(200, record) + elif segments[3] == "exec": + status = ( + service.exec_statuses.popleft() if service.exec_statuses else 200 + ) + if record["state"] in ("FAILED", "TERMINATED"): + status = 409 + if status == 409: + self.reply(409, {"detail": "container not started"}) + return + service.executed.append(body["command"]) + if status >= 400: + self.reply( + status, {"detail": "response lost after command execution"} + ) + return + if body["command"] == ["fail"]: + self.reply( + 200, {"output": "partial output", "error": "command failed"} + ) + else: + self.reply(200, {"output": "completed", "error": None}) + elif segments[3] == "logs": + service.log_started.set() + service.allow_logs.wait(5) + status = service.log_statuses.popleft() if service.log_statuses else 200 + if status != 200: + self.reply(status, {"detail": "logs not ready"}) + return + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Connection", "close") + self.end_headers() + self.close_connection = True + try: + # Byte-at-a-time writes ensure parser state is exercised, + # regardless of how the client batches socket reads. + for byte in service.log_bytes: + self.wfile.write(bytes([byte])) + self.wfile.flush() + while service.hold_logs and not service.stopping.wait(0.01): + self.wfile.write(b": heartbeat\n\n") + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + pass + finally: + service.log_disconnected.set() + + do_GET = handle_request + do_POST = handle_request + do_DELETE = handle_request + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + service.options = { + "api_key": "sandbox-test-key", + "base_url": f"http://127.0.0.1:{server.server_port}", + "request_timeout": 2, + "startup_timeout": 2, + } + try: + yield service + finally: + service.allow_create.set() + service.stopping.set() + server.shutdown() + server.server_close() + thread.join() + + +@pytest.fixture +def peer(): + with sandbox_peer() as service: + yield service + + +def test_sync_startup_conflict_and_borrowed_context_ownership(peer): + peer.exec_statuses.extend([409, 200]) + with Sandbox(image_name="python:3.12-slim", **peer.options) as owner: + with Sandbox.get(owner.id, **peer.options) as borrowed: + assert borrowed.exec(["work"]).output == "completed" + assert peer.records[owner.id]["state"] == "RUNNING" + # A rejected nested entry must not destroy the outer context's client. + with pytest.raises(RuntimeError): + with owner: + pytest.fail("nested context entry succeeded") + assert owner.refresh().state == "RUNNING" + assert peer.records[owner.id]["state"] == "TERMINATED" + assert peer.executed == [["work"]] + + +@pytest.mark.asyncio +async def test_async_startup_conflict_and_borrowed_context_ownership(peer): + peer.exec_statuses.extend([409, 200]) + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as owner: + async with await AsyncioSandbox.get(owner.id, **peer.options) as borrowed: + assert (await borrowed.exec(["work"])).output == "completed" + assert peer.records[owner.id]["state"] == "RUNNING" + with pytest.raises(RuntimeError): + async with owner: + pytest.fail("nested context entry succeeded") + assert (await owner.refresh()).state == "RUNNING" + assert peer.records[owner.id]["state"] == "TERMINATED" + assert peer.executed == [["work"]] + + +@pytest.mark.asyncio +async def test_cancellation_during_creation_recovers_id_and_terminates(peer): + peer.allow_create.clear() + + async def create(): + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options): + pytest.fail("cancelled context was entered") + + task = asyncio.create_task(create()) + assert await asyncio.to_thread(peer.create_started.wait, 2) + task.cancel() + peer.allow_create.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, 4) + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + + +def test_sync_interrupt_during_creation_waits_for_resource_cleanup(): + script = """ +import os +import signal +import threading +from runpod import Sandbox +from tests.test_sandbox import sandbox_peer + +with sandbox_peer() as peer: + peer.allow_create.clear() + def interrupt(): + assert peer.create_started.wait(2) + os.kill(os.getpid(), signal.SIGINT) + threading.Timer(0.1, peer.allow_create.set).start() + worker = threading.Thread(target=interrupt) + worker.start() + try: + Sandbox.create(image_name="python:3.12-slim", **peer.options) + raise AssertionError("SIGINT was swallowed") + except KeyboardInterrupt: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + finally: + worker.join() +""" + process = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=10 + ) + assert process.returncode == 0, process.stdout + process.stderr + + +@pytest.mark.parametrize("delete_status", [204, 503]) +@pytest.mark.parametrize("failure_type", [ValueError, KeyboardInterrupt, SystemExit]) +def test_sync_body_exception_remains_primary_during_cleanup( + peer, delete_status, failure_type +): + peer.delete_status = delete_status + failure = failure_type("application failed") + with pytest.raises(failure_type) as raised: + with Sandbox(image_name="python:3.12-slim", **peer.options): + raise failure + assert raised.value is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + else: + assert isinstance(raised.value.__cause__, QueryError) + assert raised.value.__cause__.status_code == 503 + assert peer.records["sandbox-1"]["state"] == "RUNNING" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delete_status", [204, 503]) +async def test_async_body_exception_remains_primary_during_cleanup(peer, delete_status): + peer.delete_status = delete_status + failure = ValueError("application failed") + with pytest.raises(ValueError) as raised: + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options): + raise failure + assert raised.value is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + else: + assert isinstance(raised.value.__cause__, QueryError) + assert raised.value.__cause__.status_code == 503 + + +@pytest.mark.asyncio +async def test_exec_never_replays_ambiguous_failure_and_preserves_partial_output(peer): + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as sandbox: + peer.exec_statuses.append(500) + with pytest.raises(QueryError) as raised: + await sandbox.exec(["side-effect"]) + assert raised.value.status_code == 500 + assert peer.executed == [["side-effect"]] + result = await sandbox.exec(["fail"]) + assert result.output == "partial output" + assert result.error == "command failed" + with pytest.raises(SandboxExecutionError) as failure: + await sandbox.exec(["fail"], check=True) + assert failure.value.result == result + assert failure.value.sandbox_id == sandbox.id + + +@pytest.mark.asyncio +async def test_startup_deadline_and_terminal_state_do_not_execute_commands(peer): + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as sandbox: + peer.exec_statuses.extend([409] * 20) + with pytest.raises(QueryError) as rejected: + await sandbox.exec(["work"], startup_timeout=0) + assert rejected.value.status_code == 409 + with pytest.raises(SandboxStartupTimeout) as expired: + await sandbox.exec(["work"], startup_timeout=0.03) + assert expired.value.sandbox_id == sandbox.id + peer.records[sandbox.id]["state"] = "FAILED" + with pytest.raises(SandboxStateError) as failed: + await sandbox.exec(["work"]) + assert failed.value.info.state == "FAILED" + assert peer.executed == [] + + +def test_sync_sse_decodes_fragments_and_retains_resume_cursor(peer): + with Sandbox(image_name="python:3.12-slim", **peer.options) as sandbox: + with sandbox.logs(source="container", tail=0) as logs: + events = list(logs) + assert [(event.source, event.line, event.id) for event in events] == [ + ("container", "caf\u00e9", "opaque/42"), + ("system", "started", "opaque/43"), + ] + assert events[0].timestamp.isoformat() == "2026-09-14T12:00:00+00:00" + with sandbox.logs(last_event_id=events[-1].id) as logs: + assert next(logs).line == "caf\u00e9" + log_requests = [ + request for request in peer.requests if request[1].endswith("/logs") + ] + assert log_requests[-1][3]["Last-Event-ID"] == "opaque/43" + + +@pytest.mark.asyncio +async def test_async_log_early_close_releases_live_connection(peer): + peer.hold_logs = True + # Exclude the timeout frame so this stays open until the client closes it. + peer.log_bytes = peer.log_bytes.split(b"event: timeout")[0] + peer.log_statuses.extend([409, 200]) + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as sandbox: + async with sandbox.logs() as logs: + event = await logs.__anext__() + assert event.line == "caf\u00e9" + assert await asyncio.to_thread(peer.log_disconnected.wait, 2) + + +@pytest.mark.asyncio +async def test_closing_sandbox_cancels_pending_log_handshake(peer): + peer.allow_logs.clear() + sandbox = await AsyncioSandbox.create(image_name="python:3.12-slim", **peer.options) + logs = sandbox.logs() + opening = asyncio.create_task(logs.open()) + try: + assert await asyncio.to_thread(peer.log_started.wait, 2) + await asyncio.wait_for(sandbox.close(), 3) + with pytest.raises(asyncio.CancelledError): + await opening + assert peer.records[sandbox.id]["state"] == "RUNNING" + finally: + peer.allow_logs.set() + await sandbox.terminate() + + +@pytest.mark.asyncio +async def test_list_filters_and_independent_handles_do_not_delete_resources(peer): + first = peer.create_record( + {"imageName": "image", "labels": {"team": "a", "job": "42"}} + ) + second = peer.create_record( + {"imageName": "image", "labels": {"team": "a", "job": "43"}} + ) + peer.create_record({"imageName": "image", "labels": {"team": "b", "job": "42"}}) + matches = await AsyncioSandbox.list( + labels={"team": "a", "job": "42"}, **peer.options + ) + assert [sandbox.id for sandbox in matches] == [first["id"]] + await matches[0].close() + handles = await AsyncioSandbox.list(labels={"team": "a"}, **peer.options) + await handles[0].close() + async with handles[1] as borrowed: + assert (await borrowed.exec(["work"])).output == "completed" + assert borrowed.id == second["id"] + assert all(record["state"] == "RUNNING" for record in peer.records.values()) From 4fc05cbc21d7c1a6620541e48f05a587a8062985 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:07:20 -0700 Subject: [PATCH 100/120] refactor: simplify sandbox factories and test peer --- runpod/sandbox/asyncio.py | 47 +------ runpod/sandbox/sync.py | 46 +------ tests/test_sandbox.py | 250 ++++++++++++++++---------------------- 3 files changed, 113 insertions(+), 230 deletions(-) diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index 849c9494d..17a6b8600 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -89,7 +89,8 @@ def __init__( ) -> None: if (image_name is None) == (template_id is None): raise ValueError("Supply exactly one of image_name or template_id") - options = { + self._initialize(api_key, base_url, request_timeout, startup_timeout) + self._create_body = { "imageName": image_name, "templateId": template_id, "name": name, @@ -102,10 +103,6 @@ def __init__( "maxLifetimeSeconds": max_lifetime_seconds, "labels": dict(labels) if labels is not None else None, } - self._initialize(api_key, base_url, request_timeout, startup_timeout) - self._create_body = { - key: value for key, value in options.items() if value is not None - } def _initialize( self, @@ -128,43 +125,9 @@ def _initialize( self._streams: set[AsyncSandboxLogStream] = set() @classmethod - async def create( - cls, - *, - image_name: Optional[str] = None, - template_id: Optional[str] = None, - name: Optional[str] = None, - cpu_flavor_id: Optional[str] = None, - vcpu_count: Optional[int] = None, - memory_in_gb: Optional[int] = None, - data_center_id: Optional[str] = None, - env: Optional[Mapping[str, str]] = None, - idle_timeout_seconds: Optional[int] = None, - max_lifetime_seconds: Optional[int] = None, - labels: Optional[Mapping[str, str]] = None, - api_key: Optional[str] = None, - base_url: Optional[str] = None, - request_timeout: float = 30, - startup_timeout: float = 60, - ) -> "AsyncioSandbox": + async def create(cls, **options: Any) -> "AsyncioSandbox": """Create and return an owned handle, without claiming container readiness.""" - sandbox = cls( - image_name=image_name, - template_id=template_id, - name=name, - cpu_flavor_id=cpu_flavor_id, - vcpu_count=vcpu_count, - memory_in_gb=memory_in_gb, - data_center_id=data_center_id, - env=env, - idle_timeout_seconds=idle_timeout_seconds, - max_lifetime_seconds=max_lifetime_seconds, - labels=labels, - api_key=api_key, - base_url=base_url, - request_timeout=request_timeout, - startup_timeout=startup_timeout, - ) + sandbox = cls(**options) await sandbox._create() return sandbox @@ -208,7 +171,6 @@ async def list( snapshots = [SandboxInfo.from_dict(row) for row in rows] except BaseException as error: await _cleanup(api.close(), request_timeout, error) - raise else: await _cleanup(api.close(), request_timeout) handles = [] @@ -534,7 +496,6 @@ async def __anext__(self) -> LogEvent: raise except BaseException as error: await _cleanup(self.aclose(), self._sandbox._request_timeout, error) - raise async def aclose(self) -> None: """Release the response and detach from the sandbox; idempotent.""" diff --git a/runpod/sandbox/sync.py b/runpod/sandbox/sync.py index b5f9db64e..9658eeb19 100644 --- a/runpod/sandbox/sync.py +++ b/runpod/sandbox/sync.py @@ -245,51 +245,15 @@ def _from_async( return instance @classmethod - def create( - cls, - *, - image_name: Optional[str] = None, - template_id: Optional[str] = None, - name: Optional[str] = None, - cpu_flavor_id: Optional[str] = None, - vcpu_count: Optional[int] = None, - memory_in_gb: Optional[int] = None, - data_center_id: Optional[str] = None, - env: Optional[Mapping[str, str]] = None, - idle_timeout_seconds: Optional[int] = None, - max_lifetime_seconds: Optional[int] = None, - labels: Optional[Mapping[str, str]] = None, - api_key: Optional[str] = None, - base_url: Optional[str] = None, - request_timeout: float = 30, - startup_timeout: float = 60, - ) -> "Sandbox": + def create(cls, **options: Any) -> "Sandbox": """Create immediately; the returned snapshot may still be CREATING.""" - runner = _LoopRunner() + sandbox = cls(**options) try: - sandbox = runner.run( - AsyncioSandbox.create( - image_name=image_name, - template_id=template_id, - name=name, - cpu_flavor_id=cpu_flavor_id, - vcpu_count=vcpu_count, - memory_in_gb=memory_in_gb, - data_center_id=data_center_id, - env=env, - idle_timeout_seconds=idle_timeout_seconds, - max_lifetime_seconds=max_lifetime_seconds, - labels=labels, - api_key=api_key, - base_url=base_url, - request_timeout=request_timeout, - startup_timeout=startup_timeout, - ) - ) + sandbox._runner.run(sandbox._sandbox._create()) except BaseException: - runner.stop() + sandbox.close() raise - return cls._from_async(sandbox, runner) + return sandbox @classmethod def get( diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index a326e0f3b..88ba9d10c 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -1,16 +1,14 @@ """Sandbox lifecycle and streaming regressions over an actual HTTP connection.""" import asyncio -import json import subprocess import sys import threading from collections import deque from contextlib import contextmanager -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from urllib.parse import parse_qs, unquote, urlsplit import pytest +from aiohttp import web from runpod import AsyncioSandbox, Sandbox from runpod.error import QueryError @@ -82,158 +80,118 @@ def create_record(self, body): @contextmanager def sandbox_peer(): service = SandboxService() - - class Handler(BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def log_message(self, *_args): - pass - - def reply(self, status, data=None): - payload = json.dumps(data).encode() if data is not None else b"" - self.send_response(status) - self.send_header( - "Content-Type", - "application/problem+json" if status >= 400 else "application/json", + ready, stopped = threading.Event(), asyncio.Event() + + def failure(status, detail): + return web.json_response( + {"detail": detail}, status=status, content_type="application/problem+json" + ) + + async def handle(request): + body = await request.json() if request.can_read_body else None + service.requests.append( + (request.method, request.path, body, dict(request.headers)) + ) + if request.headers.get("Authorization") != "Bearer sandbox-test-key": + return failure(401, "invalid key") + sandbox_id = request.match_info.get("id") + if sandbox_id is None: + if request.method == "POST": + record = service.create_record(body) + service.create_started.set() + await asyncio.to_thread(service.allow_create.wait, 5) + return web.json_response(record, status=201) + state = request.query.get("state") + labels = [term.split("=", 1) for term in request.query.getall("labels", [])] + records = [ + record + for record in service.records.values() + if ( + record["state"] == state + if state + else record["state"] != "TERMINATED" + ) + and all(record["labels"].get(key) == value for key, value in labels) + ] + return web.json_response({"sandboxes": records}) + if sandbox_id not in service.records: + return failure(404, "missing sandbox") + record = service.records[sandbox_id] + operation = request.match_info.get("operation") + if request.method == "DELETE": + if service.delete_status != 204: + return failure(service.delete_status, "cleanup unavailable") + record.update(state="TERMINATED", compute=None) + return web.Response(status=204) + if operation == "exec": + status = service.exec_statuses.popleft() if service.exec_statuses else 200 + if status == 409 or record["state"] in ("FAILED", "TERMINATED"): + return failure(409, "container not started") + service.executed.append(body["command"]) + if status >= 400: + return failure(status, "response lost after command execution") + result = ( + {"output": "partial output", "error": "command failed"} + if body["command"] == ["fail"] + else {"output": "completed", "error": None} ) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() + return web.json_response(result) + if operation == "logs": + service.log_started.set() + await asyncio.to_thread(service.allow_logs.wait, 5) + status = service.log_statuses.popleft() if service.log_statuses else 200 + if status != 200: + return failure(status, "logs not ready") + response = web.StreamResponse(headers={"Content-Type": "text/event-stream"}) try: - self.wfile.write(payload) - except (BrokenPipeError, ConnectionResetError): + await response.prepare(request) + for byte in service.log_bytes: + await response.write(bytes([byte])) + await asyncio.sleep(0) + while service.hold_logs and not service.stopping.is_set(): + await response.write(b": heartbeat\n\n") + await asyncio.sleep(0.01) + except ConnectionResetError: pass - - def handle_request(self): - parsed = urlsplit(self.path) - segments = [unquote(part) for part in parsed.path.split("/") if part] - content_length = int(self.headers.get("Content-Length", "0")) - body = ( - json.loads(self.rfile.read(content_length)) if content_length else None - ) - service.requests.append( - (self.command, parsed.path, body, dict(self.headers)) - ) - if self.headers.get("Authorization") != "Bearer sandbox-test-key": - self.reply(401, {"detail": "invalid key"}) - return - if segments == ["v2", "sandboxes"]: - if self.command == "POST": - record = service.create_record(body) - service.create_started.set() - service.allow_create.wait(5) - self.reply(201, record) - else: - query = parse_qs(parsed.query) - state = query.get("state", [None])[0] - records = [ - record - for record in service.records.values() - if ( - record["state"] == state - if state - else record["state"] != "TERMINATED" - ) - ] - for label in query.get("labels", []): - key, value = label.split("=", 1) - records = [ - record - for record in records - if record["labels"].get(key) == value - ] - self.reply(200, {"sandboxes": records}) - return - if ( - len(segments) < 3 - or segments[:2] != ["v2", "sandboxes"] - or segments[2] not in service.records - ): - self.reply(404, {"detail": "missing sandbox"}) - return - record = service.records[segments[2]] - if len(segments) == 3: - if self.command == "DELETE": - if service.delete_status == 204: - record.update( - state="TERMINATED", - terminatedAt="2026-09-14T12:01:00Z", - terminationReason="USER", - compute=None, - ) - self.reply(204) - else: - self.reply( - service.delete_status, {"detail": "cleanup unavailable"} - ) - else: - self.reply(200, record) - elif segments[3] == "exec": - status = ( - service.exec_statuses.popleft() if service.exec_statuses else 200 - ) - if record["state"] in ("FAILED", "TERMINATED"): - status = 409 - if status == 409: - self.reply(409, {"detail": "container not started"}) - return - service.executed.append(body["command"]) - if status >= 400: - self.reply( - status, {"detail": "response lost after command execution"} - ) - return - if body["command"] == ["fail"]: - self.reply( - 200, {"output": "partial output", "error": "command failed"} - ) - else: - self.reply(200, {"output": "completed", "error": None}) - elif segments[3] == "logs": - service.log_started.set() - service.allow_logs.wait(5) - status = service.log_statuses.popleft() if service.log_statuses else 200 - if status != 200: - self.reply(status, {"detail": "logs not ready"}) - return - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Connection", "close") - self.end_headers() - self.close_connection = True - try: - # Byte-at-a-time writes ensure parser state is exercised, - # regardless of how the client batches socket reads. - for byte in service.log_bytes: - self.wfile.write(bytes([byte])) - self.wfile.flush() - while service.hold_logs and not service.stopping.wait(0.01): - self.wfile.write(b": heartbeat\n\n") - self.wfile.flush() - except (BrokenPipeError, ConnectionResetError): - pass - finally: - service.log_disconnected.set() - - do_GET = handle_request - do_POST = handle_request - do_DELETE = handle_request - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) + finally: + service.log_disconnected.set() + return response + return web.json_response(record) + + async def serve(): + service.loop = asyncio.get_running_loop() + app = web.Application() + for path in ( + "/v2/sandboxes", + "/v2/sandboxes/{id}", + "/v2/sandboxes/{id}/{operation}", + ): + app.router.add_route("*", path, handle) + runner = web.AppRunner(app, access_log=None, shutdown_timeout=1) + await runner.setup() + try: + await web.TCPSite(runner, "127.0.0.1", 0).start() + service.options = { + "api_key": "sandbox-test-key", + "base_url": f"http://127.0.0.1:{runner.addresses[0][1]}", + "request_timeout": 2, + "startup_timeout": 2, + } + ready.set() + await stopped.wait() + finally: + await runner.cleanup() + + thread = threading.Thread(target=lambda: asyncio.run(serve()), daemon=True) thread.start() - service.options = { - "api_key": "sandbox-test-key", - "base_url": f"http://127.0.0.1:{server.server_port}", - "request_timeout": 2, - "startup_timeout": 2, - } + assert ready.wait(5), "local sandbox peer did not start" try: yield service finally: service.allow_create.set() + service.allow_logs.set() service.stopping.set() - server.shutdown() - server.server_close() + service.loop.call_soon_threadsafe(stopped.set) thread.join() From a495fbe94e2f2cb4c008bf9f35be4d922f111ace Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:02:37 -0700 Subject: [PATCH 101/120] test: remove closed public export inventory --- tests/test_init.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 912423288..694ec2894 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -86,30 +86,6 @@ def test_private_imports_not_exported(self): for private_symbol in private_symbols: assert private_symbol not in all_symbols, f"Private symbol '{private_symbol}' should not be in __all__" - def test_all_covers_expected_public_api(self): - """Test that __all__ contains the expected public API symbols.""" - expected_symbols = { - # API functions - 'create_container_registry_auth', 'create_endpoint', 'create_pod', 'create_template', - 'delete_container_registry_auth', 'get_endpoints', 'get_gpu', 'get_gpus', - 'get_pod', 'get_pods', 'get_user', 'resume_pod', 'stop_pod', 'terminate_pod', - 'update_container_registry_auth', 'update_endpoint_template', 'update_user_settings', - # Config functions - 'check_credentials', 'get_credentials', 'set_credentials', - # Endpoint classes - 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', - # Serverless module - 'serverless', - # Logger class - 'RunPodLogger', - # Version - '__version__', - # Module variables - 'SSH_KEY_PATH', 'profile', 'api_key', 'endpoint_url_base' - } - - actual_symbols = set(runpod.__all__) - assert expected_symbols == actual_symbols, f"Expected {expected_symbols}, got {actual_symbols}" def test_no_duplicate_symbols_in_all(self): """Test that __all__ contains no duplicate symbols.""" From 73ffd7b3f42d84683a59e8570b2ee75063796e0c Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:16:00 -0700 Subject: [PATCH 102/120] docs: describe sandbox lifecycle and streaming --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 11b6d9b3b..1fd18ab7d 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Welcome to the official Python library for Runpod API & SDK. - [Quick Start](#quick-start) - [Local Test Worker](#local-test-worker) - [📚 | REST API v2 Wrapper](#--rest-api-v2-wrapper) + - [Sandboxes](#sandboxes) - [Endpoints](#endpoints) - [GPU Cloud (Pods)](#gpu-cloud-pods) - [📁 | Directory](#--directory) @@ -172,6 +173,41 @@ import runpod runpod.api_key = "your_runpod_api_key_found_under_settings" ``` +### Sandboxes + +`Sandbox` and `AsyncioSandbox` manage isolated CPU sandboxes through REST API v2. Set `RUNPOD_API_KEY`, assign `runpod.api_key`, or pass `api_key` to a handle. Supply exactly one of `image_name` or `template_id`. + +```python +from runpod import Sandbox + +with Sandbox(image_name="python:3.12-slim") as sandbox: + result = sandbox.exec(["python", "-c", "print('hello from a sandbox')"], check=True) + print(result.output) +``` + +Use `AsyncioSandbox` in asynchronous applications: + +```python +import asyncio +from runpod import AsyncioSandbox + +async def main(): + async with AsyncioSandbox(image_name="python:3.12-slim") as sandbox: + result = await sandbox.exec( + ["python", "-c", "print('hello from a sandbox')"], + check=True, + ) + print(result.output) + +asyncio.run(main()) +``` + +These contexts create a sandbox on entry and terminate it on exit, including when the body raises. `Sandbox.create(...)` and `await AsyncioSandbox.create(...)` return owned handles for explicit lifetime management. Call `terminate()` to release remote compute; `close()` releases local connections only. Handles returned by `get(sandbox_id)` or `list(state=..., labels=...)` are borrowed, so their contexts only close local resources. Async factories and lifecycle methods are awaited. + +Handle properties such as `state`, `compute`, and `expires_at` read cached metadata; `refresh()` fetches a current snapshot. Command execution accepts an argument sequence. `check=True` raises `SandboxExecutionError` with the partial output available in `error.result`. Explicit startup rejections are retried within `startup_timeout`; ambiguous transport failures are not replayed. + +`sandbox.logs(source="container", tail=10)` streams typed log events from the container's main process; `source="system"` selects lifecycle logs. Command output is returned by `exec`, not this stream. Use `with` and regular iteration for synchronous log streams, or `async with` and `async for` for asynchronous streams, to close the connection when stopping early. Preserve an event's `id` and pass it as `last_event_id` to resume a stream, or filter by `since`. + ### Endpoints You can interact with Runpod endpoints via a `run` or `run_sync` method. From 8d5f8a501e9990db74457fdef30ce6010aa2c1d0 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:26:35 -0700 Subject: [PATCH 103/120] fix: preserve sandbox errors through cleanup --- runpod/sandbox/asyncio.py | 32 +++++++++++++++++---------- runpod/sandbox/sync.py | 6 ++--- tests/test_sandbox.py | 46 +++++++++++++++++++++++++-------------- 3 files changed, 53 insertions(+), 31 deletions(-) diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index 17a6b8600..f45349a5e 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -38,7 +38,7 @@ async def _cleanup( timeout: float, original: Optional[BaseException] = None, ) -> None: - """Finish bounded cleanup despite cancellation, keeping the original primary.""" + """finish bounded cleanup; callers propagate an existing error on success.""" task = asyncio.create_task(asyncio.wait_for(operation, timeout)) interrupted = original while not task.done(): @@ -48,6 +48,7 @@ async def _cleanup( if interrupted is None: interrupted = error except BaseException: + # task.result() below propagates the failure with its cleanup context. break try: task.result() @@ -55,7 +56,7 @@ async def _cleanup( if interrupted is not None: raise interrupted from error raise - if interrupted is not None: + if interrupted is not None and original is None: raise interrupted @@ -149,6 +150,7 @@ async def get( await sandbox.refresh() except BaseException as error: await _cleanup(sandbox.close(), request_timeout, error) + raise return sandbox @classmethod @@ -171,6 +173,7 @@ async def list( snapshots = [SandboxInfo.from_dict(row) for row in rows] except BaseException as error: await _cleanup(api.close(), request_timeout, error) + raise else: await _cleanup(api.close(), request_timeout) handles = [] @@ -240,6 +243,7 @@ async def recover_and_release() -> None: else: task.result() except BaseException as error: + # finish releasing the resource before propagating recovery errors. if error is not original: recovery_error = error try: @@ -255,6 +259,7 @@ async def recover_and_release() -> None: raise recovery_error await _cleanup(recover_and_release(), 2 * self._request_timeout, original) + raise finally: self._creating = False @@ -407,6 +412,7 @@ async def terminate(self) -> None: self._info = replace(self._info, state="TERMINATED", compute=None) except BaseException as error: await _cleanup(self.close(), self._request_timeout, error) + raise else: await _cleanup(self.close(), self._request_timeout) @@ -417,6 +423,7 @@ async def close(self) -> None: try: await stream.aclose() except BaseException as stream_error: + # close every stream before propagating cancellation or another failure. if error is None: error = stream_error try: @@ -461,6 +468,7 @@ async def _open_once(self) -> None: except BaseException as error: self._stream = None await _cleanup(stream.aclose(), self._sandbox._request_timeout, error) + raise async def open(self) -> "AsyncSandboxLogStream": """Complete the HTTP handshake without waiting for the first log event.""" @@ -480,6 +488,7 @@ async def open(self) -> "AsyncSandboxLogStream": self._opened = True except BaseException as error: await _cleanup(self.aclose(), self._sandbox._request_timeout, error) + raise return self def __aiter__(self) -> "AsyncSandboxLogStream": @@ -496,6 +505,7 @@ async def __anext__(self) -> LogEvent: raise except BaseException as error: await _cleanup(self.aclose(), self._sandbox._request_timeout, error) + raise async def aclose(self) -> None: """Release the response and detach from the sandbox; idempotent.""" @@ -503,16 +513,14 @@ async def aclose(self) -> None: return self._closed = True self._sandbox._streams.discard(self) - if self._opening is not None and not self._opening.done(): - self._opening.cancel() - try: - await self._opening - except BaseException: - # The open caller receives its failure; close only owns disposal. - pass - stream, self._stream = self._stream, None - if stream is not None: - await stream.aclose() + try: + if self._opening is not None and not self._opening.done(): + self._opening.cancel() + await asyncio.gather(self._opening, return_exceptions=True) + finally: + stream, self._stream = self._stream, None + if stream is not None: + await stream.aclose() async def __aenter__(self) -> "AsyncSandboxLogStream": return await self.open() diff --git a/runpod/sandbox/sync.py b/runpod/sandbox/sync.py index 9658eeb19..73dbb8569 100644 --- a/runpod/sandbox/sync.py +++ b/runpod/sandbox/sync.py @@ -6,7 +6,7 @@ from datetime import datetime from typing import Any, Coroutine, Iterator, Mapping, Optional, Sequence, TypeVar -from .asyncio import AsyncioSandbox +from runpod.sandbox.asyncio import AsyncioSandbox from .models import ( ExecResult, LogEvent, @@ -73,8 +73,7 @@ async def invoke() -> tuple[Optional[_T], Optional[BaseException]]: try: return await coroutine, None except BaseException as error: - # KeyboardInterrupt/SystemExit must reach the calling thread, - # not escape Task execution and stop the background loop. + # interrupts belong to the calling thread, not the background loop. return None, error def complete(done: asyncio.Task) -> None: @@ -85,6 +84,7 @@ def complete(done: asyncio.Task) -> None: else: future.set_result(result) except BaseException as error: + # the waiting caller receives the task's cancellation or failure. future.set_exception(error) finally: if not invoked: diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 88ba9d10c..286e99f49 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -11,7 +11,7 @@ from aiohttp import web from runpod import AsyncioSandbox, Sandbox -from runpod.error import QueryError +from runpod.error import AuthenticationError, QueryError from runpod.sandbox import ( SandboxExecutionError, SandboxStartupTimeout, @@ -152,6 +152,7 @@ async def handle(request): await response.write(b": heartbeat\n\n") await asyncio.sleep(0.01) except ConnectionResetError: + # clients may stop reading before the fixture finishes streaming. pass finally: service.log_disconnected.set() @@ -285,16 +286,19 @@ def test_sync_body_exception_remains_primary_during_cleanup( ): peer.delete_status = delete_status failure = failure_type("application failed") - with pytest.raises(failure_type) as raised: + try: with Sandbox(image_name="python:3.12-slim", **peer.options): raise failure - assert raised.value is failure - if delete_status == 204: - assert peer.records["sandbox-1"]["state"] == "TERMINATED" + except failure_type as caught: + assert caught is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + else: + assert isinstance(caught.__cause__, QueryError) + assert caught.__cause__.status_code == 503 + assert peer.records["sandbox-1"]["state"] == "RUNNING" else: - assert isinstance(raised.value.__cause__, QueryError) - assert raised.value.__cause__.status_code == 503 - assert peer.records["sandbox-1"]["state"] == "RUNNING" + pytest.fail("sandbox context suppressed the application failure") @pytest.mark.asyncio @@ -302,15 +306,18 @@ def test_sync_body_exception_remains_primary_during_cleanup( async def test_async_body_exception_remains_primary_during_cleanup(peer, delete_status): peer.delete_status = delete_status failure = ValueError("application failed") - with pytest.raises(ValueError) as raised: + try: async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options): raise failure - assert raised.value is failure - if delete_status == 204: - assert peer.records["sandbox-1"]["state"] == "TERMINATED" + except ValueError as caught: + assert caught is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + else: + assert isinstance(caught.__cause__, QueryError) + assert caught.__cause__.status_code == 503 else: - assert isinstance(raised.value.__cause__, QueryError) - assert raised.value.__cause__.status_code == 503 + pytest.fail("sandbox context suppressed the application failure") @pytest.mark.asyncio @@ -386,8 +393,8 @@ async def test_closing_sandbox_cancels_pending_log_handshake(peer): try: assert await asyncio.to_thread(peer.log_started.wait, 2) await asyncio.wait_for(sandbox.close(), 3) - with pytest.raises(asyncio.CancelledError): - await opening + await asyncio.gather(opening, return_exceptions=True) + assert opening.cancelled() assert peer.records[sandbox.id]["state"] == "RUNNING" finally: peer.allow_logs.set() @@ -414,3 +421,10 @@ async def test_list_filters_and_independent_handles_do_not_delete_resources(peer assert (await borrowed.exec(["work"])).output == "completed" assert borrowed.id == second["id"] assert all(record["state"] == "RUNNING" for record in peer.records.values()) + + +@pytest.mark.asyncio +async def test_list_authentication_failure_propagates_after_cleanup(peer): + options = {**peer.options, "api_key": "invalid-key"} + with pytest.raises(AuthenticationError): + await AsyncioSandbox.list(**options) From 7211c6a324dbdf7dad9bfd3aa91a6087801da294 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:29:14 -0700 Subject: [PATCH 104/120] test: assert sandbox failures after context exit --- tests/test_sandbox.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 286e99f49..8b347512f 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -286,19 +286,19 @@ def test_sync_body_exception_remains_primary_during_cleanup( ): peer.delete_status = delete_status failure = failure_type("application failed") + caught = None try: with Sandbox(image_name="python:3.12-slim", **peer.options): raise failure - except failure_type as caught: - assert caught is failure - if delete_status == 204: - assert peer.records["sandbox-1"]["state"] == "TERMINATED" - else: - assert isinstance(caught.__cause__, QueryError) - assert caught.__cause__.status_code == 503 - assert peer.records["sandbox-1"]["state"] == "RUNNING" + except failure_type as error: + caught = error + assert caught is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" else: - pytest.fail("sandbox context suppressed the application failure") + assert isinstance(caught.__cause__, QueryError) + assert caught.__cause__.status_code == 503 + assert peer.records["sandbox-1"]["state"] == "RUNNING" @pytest.mark.asyncio @@ -306,18 +306,18 @@ def test_sync_body_exception_remains_primary_during_cleanup( async def test_async_body_exception_remains_primary_during_cleanup(peer, delete_status): peer.delete_status = delete_status failure = ValueError("application failed") + caught = None try: async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options): raise failure - except ValueError as caught: - assert caught is failure - if delete_status == 204: - assert peer.records["sandbox-1"]["state"] == "TERMINATED" - else: - assert isinstance(caught.__cause__, QueryError) - assert caught.__cause__.status_code == 503 + except ValueError as error: + caught = error + assert caught is failure + if delete_status == 204: + assert peer.records["sandbox-1"]["state"] == "TERMINATED" else: - pytest.fail("sandbox context suppressed the application failure") + assert isinstance(caught.__cause__, QueryError) + assert caught.__cause__.status_code == 503 @pytest.mark.asyncio From 7a24ee54dd5b4df46d67272b5be22bb3ee455f29 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:15:58 -0700 Subject: [PATCH 105/120] fix: enforce apps lifecycle and resource invariants --- README.md | 59 -------- docs/cli/references/command_line_interface.md | 9 ++ docs/cli/start_here.md | 2 + examples/apps/README.md | 5 + runpod/apps/api.py | 68 ++++++--- runpod/apps/app.py | 85 ++++------- runpod/apps/build.py | 8 +- runpod/apps/datacenter.py | 2 +- runpod/apps/deploy.py | 15 +- runpod/apps/dev.py | 60 +++++--- runpod/apps/discovery.py | 121 ++++++++++------ runpod/apps/handles.py | 2 + runpod/apps/markers.py | 12 +- runpod/apps/placement.py | 73 ++++++---- runpod/apps/serialization.py | 2 +- runpod/apps/shim.py | 6 +- runpod/apps/spec.py | 108 ++++++++++++-- runpod/apps/stubs.py | 24 ++- runpod/apps/targets.py | 109 +++++++++----- runpod/apps/tasks.py | 66 +++++---- runpod/apps/volume.py | 2 + runpod/rp_cli/main.py | 38 ++--- tests/test_apps/test_api_client.py | 61 ++++++-- tests/test_apps/test_app.py | 40 +++++ tests/test_apps/test_build.py | 52 ++++++- tests/test_apps/test_config_params.py | 102 +++++-------- tests/test_apps/test_deploy.py | 41 +++++- tests/test_apps/test_dev.py | 103 ++++++++++++- tests/test_apps/test_discovery.py | 119 +++++++++++++++ tests/test_apps/test_markers.py | 10 ++ tests/test_apps/test_placement.py | 48 +++++- tests/test_apps/test_retry.py | 41 +++++- tests/test_apps/test_shim.py | 94 ++++++++---- tests/test_apps/test_targets.py | 137 +++++++++++++++--- tests/test_apps/test_tasks.py | 110 +++++++++++++- tests/test_apps/test_volume.py | 19 +-- tests/test_cli/test_deploy_command.py | 91 +++++++++++- tests/test_cli/test_init_command.py | 16 +- tests/test_cli/test_management_commands.py | 49 ++++--- 39 files changed, 1476 insertions(+), 533 deletions(-) create mode 100644 tests/test_apps/test_discovery.py create mode 100644 tests/test_apps/test_markers.py diff --git a/README.md b/README.md index 1a378fc01..132596e05 100644 --- a/README.md +++ b/README.md @@ -82,65 +82,6 @@ make setup make test ``` -## REST API v2 - -Manage pods, endpoints, and templates through the API helpers. See the -[REST wrapper example](examples/rest_wrapper.py) and [API guide](docs/api/queries.md). - -```python -import runpod - -runpod.api_key = "your_runpod_api_key_found_under_settings" - -# get all my pods -pods = runpod.get_pods() - -# get a specific pod -pod = runpod.get_pod(pods[0]["id"]) - -# create a pod with a gpu -pod = runpod.create_pod("test", "runpod/stack", "NVIDIA GeForce RTX 4090") - -# create a pod with a cpu -pod = runpod.create_pod("test", "runpod/stack", instance_id="cpu3c-2-4") - -# stop the pod -runpod.stop_pod(pod["id"]) - -# resume the pod -runpod.resume_pod(pod["id"]) - -# terminate the pod -runpod.terminate_pod(pod["id"]) -``` - -### Template and placement options - -- With `create_pod(template_id=...)`, omitting `docker_args` inherits the template's - command. Pass `docker_args=""` to clear that inherited command. -- Omitting `volume_mount_path` preserves an inherited GPU template volume's path. - Setting it explicitly changes the path while retaining the inherited volume size. - New persistent volumes and network volume mounts default to `/runpod-volume`. - CPU pods do not inherit template persistent volumes; a CPU `volume_mount_path` - requires `network_volume_id`. -- For GPU pods, `min_memory_in_gb` and `min_vcpu_count` specify minimum host RAM - and vCPUs **per GPU**, not GPU VRAM or totals for the pod. -- CPU `instance_id` must use `--`, with positive - integer vCPU and memory values, for example `cpu3c-4-8`. -- REST v2 cannot require a public-IP-capable host. `support_public_ip=False` is - the default and does not disable public networking. `True` raises `ValueError` - rather than silently ignoring that requirement. -- `create_template(volume_in_gb=0)` creates a template without a persistent volume. -- `create_endpoint(locations="US-KS-2,EU-RO-1")` accepts comma-separated - datacenter IDs. Country codes such as `US` and `RO` are not supported. - Endpoint creation sends a single REST request without catalog lookups. -- Endpoint `gpu_ids` accepts pool IDs and excluded GPU types, for example - `gpu_ids="ADA_48_PRO,-NVIDIA L40"` selects that pool without NVIDIA L40 GPUs. -- `create_endpoint` accepts only `QUEUE_DELAY` and `REQUEST_COUNT` scaling. - `idle_timeout` applies to `QUEUE_DELAY` (default: 5 seconds); explicitly setting - it with `REQUEST_COUNT` raises `ValueError`. -- `resume_pod(pod_id)` keeps the existing GPU allocation. REST v2 does not support - resizing on resume, so providing `gpu_count` raises `ValueError`.
![Discord](https://discordapp.com/api/guilds/912829806415085598/widget.png?style=banner2) diff --git a/docs/cli/references/command_line_interface.md b/docs/cli/references/command_line_interface.md index f855bbc6e..b3a32bd98 100644 --- a/docs/cli/references/command_line_interface.md +++ b/docs/cli/references/command_line_interface.md @@ -36,9 +36,11 @@ rp --help ```bash rp login # browser approval rp login --api-key YOUR_KEY # store a key directly +rp login --profile staging # save browser credentials in a named profile ``` Credentials are stored in `~/.runpod/config.toml`. +Use `--profile NAME` with either login method to update only that profile; the default is `default`. ### Flash apps @@ -50,3 +52,10 @@ rp flash app list # list deployed apps rp flash env list --app my-app # list an app's environments rp flash undeploy --app my-app # delete an environment's endpoints ``` + +App discovery imports modules synchronously. Keep blocking work inside an +entrypoint or a resource function; interrupt discovery if an import blocks. + +On Ctrl-C, a dev session waits for its running local entrypoint to finish before +deleting endpoints. Blocking entrypoint code can delay shutdown. Cleanup failures +exit with an error and report the endpoint IDs that still need deletion. diff --git a/docs/cli/start_here.md b/docs/cli/start_here.md index 0269595db..d67e129a4 100644 --- a/docs/cli/start_here.md +++ b/docs/cli/start_here.md @@ -10,6 +10,8 @@ Note: This CLI is not the same as runpodctl and provides a different set of feat Store your Runpod API key by running `rp login` (browser approval) or `rp login --api-key YOUR_KEY`. Credentials are stored in `~/.runpod/config.toml`. +Pass `--profile staging` with either login method to update a named credential profile without changing the others. The default profile is `default`. + ### SSH Add an SSH key to your account by running `rp ssh add`. To use an existing key pass `--key` or `--key-file`. Keys are stored in `~/.runpod/ssh/`. If no key is specified a new one is generated and stored. diff --git a/examples/apps/README.md b/examples/apps/README.md index 49e9802c4..4fd1fa8ab 100644 --- a/examples/apps/README.md +++ b/examples/apps/README.md @@ -18,3 +18,8 @@ rp flash dev examples/apps/hello_world.py edit a file while the session is running, then press enter to re-run — workers pick up the new code automatically. the exhaustive feature-by-feature suite lives in [`tests/e2e/examples`](../../tests/e2e/examples). + +`await task.spawn.aio(...)` returns a task job that owns its pod. +`await job.wait(timeout=...)` terminates that pod when waiting finishes, including +timeout, task failure, or cancellation. use `await job.cancel()` to abandon a +spawned task explicitly. a failed termination retains `job.pod_id` for cleanup. diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 2483fdd8b..fb6104ee6 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -19,6 +19,19 @@ _TRANSPORT_RETRIES = 4 +def is_capacity_error(exc: Exception) -> bool: + message = str(exc).lower() + return any( + phrase in message + for phrase in ( + "resources to deploy your pod", + "insufficient capacity", + "no available machine", + "out of stock", + ) + ) + + class AppsApiClient: """async control-plane client scoped to apps provisioning.""" @@ -31,13 +44,12 @@ async def _execute( variables: Optional[Dict[str, Any]] = None, *, anonymous: bool = False, + retry: bool = False, ) -> Dict[str, Any]: import asyncio - last_exc: Optional[Exception] = None - for attempt in range(_TRANSPORT_RETRIES): - if attempt: - await asyncio.sleep(2 ** (attempt - 1)) + attempts = _TRANSPORT_RETRIES if retry else 1 + for attempt in range(attempts): try: response = await run_graphql_query_async( query, @@ -46,13 +58,10 @@ async def _execute( anonymous=anonymous, ) return response["data"] - except (aiohttp.ClientError, OSError, asyncio.TimeoutError) as exc: - # transport-level failures (ssl hiccups, resets, dns) - # are transient; graphql/auth errors propagate untouched - last_exc = exc - if last_exc is None: # pragma: no cover - loop always runs once - raise RuntimeError("graphql transport retry loop exited cleanly") - raise last_exc + except (aiohttp.ClientError, OSError, asyncio.TimeoutError): + if attempt == attempts - 1: + raise + await asyncio.sleep(2**attempt) async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: """create or update a serverless endpoint. include id to update.""" @@ -63,11 +72,13 @@ async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: async def delete_endpoint(self, endpoint_id: str) -> bool: mutation = app_mutations.MUTATION_DELETE_ENDPOINT data = await self._execute(mutation, {"id": endpoint_id}) - return bool(data.get("deleteEndpoint")) + if not data.get("deleteEndpoint"): + raise QueryError(f"endpoint {endpoint_id!r} was not deleted", mutation) + return True async def list_my_endpoints(self) -> List[Dict[str, Any]]: query = app_queries.QUERY_MY_ENDPOINTS - data = await self._execute(query) + data = await self._execute(query, retry=True) return data["myself"]["endpoints"] async def deploy_task_pod( @@ -76,13 +87,18 @@ async def deploy_task_pod( """deploy an on-demand pod for a task run.""" pod_input = dict(pod_input) if is_cpu: - # deployCpuPod takes a single instanceId - instance_ids = pod_input.pop("instanceIds", None) - if instance_ids: - pod_input["instanceId"] = instance_ids[0] + instance_ids = pod_input.pop("instanceIds", None) or [ + pod_input.get("instanceId") + ] mutation = app_mutations.MUTATION_DEPLOY_CPU_POD - data = await self._execute(mutation, {"input": pod_input}) - return data["deployCpuPod"] + for index, instance_id in enumerate(instance_ids): + candidate = dict(pod_input, instanceId=instance_id) + try: + data = await self._execute(mutation, {"input": candidate}) + return data["deployCpuPod"] + except QueryError as exc: + if not is_capacity_error(exc) or index == len(instance_ids) - 1: + raise mutation = app_mutations.MUTATION_DEPLOY_POD data = await self._execute(mutation, {"input": pod_input}) @@ -95,7 +111,7 @@ async def terminate_pod(self, pod_id: str) -> None: async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: query = app_queries.QUERY_FLASH_APP_BY_NAME try: - data = await self._execute(query, {"flashAppName": name}) + data = await self._execute(query, {"flashAppName": name}, retry=True) except QueryError as exc: if "not found" in str(exc).lower(): return None @@ -138,6 +154,7 @@ async def gpu_stock_status( "includeAiApi": not pods, }, }, + retry=True, ) gpu_types = data.get("gpuTypes") or [] first = gpu_types[0] if gpu_types else {} @@ -159,6 +176,7 @@ async def cpu_stock_status( "instanceId": instance_id, }, }, + retry=True, ) flavors = data.get("cpuFlavors") or [] first = flavors[0] if flavors else {} @@ -167,7 +185,7 @@ async def cpu_stock_status( async def list_network_volumes(self) -> List[Dict[str, Any]]: query = app_queries.QUERY_NETWORK_VOLUMES - data = await self._execute(query) + data = await self._execute(query, retry=True) return data["myself"].get("networkVolumes") or [] async def create_network_volume( @@ -188,7 +206,7 @@ async def create_network_volume( async def list_registry_auths(self) -> List[Dict[str, Any]]: query = app_queries.QUERY_REGISTRY_AUTHS - data = await self._execute(query) + data = await self._execute(query, retry=True) return data["myself"].get("containerRegistryCreds") or [] async def create_registry_auth( @@ -214,7 +232,7 @@ async def delete_registry_auth(self, auth_id: str) -> bool: async def list_secrets(self) -> List[Dict[str, Any]]: query = app_queries.QUERY_SECRETS - data = await self._execute(query) + data = await self._execute(query, retry=True) return data["myself"].get("secrets") or [] async def create_secret( @@ -241,7 +259,7 @@ async def delete_secret(self, secret_id: str) -> bool: async def list_apps(self) -> List[Dict[str, Any]]: """all flash apps with their environments and builds.""" query = app_queries.QUERY_FLASH_APPS - data = await self._execute(query) + data = await self._execute(query, retry=True) return data["myself"].get("flashApps") or [] async def get_environment_by_name( @@ -256,6 +274,7 @@ async def get_environment_by_name( data = await self._execute( query, {"input": {"flashAppId": app["id"], "name": env_name}}, + retry=True, ) except QueryError as exc: if "not found" in str(exc).lower(): @@ -285,6 +304,7 @@ async def get_auth_request_status(self, request_id: str) -> Dict[str, Any]: query, {"flashAuthRequestId": request_id}, anonymous=True, + retry=True, ) return data.get("flashAuthRequestStatus") or {} diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 45e28a70a..f0cc9cb93 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -25,11 +25,6 @@ async def transcribe(audio_url: str): ... DEFAULT_SCALER_VALUE, ResourceKind, ResourceSpec, - normalize_cpu, - normalize_cuda_version, - normalize_gpu, - normalize_scaler_type, - normalize_workers, ) from .targets import InvocationTarget, PodTarget, SentinelTarget @@ -48,6 +43,11 @@ def _clear_registry() -> None: _REGISTRY.clear() +def _restore_registry(apps: List["App"]) -> None: + """restore registrations after a failed discovery import.""" + _REGISTRY[:] = apps + + class App: """a named collection of deployable resources.""" @@ -104,25 +104,25 @@ def decorator(fn: Callable) -> FunctionHandle: spec = ResourceSpec( kind=ResourceKind.QUEUE, name=name or fn.__name__, - gpu=normalize_gpu(gpu), - cpu=normalize_cpu(cpu), + gpu=gpu, + cpu=cpu, gpu_count=gpu_count, - workers=normalize_workers(workers), + workers=workers, idle_timeout=idle_timeout, dependencies=dependencies, system_dependencies=system_dependencies, - volume=_volume_ref(volume), + volume=volume, env=env, - datacenter=_datacenter_list(datacenter), + datacenter=datacenter, image=image, registry_auth=registry_auth, model=model, max_concurrency=max_concurrency, execution_timeout_ms=execution_timeout_ms, flashboot=flashboot, - scaler_type=normalize_scaler_type(scaler_type), + scaler_type=scaler_type, scaler_value=scaler_value, - min_cuda_version=normalize_cuda_version(min_cuda_version), + min_cuda_version=min_cuda_version, accelerate_downloads=accelerate_downloads, container_disk_gb=container_disk_gb, ) @@ -156,17 +156,17 @@ def decorator(fn: Callable) -> FunctionHandle: spec = ResourceSpec( kind=ResourceKind.TASK, name=name or fn.__name__, - gpu=normalize_gpu(gpu), - cpu=normalize_cpu(cpu), + gpu=gpu, + cpu=cpu, gpu_count=gpu_count, dependencies=dependencies, system_dependencies=system_dependencies, - volume=_volume_ref(volume), + volume=volume, env=env, image=image, registry_auth=registry_auth, - datacenter=_datacenter_list(datacenter), - min_cuda_version=normalize_cuda_version(min_cuda_version), + datacenter=datacenter, + min_cuda_version=min_cuda_version, accelerate_downloads=accelerate_downloads, container_disk_gb=container_disk_gb, ) @@ -227,24 +227,24 @@ def decorator(target: Any) -> ApiHandle: spec = ResourceSpec( kind=ResourceKind.API, name=name or target.__name__, - gpu=normalize_gpu(gpu), - cpu=normalize_cpu(cpu), + gpu=gpu, + cpu=cpu, gpu_count=gpu_count, - workers=normalize_workers(workers), + workers=workers, idle_timeout=idle_timeout, dependencies=dependencies, system_dependencies=system_dependencies, - volume=_volume_ref(volume), + volume=volume, env=env, - datacenter=_datacenter_list(datacenter), + datacenter=datacenter, image=image, registry_auth=registry_auth, model=model, execution_timeout_ms=execution_timeout_ms, flashboot=flashboot, - scaler_type=normalize_scaler_type(scaler_type), + scaler_type=scaler_type, scaler_value=scaler_value, - min_cuda_version=normalize_cuda_version(min_cuda_version), + min_cuda_version=min_cuda_version, accelerate_downloads=accelerate_downloads, container_disk_gb=container_disk_gb, ) @@ -256,10 +256,16 @@ def decorator(target: Any) -> ApiHandle: async def _resolve(self, spec: ResourceSpec) -> InvocationTarget: """resolve a resource spec to an invocation target.""" + spec.validate() if spec.kind is ResourceKind.TASK: handle = self._resources.get(spec.name) fn = getattr(handle, "_fn", None) - return PodTarget(spec, fn, events=self._dev_events) + return PodTarget( + spec, + fn, + events=self._dev_events, + specs=[handle.spec for handle in self._resources.values()], + ) ctx = current_context() @@ -297,32 +303,3 @@ async def _resolve_dev_sibling(self, spec: ResourceSpec) -> InvocationTarget: def __repr__(self) -> str: return f"" - - -def _datacenter_list( - datacenter: Optional[Union[str, List[str]]], -) -> Optional[List[str]]: - """normalize datacenter input to a list of location strings.""" - if datacenter is None: - return None - if isinstance(datacenter, str): - return [datacenter] - return [str(d) for d in datacenter] - - -def _volume_ref(volume: Any) -> Optional[Any]: - """validate a volume argument: a Volume, or a name/id string. - - Volume objects pass through whole so creation config (size, - datacenter) survives to provision time. - """ - if volume is None: - return None - from .volume import Volume - - if isinstance(volume, (str, Volume)): - return volume - raise InvalidResourceError( - f"volume must be a runpod.Volume or a name/id string, " - f"got {type(volume).__name__}" - ) diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 967f0d60c..7be2b2fee 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -8,6 +8,7 @@ import sys import tempfile from dataclasses import dataclass, field +from collections import deque from pathlib import Path from typing import List, Optional, Tuple @@ -280,11 +281,13 @@ def vendor( proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, ) count = 0 + output = deque(maxlen=30) for line in proc.stdout: + output.append(line) line = line.strip() # pip emits one "Collecting " per resolved distribution if line.startswith("Collecting "): @@ -292,11 +295,10 @@ def vendor( if progress is not None: name = re.split(r"[><=!~\[ (]", line[11:], maxsplit=1)[0] progress(count, name) - stderr = proc.stderr.read() proc.wait() if proc.returncode != 0: raise BuildError( - f"dependency vendoring failed: {stderr[-3000:]}\n" + f"dependency vendoring failed: {''.join(output)[-3000:]}\n" f"packages without linux wheels cannot be vendored; " f"exclude them (they must then come from the image)" ) diff --git a/runpod/apps/datacenter.py b/runpod/apps/datacenter.py index d98236a9d..04b159b1a 100644 --- a/runpod/apps/datacenter.py +++ b/runpod/apps/datacenter.py @@ -34,7 +34,7 @@ def from_string(cls, value: str) -> "DataCenter": valid = ", ".join(dc.value for dc in cls) raise ValueError( f"unknown datacenter '{value}'. valid datacenters: {valid}" - ) + ) from None @classmethod def all(cls) -> List["DataCenter"]: diff --git a/runpod/apps/deploy.py b/runpod/apps/deploy.py index 9140fb79f..a864099a8 100644 --- a/runpod/apps/deploy.py +++ b/runpod/apps/deploy.py @@ -207,6 +207,8 @@ def _deployed_endpoint_input( environment_id: str, build_id: str, python_version: str, + *, + env_name: Optional[str] = None, ) -> Dict[str, Any]: """saveEndpoint payload for one deployed queue/api resource. @@ -220,11 +222,13 @@ def _deployed_endpoint_input( from .secret import render_env + spec.validate() template_env = { "FLASH_RESOURCE_NAME": spec.name, # version-triggering: a new build recreates all workers "RUNPOD_BUILD_ID": build_id, **render_env(spec.env), + "FLASH_ENVIRONMENT": env_name or app.env, } # cross-resource calls from inside a worker go through the sentinel, @@ -317,6 +321,8 @@ async def reconcile_endpoints( endpoints (matched by name) are updated in place; endpoints whose resources disappeared are deleted. """ + for handle in app.resources.values(): + handle.spec.validate() existing = {e["name"]: e["id"] for e in environment.get("endpoints") or []} provisionable = { h.spec.name: h @@ -331,7 +337,12 @@ async def reconcile_endpoints( endpoints: Dict[str, str] = {} for name, handle in sorted(provisionable.items()): payload = _deployed_endpoint_input( - app, handle.spec, environment["id"], build_id, python_version + app, + handle.spec, + environment["id"], + build_id, + python_version, + env_name=environment["name"], ) await attach_endpoint_volumes(payload, handle.spec, resolver, app) auth_id = await resolve_registry_auth(handle.spec.registry_auth, api=client) @@ -416,6 +427,8 @@ async def deploy_app( """run the full deploy pipeline for one app.""" client = api or AppsApiClient() env_name = env_name or app.env + for handle in app.resources.values(): + handle.spec.validate() # fail fast on unresolvable secret references (workers would boot # with the literal template string otherwise) diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index f3c36026a..ef12f1d9e 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -12,6 +12,7 @@ from .api import AppsApiClient from .app import App +from .errors import AppError from .handles import ApiHandle, FunctionHandle from .shim import runtime_launcher from .spec import ResourceKind, ResourceSpec @@ -22,11 +23,6 @@ log = logging.getLogger(__name__) -def _resource_of(endpoint_name: str) -> str: - """resource name from a dev endpoint name (display only).""" - return endpoint_name.rsplit("-", 2)[-2] - - def _comparable(payload: Dict) -> Dict: """payload normalized for change detection. @@ -118,7 +114,10 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: the endpoint and cascades on deleteEndpoint. """ + spec.validate() resource_env = _render_env(spec.env) + resource_env["FLASH_RESOURCE_NAME"] = spec.name + resource_env["RUNPOD_RESOURCE_NAME"] = spec.name if spec.kind is ResourceKind.API: resource_env["PORT"] = "80" resource_env["PORT_HEALTH"] = "80" @@ -217,6 +216,7 @@ def __init__( self.events = events # endpoint name -> id for everything this session owns self._endpoints: Dict[str, str] = {} + self._resource_names: Dict[str, str] = {} # endpoint name -> comparable payload, for refresh diffing self._payloads: Dict[str, Dict] = {} # volumes resolve once per session (placement is stable) @@ -278,11 +278,14 @@ async def start(self) -> None: for handle in self._provisionable(app): spec = handle.spec name = dev_endpoint_name(app.name, spec.name) + found = existing.get(name) + if found: + self._endpoints[name] = found["id"] + self._resource_names[name] = spec.name payload = _endpoint_input(app, spec, self.generation) await self._attach_volumes(payload, spec, app) hardware = ",".join(spec.cpu or spec.gpu or ["any"]) - found = existing.get(name) if found: # adopt: reconcile the leftover endpoint to the # current spec instead of creating a duplicate @@ -297,8 +300,9 @@ async def start(self) -> None: endpoint_id = result["id"] log.info("provisioned dev endpoint %s (%s)", name, endpoint_id) - self._emit("ready", spec.name, endpoint_id) self._endpoints[name] = endpoint_id + self._resource_names[name] = spec.name + self._emit("ready", spec.name, endpoint_id) self._payloads[name] = _comparable(payload) app._dev_targets[spec.name] = LiveTarget( endpoint_id, @@ -332,16 +336,26 @@ async def refresh(self, apps: List[App]) -> None: desired[name] = (app, handle) # delete endpoints whose resources disappeared + failures = [] for name in list(self._endpoints): if name not in desired: - endpoint_id = self._endpoints.pop(name) - self._payloads.pop(name, None) + endpoint_id = self._endpoints[name] + resource = self._resource_names[name] try: await self.api.delete_endpoint(endpoint_id) - self._emit("resource_removed", _resource_of(name)) - log.info("deleted removed dev endpoint %s", name) except Exception as exc: - log.warning("failed to delete %s: %s", name, exc) + failures.append(f"{resource} ({endpoint_id}): {exc}") + self._emit("delete_failed", resource) + log.warning("failed to delete %s (%s): %s", name, endpoint_id, exc) + else: + del self._endpoints[name] + del self._resource_names[name] + self._payloads.pop(name, None) + self._emit("resource_removed", resource) + log.info("deleted removed dev endpoint %s", name) + + if failures: + raise AppError("dev cleanup incomplete: " + "; ".join(failures)) # update survivors (config + generation bump) and create additions for name, (app, handle) in desired.items(): @@ -359,6 +373,7 @@ async def refresh(self, apps: List[App]) -> None: endpoint_id = result["id"] self._endpoints[name] = endpoint_id + self._resource_names[name] = handle.spec.name self._payloads[name] = comparable app._dev_targets[handle.spec.name] = LiveTarget( endpoint_id, @@ -396,24 +411,35 @@ async def stop(self, events: Optional[object] = None) -> None: sink = events if events is not None else self.events pending = list(self._endpoints.items()) + failures = [] emit(sink, "cleanup_started", len(pending)) for name, endpoint_id in pending: - resource = _resource_of(name) + resource = self._resource_names[name] emit(sink, "deleting", resource) try: await self.api.delete_endpoint(endpoint_id) - emit(sink, "deleted", resource) - log.info("deleted dev endpoint %s (%s)", name, endpoint_id) except Exception as exc: + failures.append(f"{resource} ({endpoint_id}): {exc}") emit(sink, "delete_failed", resource) log.warning("failed to delete dev endpoint %s: %s", endpoint_id, exc) - self._endpoints.clear() + else: + del self._endpoints[name] + del self._resource_names[name] + self._payloads.pop(name, None) + emit(sink, "deleted", resource) + log.info("deleted dev endpoint %s (%s)", name, endpoint_id) for app in self.apps: app._dev_targets.clear() app._dev_events = None + if failures: + raise AppError("dev cleanup incomplete: " + "; ".join(failures)) async def __aenter__(self) -> "DevSession": - await self.start() + try: + await self.start() + except BaseException: + await self.stop() + raise return self async def __aexit__(self, *exc_info) -> None: diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 4a1f0724d..848017d81 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -1,24 +1,23 @@ """module discovery for `rp flash deploy` and `rp flash dev`. imports target modules under __name__ != "__main__" (so main guards -never run) and collects App instances from the registry. a per-module -import timeout guards against module-level code that blocks. +never run) and collects App instances from the registry. imports run on +the caller thread so discovery never abandons executing module code. """ +import importlib import importlib.util import logging import os import sys -import threading from pathlib import Path from typing import List -from .app import App, get_registered_apps +from .app import App, _restore_registry, get_registered_apps from .discovery_state import DISCOVERY_ENV log = logging.getLogger(__name__) -IMPORT_TIMEOUT_SECONDS = 30 _SKIP_DIRS = { ".git", @@ -50,37 +49,56 @@ def _python_files(target: Path) -> List[Path]: return files -def _import_module(path: Path) -> None: - """import one file as a uniquely-named module, never as __main__.""" - module_name = f"_runpod_discovered_{path.stem}_{abs(hash(str(path)))}" - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - raise DiscoveryError(f"cannot load {path}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - - error: List[BaseException] = [] - - def run() -> None: - try: - spec.loader.exec_module(module) - # user module control-flow exceptions must surface as discovery - # errors on the caller thread, not kill this worker thread - except (Exception, KeyboardInterrupt, SystemExit) as exc: - error.append(exc) +def _module_name(path: Path) -> str: + parts = [] if path.name == "__init__.py" else [path.stem] + parent = path.parent + while (parent / "__init__.py").is_file(): + parts.insert(0, parent.name) + parent = parent.parent + if parent != path.parent: + return ".".join(parts) + return f"_runpod_discovered_{path.stem}_{abs(hash(str(path)))}" - thread = threading.Thread(target=run, daemon=True) - thread.start() - thread.join(IMPORT_TIMEOUT_SECONDS) - if thread.is_alive(): - raise DiscoveryError( - f"importing {path} timed out after {IMPORT_TIMEOUT_SECONDS}s; " - f"module-level code must not block (guard it with " - f'`if __name__ == "__main__":` or runpod.is_local())' - ) - if error: - raise DiscoveryError(f"importing {path} failed: {error[0]}") from error[0] +def _rollback_modules(before: set, root: Path) -> None: + for name in set(sys.modules) - before: + module = sys.modules.get(name) + source = getattr(module, "__file__", None) + if not source or not Path(source).resolve().is_relative_to(root): + continue + sys.modules.pop(name, None) + parent_name, _, child_name = name.rpartition(".") + parent = sys.modules.get(parent_name) + if parent is not None and getattr(parent, child_name, None) is module: + delattr(parent, child_name) + + +def _import_module(path: Path) -> None: + """import a file in its package context, never as __main__.""" + module_name = _module_name(path) + previous_module = sys.modules.get(module_name) + try: + if (path.parent / "__init__.py").is_file(): + module = importlib.import_module(module_name) + if Path(module.__file__).resolve() != path: + raise DiscoveryError( + f"module {module_name!r} is already loaded from {module.__file__}" + ) + else: + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None or spec.loader is None: + raise DiscoveryError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + except BaseException as exc: + if previous_module is not None: + sys.modules[module_name] = previous_module + else: + sys.modules.pop(module_name, None) + if isinstance(exc, (Exception, SystemExit)): + raise DiscoveryError(f"importing {path} failed: {exc}") from exc + raise def discover_apps(target: Path) -> List[App]: @@ -92,25 +110,35 @@ def discover_apps(target: Path) -> List[App]: app was found anywhere (the failures are then the likely cause and are included in the error). """ + target = target.resolve() before = set(id(a) for a in get_registered_apps()) - sys_path_added = False - root = target if target.is_dir() else target.parent - root_str = str(root.resolve()) - if root_str not in sys.path: - sys.path.insert(0, root_str) - sys_path_added = True + local_root = target if target.is_dir() else target.parent + root = local_root + while (root / "__init__.py").is_file(): + root = root.parent + added_paths = [] + for import_root in (local_root, root): + root_str = str(import_root) + if root_str not in sys.path: + sys.path.insert(0, root_str) + added_paths.append(root_str) strict = target.is_file() failures: List[str] = [] + previous_discovery = os.environ.get(DISCOVERY_ENV) os.environ[DISCOVERY_ENV] = "1" try: for path in _python_files(target): - seen = set(id(a) for a in get_registered_apps()) + registered = get_registered_apps() + seen = {id(a) for a in registered} + modules_before = set(sys.modules) try: _import_module(path) - except DiscoveryError as exc: - if strict: + except BaseException as exc: + _restore_registry(registered) + _rollback_modules(modules_before, root) + if strict or not isinstance(exc, DiscoveryError): raise failures.append(str(exc)) log.warning("%s", exc) @@ -121,8 +149,11 @@ def discover_apps(target: Path) -> List[App]: if id(app) not in seen and not hasattr(app, "_source_file"): app._source_file = path finally: - os.environ.pop(DISCOVERY_ENV, None) - if sys_path_added: + if previous_discovery is None: + os.environ.pop(DISCOVERY_ENV, None) + else: + os.environ[DISCOVERY_ENV] = previous_discovery + for root_str in added_paths: sys.path.remove(root_str) found = [a for a in get_registered_apps() if id(a) not in before] diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index 9587d8b11..b2f006204 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -119,6 +119,7 @@ def local(self, *args: Any, **kwargs: Any) -> Any: async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: self._guard_discovery() + self.spec.validate() ctx = current_context() if ctx is Context.WORKER and self._is_current_worker(): @@ -141,6 +142,7 @@ async def _stream_async(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: """invoke a generator function remotely, yielding partial outputs as the worker produces them.""" self._guard_discovery() + self.spec.validate() self._guard_generator() ctx = current_context() diff --git a/runpod/apps/markers.py b/runpod/apps/markers.py index 9f8e1e09b..84d1ad651 100644 --- a/runpod/apps/markers.py +++ b/runpod/apps/markers.py @@ -22,13 +22,13 @@ async def generate(self, body: dict): _VALID_METHODS = frozenset({"GET", "POST", "PUT", "DELETE", "PATCH"}) # paths used by the worker runtime; user routes must not collide -RESERVED_PATHS = frozenset({"/execute", "/ping"}) +RESERVED_PATHS = frozenset({"/execute", "/ping", "/_runpod/sync"}) -def _route_marker(method: str, path: str) -> Callable[[Callable], Callable]: - if method not in _VALID_METHODS: +def validate_route(method: str, path: str) -> None: + if not isinstance(method, str) or method not in _VALID_METHODS: raise ValueError(f"method must be one of {sorted(_VALID_METHODS)}") - if not path.startswith("/"): + if not isinstance(path, str) or not path.startswith("/"): raise ValueError(f"path must start with '/', got: {path!r}") if path in RESERVED_PATHS: raise ValueError( @@ -36,6 +36,10 @@ def _route_marker(method: str, path: str) -> Callable[[Callable], Callable]: f"(reserved: {', '.join(sorted(RESERVED_PATHS))})" ) + +def _route_marker(method: str, path: str) -> Callable[[Callable], Callable]: + validate_route(method, path) + def marker(fn: Callable) -> Callable: setattr(fn, ROUTE_ATTR, (method, path)) return fn diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py index 3ef7498aa..aa1b227e4 100644 --- a/runpod/apps/placement.py +++ b/runpod/apps/placement.py @@ -20,6 +20,7 @@ # stock signal ranking; unknown/none scores zero _STOCK_SCORE = {"HIGH": 3, "MEDIUM": 2, "LOW": 1} +StockKey = Tuple[str, str, int] class PlacementError(AppError): @@ -32,8 +33,8 @@ def _score(status: Optional[str]) -> int: return _STOCK_SCORE.get(status.strip().upper(), 0) -def _hardware_keys(spec) -> List[Tuple[str, str]]: - """(kind, id) stock lookup keys for a resource's hardware. +def _hardware_keys(spec) -> List[StockKey]: + """(kind, id, count) stock lookup keys for a resource's hardware. gpu entries may be pool ids or device names; pools expand to their device names since the stock api takes devices. tasks run on pods, @@ -43,18 +44,18 @@ def _hardware_keys(spec) -> List[Tuple[str, str]]: from .spec import ResourceKind if spec.is_cpu: - return [("cpu", c) for c in spec.cpu or []] + return [("cpu", c, 0) for c in spec.cpu or []] gpu_kind = "gpu-pod" if spec.kind is ResourceKind.TASK else "gpu" gpu = spec.gpu if not gpu or any(str(g).lower() == "any" for g in gpu): - return [(gpu_kind, "*")] - keys: List[Tuple[str, str]] = [] + return [(gpu_kind, "*", spec.gpu_count)] + keys: List[StockKey] = [] for entry in gpu: try: for device in GpuGroup(str(entry)).device_names(): - keys.append((gpu_kind, device)) + keys.append((gpu_kind, device, spec.gpu_count)) except ValueError: - keys.append((gpu_kind, str(entry))) + keys.append((gpu_kind, str(entry), spec.gpu_count)) return keys @@ -63,45 +64,47 @@ class StockMap: def __init__(self, api=None): self._api = api - self._gpu: Dict[Tuple[str, str], int] = {} - self._gpu_pod: Dict[Tuple[str, str], int] = {} + self._gpu: Dict[Tuple[str, int, str], int] = {} + self._gpu_pod: Dict[Tuple[str, int, str], int] = {} self._cpu: Dict[Tuple[str, str], int] = {} - self._fetched_gpu: Set[str] = set() - self._fetched_gpu_pod: Set[str] = set() + self._fetched_gpu: Set[Tuple[str, int]] = set() + self._fetched_gpu_pod: Set[Tuple[str, int]] = set() self._fetched_cpu: Set[str] = set() async def _client(self): self._api = default_client(self._api) return self._api - async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: - """populate stock for the given hardware keys across all DCs.""" + async def fetch(self, keys: Iterable[StockKey]) -> None: + """populate stock for the given hardware keys across all datacenters.""" keys = list(keys) gpu_ids = { - k[1] + (k[1], k[2]) for k in keys - if k[0] == "gpu" and k[1] != "*" and k[1] not in self._fetched_gpu + if k[0] == "gpu" + and k[1] != "*" + and (k[1], k[2]) not in self._fetched_gpu } gpu_pod_ids = { - k[1] + (k[1], k[2]) for k in keys if k[0] == "gpu-pod" and k[1] != "*" - and k[1] not in self._fetched_gpu_pod + and (k[1], k[2]) not in self._fetched_gpu_pod } cpu_ids = { k[1] for k in keys if k[0] == "cpu" and k[1] not in self._fetched_cpu } client = await self._client() jobs = [] - for gpu_id in gpu_ids: - self._fetched_gpu.add(gpu_id) + for gpu_id, count in gpu_ids: + self._fetched_gpu.add((gpu_id, count)) for dc in DataCenter.all(): - jobs.append(self._fetch_gpu(client, gpu_id, dc.value, False)) - for gpu_id in gpu_pod_ids: - self._fetched_gpu_pod.add(gpu_id) + jobs.append(self._fetch_gpu(client, gpu_id, count, dc.value, False)) + for gpu_id, count in gpu_pod_ids: + self._fetched_gpu_pod.add((gpu_id, count)) for dc in DataCenter.all(): - jobs.append(self._fetch_gpu(client, gpu_id, dc.value, True)) + jobs.append(self._fetch_gpu(client, gpu_id, count, dc.value, True)) for cpu_id in cpu_ids: self._fetched_cpu.add(cpu_id) for dc in DataCenter.all(): @@ -110,15 +113,17 @@ async def fetch(self, keys: Iterable[Tuple[str, str]]) -> None: await asyncio.gather(*jobs) async def _fetch_gpu( - self, client, gpu_id: str, dc: str, pods: bool + self, client, gpu_id: str, gpu_count: int, dc: str, pods: bool ) -> None: try: - status = await client.gpu_stock_status(gpu_id, dc, pods=pods) + status = await client.gpu_stock_status( + gpu_id, dc, gpu_count=gpu_count, pods=pods + ) except Exception: # noqa: BLE001 - stock is advisory log.debug("gpu stock query failed for %s@%s", gpu_id, dc, exc_info=True) status = None target = self._gpu_pod if pods else self._gpu - target[(gpu_id, dc)] = _score(status) + target[(gpu_id, gpu_count, dc)] = _score(status) async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: try: @@ -128,15 +133,19 @@ async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: status = None self._cpu[(instance_id, dc)] = _score(status) - def score(self, key: Tuple[str, str], dc: str) -> int: - kind, hw = key + def score(self, key: StockKey, dc: str) -> int: + kind, hw, count = key if kind in ("gpu", "gpu-pod"): table = self._gpu_pod if kind == "gpu-pod" else self._gpu if hw == "*": # any gpu: best signal among fetched devices, else assume ok - scores = [s for (g, d), s in table.items() if d == dc] + scores = [ + score + for (_, gpu_count, datacenter), score in table.items() + if datacenter == dc and gpu_count == count + ] return max(scores, default=1) - return table.get((hw, dc), 0) + return table.get((hw, count, dc), 0) return self._cpu.get((hw, dc), 0) @@ -146,9 +155,10 @@ def candidates(spec, stock: StockMap) -> Set[str]: hardware needs stock in the DC (any of the resource's acceptable devices/flavors), intersected with an explicit datacenter pin. """ + spec.validate() allowed = {dc.value for dc in DataCenter.all()} if spec.datacenter: - allowed &= {str(d) for d in spec.datacenter} + allowed &= set(spec.datacenter) keys = _hardware_keys(spec) if not keys: @@ -184,6 +194,7 @@ def solve_placement( per_resource = {spec.name: candidates(spec, stock) for spec in specs} if existing_dc is not None: + existing_dc = DataCenter.from_string(existing_dc).value blocked = [ name for name, dcs in per_resource.items() if existing_dc not in dcs ] diff --git a/runpod/apps/serialization.py b/runpod/apps/serialization.py index 684ccd719..425189e1a 100644 --- a/runpod/apps/serialization.py +++ b/runpod/apps/serialization.py @@ -36,7 +36,7 @@ def deserialize_result(result_b64: str) -> Any: raise RemoteExecutionError( f"the remote result contains an object from the " f"'{exc.name}' package, which is not installed locally. " - f"return plain python types (e.g. str(...) it) or install " + f"return plain python types (e.g. str(...)) or install " f"'{exc.name}' on this machine." ) from exc diff --git a/runpod/apps/shim.py b/runpod/apps/shim.py index dd61c9546..aae10cd20 100644 --- a/runpod/apps/shim.py +++ b/runpod/apps/shim.py @@ -54,8 +54,10 @@ def runtime_launcher(kind: str) -> str: f'! "$PY" -c "import runpod_sdk_runtime" >/dev/null 2>&1; then ' f'"$PY" -m pip install -q --upgrade "$RUNTIME_SPEC" || exit 1; ' f"fi; " - f'if [ -n "${{RUNPOD_PACKAGE_SPEC:-}}" ]; then ' - f'"$PY" -m pip install -q --upgrade "$RUNPOD_PACKAGE_SPEC" || exit 1; ' + f'SDK_SPEC="${{RUNPOD_PACKAGE_SPEC:-runpod}}"; ' + f'if [ -n "${{RUNPOD_PACKAGE_SPEC:-}}" ] || ' + f'! "$PY" -c "import runpod" >/dev/null 2>&1; then ' + f'"$PY" -m pip install -q --upgrade "$SDK_SPEC" || exit 1; ' f"fi; " f'exec "$PY" -m {module}' ) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index df5958869..881233e63 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -10,7 +10,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union from .errors import InvalidResourceError -from .gpu import GpuGroup, GpuType +from .datacenter import DataCenter +from .gpu import GpuGroup, GpuLike, GpuType DEFAULT_WORKERS: Tuple[int, int] = (0, 3) @@ -145,22 +146,58 @@ def normalize_cuda_version(version: Optional[str]) -> Optional[str]: return normalized +def normalize_datacenter( + datacenter: Optional[Union[str, List[str]]], +) -> Optional[List[str]]: + """normalize enum values and string variants to api datacenter ids.""" + if datacenter is None: + return None + values = [datacenter] if isinstance(datacenter, str) else datacenter + try: + return [DataCenter.from_string(value).value for value in values] + except (AttributeError, TypeError, ValueError) as exc: + raise InvalidResourceError(f"invalid datacenter: {datacenter!r}") from exc + + +def normalize_volume(volume: Any) -> Any: + """preserve volume configuration while normalizing reference collections.""" + from .volume import Volume + + if volume is None or isinstance(volume, (str, Volume)): + return volume + if isinstance(volume, (list, tuple)): + refs = [] + for item in volume: + if item is None: + raise InvalidResourceError("volume collections must contain references") + normalized = normalize_volume(item) + if isinstance(normalized, list): + refs.extend(normalized) + else: + refs.append(normalized) + return refs + raise InvalidResourceError( + f"volume must be a runpod.Volume, a name/id string, or a list/tuple " + f"of references, got {type(volume).__name__}" + ) + + @dataclass class ResourceSpec: """declarative config for one app resource.""" kind: ResourceKind name: str - gpu: Optional[List[str]] = None - cpu: Optional[List[str]] = None + gpu: Optional[Union[GpuLike, List[GpuLike]]] = None + cpu: Optional[Union[str, List[str]]] = None gpu_count: int = 1 - workers: Tuple[int, int] = DEFAULT_WORKERS + workers: Union[int, Tuple[int, int], None] = DEFAULT_WORKERS idle_timeout: int = 60 dependencies: Optional[List[str]] = None system_dependencies: Optional[List[str]] = None volume: Optional[Any] = None env: Optional[Dict[str, Any]] = None - datacenter: Optional[List[str]] = None + datacenter: Optional[Union[str, List[str]]] = None image: Optional[str] = None registry_auth: Optional[str] = None model: Optional[Any] = None @@ -177,12 +214,31 @@ class ResourceSpec: asgi_factory: Optional[str] = None def __post_init__(self) -> None: + self.validate() + + def validate(self) -> None: + """normalize and validate mutable configuration before it is consumed.""" + try: + self.kind = ResourceKind(self.kind) + except ValueError as exc: + raise InvalidResourceError(f"invalid resource kind: {self.kind!r}") from exc + self.gpu = normalize_gpu(self.gpu) + self.cpu = normalize_cpu(self.cpu) + self.workers = normalize_workers(self.workers) + self.scaler_type = normalize_scaler_type(self.scaler_type) + self.min_cuda_version = normalize_cuda_version(self.min_cuda_version) + self.datacenter = normalize_datacenter(self.datacenter) + self.volume = normalize_volume(self.volume) if self.gpu is not None and self.cpu is not None: raise InvalidResourceError( f"resource '{self.name}': gpu and cpu are mutually exclusive" ) - if not self.name: - raise InvalidResourceError("resource name must not be empty") + if not isinstance(self.name, str) or not self.name: + raise InvalidResourceError("resource name must be a non-empty string") + if self.gpu_count < 1: + raise InvalidResourceError( + f"resource '{self.name}': gpu_count must be >= 1" + ) if self.max_concurrency < 1: raise InvalidResourceError( f"resource '{self.name}': max_concurrency must be >= 1, " @@ -211,6 +267,31 @@ def __post_init__(self) -> None: f"available on queue and api resources; tasks download " f"weights themselves" ) + if self.kind is ResourceKind.TASK and isinstance(self.volume, list): + if len(self.volume) > 1: + raise InvalidResourceError( + f"task '{self.name}': pods mount exactly one volume" + ) + if self.schedule is not None and ( + not isinstance(self.schedule, str) or not self.schedule + ): + raise InvalidResourceError("cron must be a non-empty string") + from .markers import validate_route + + seen = set() + for route in self.routes: + if not isinstance(route, RouteSpec): + raise InvalidResourceError("routes must contain RouteSpec values") + try: + validate_route(route.method, route.path) + except ValueError as exc: + raise InvalidResourceError(str(exc)) from exc + key = (route.method, route.path) + if key in seen: + raise InvalidResourceError( + f"duplicate route {route.method} {route.path} on {self.name}" + ) + seen.add(key) @property def is_cpu(self) -> bool: @@ -227,6 +308,7 @@ def effective_scaler_type(self) -> str: def to_manifest(self) -> Dict[str, Any]: """serialize for the deploy manifest.""" + self.validate() data: Dict[str, Any] = { "kind": self.kind.value, "name": self.name, @@ -244,9 +326,15 @@ def to_manifest(self) -> Dict[str, Any]: if self.system_dependencies: data["systemDependencies"] = self.system_dependencies if self.volume: - data["networkVolume"] = getattr( - self.volume, "name", None - ) or str(self.volume) + if isinstance(self.volume, list): + data["networkVolumes"] = [ + getattr(volume, "name", None) or str(volume) + for volume in self.volume + ] + else: + data["networkVolume"] = getattr( + self.volume, "name", None + ) or str(self.volume) if self.env: from .secret import render_env diff --git a/runpod/apps/stubs.py b/runpod/apps/stubs.py index 04579d019..f0e5aeef4 100644 --- a/runpod/apps/stubs.py +++ b/runpod/apps/stubs.py @@ -31,7 +31,7 @@ def __init__( name: Optional[str] = None, id: Optional[str] = None, env: Optional[str] = None, - ): + ) -> None: if (name is None) == (id is None): raise ValueError("provide exactly one of name= or id=") self.app_name = app @@ -49,8 +49,15 @@ def _target(self) -> SentinelTarget: class Queue(_StubBase): """client for a queue resource deployed elsewhere.""" - def __init__(self, **kwargs: Any): - super().__init__(**kwargs) + def __init__( + self, + *, + app: str, + name: Optional[str] = None, + id: Optional[str] = None, + env: Optional[str] = None, + ) -> None: + super().__init__(app=app, name=name, id=id, env=env) self._job_options: dict = {} self.remote = Invoker(self._remote_async) self.stream = StreamInvoker(self._stream_async) @@ -137,8 +144,15 @@ async def aio(self, path: str, body: Any = None, **kwargs: Any) -> Any: class Api(_StubBase): """client for an api resource deployed elsewhere.""" - def __init__(self, **kwargs: Any): - super().__init__(**kwargs) + def __init__( + self, + *, + app: str, + name: Optional[str] = None, + id: Optional[str] = None, + env: Optional[str] = None, + ) -> None: + super().__init__(app=app, name=name, id=id, env=env) self.get = _StubRouteCaller(self, "GET") self.post = _StubRouteCaller(self, "POST") self.put = _StubRouteCaller(self, "PUT") diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index fbe626597..8ff7b9b30 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -5,10 +5,11 @@ runpod api); no local state is kept. """ +import asyncio import inspect import json from abc import ABC, abstractmethod -from typing import Any, AsyncIterator, Callable, Dict, Optional +from typing import Any, AsyncIterator, Callable, Dict, List, Optional import aiohttp @@ -59,6 +60,10 @@ def args_to_input(fn: Callable, args: tuple, kwargs: dict) -> Dict[str, Any]: f"{fn.__name__}() got {len(args)} positional args, " f"expected at most {len(params)}" ) + if params[i] in kwargs: + raise TypeError( + f"{fn.__name__}() got multiple values for argument '{params[i]}'" + ) body[params[i]] = arg body.update(kwargs) # the platform strips empty input dicts from jobs; keep one field so @@ -113,10 +118,22 @@ def merge_job_options( def unwrap_job_output(data: Dict[str, Any]) -> Any: """extract output from a runsync-style response, raising on failure.""" - if data.get("status") == "FAILED" or data.get("error"): - err = data.get("error") or data.get("output", {}).get("error", "unknown") - raise RemoteExecutionError(f"remote execution failed: {err}") output = data.get("output", data) + if data.get("status") == "FAILED" or data.get("error"): + err = data.get("error") + if not err: + if isinstance(output, dict): + err = output.get("error") + elif isinstance(output, list): + err = next( + ( + item["error"] + for item in reversed(output) + if isinstance(item, dict) and item.get("error") + ), + None, + ) + raise RemoteExecutionError(f"remote execution failed: {err or 'unknown'}") if isinstance(output, dict) and "error" in output: raise RemoteExecutionError(f"remote execution failed: {output['error']}") return output @@ -191,9 +208,8 @@ async def request( raise NotImplementedError(f"{type(self).__name__} does not serve http routes") -# transient statuses worth retrying: gateway/edge errors that occur -# while the sentinel cache warms or an edge node hiccups. 4xx (other -# than 429) are never retried; the request itself is wrong. +# transient statuses are retried only for idempotent requests; a failed +# submission response does not prove the job was rejected. RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504, 520, 522, 524}) RETRY_ATTEMPTS = 4 JOB_PROPAGATION_ATTEMPTS = 7 @@ -212,11 +228,15 @@ async def _request_json( retry_not_found: bool = False, ) -> Dict[str, Any]: """http json call with exponential backoff on transient failures.""" - import asyncio + retry_transient = method.upper() in {"GET", "HEAD", "OPTIONS"} client_timeout = aiohttp.ClientTimeout(total=timeout) last_exc: Optional[Exception] = None - attempts = JOB_PROPAGATION_ATTEMPTS if retry_not_found else RETRY_ATTEMPTS + attempts = ( + JOB_PROPAGATION_ATTEMPTS + if retry_not_found + else RETRY_ATTEMPTS if retry_transient else 1 + ) for attempt in range(attempts): if attempt: await asyncio.sleep(RETRY_BASE_DELAY * (2 ** (attempt - 1))) @@ -237,13 +257,14 @@ async def _request_json( status=resp.status, message=await resp.text(), ) + if not retry_transient: + raise last_exc continue resp.raise_for_status() return await resp.json() except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as exc: - # connection-level failures (reset, ssl hiccup, dns) are - # transient by nature; response-level errors above already - # decided retryability + if not retry_transient: + raise last_exc = exc continue if last_exc is None: # pragma: no cover - loop always runs once @@ -320,7 +341,10 @@ async def _wait_terminal( while data.get("status") not in FINAL_STATUSES: job_id = data.get("id") if not job_id: - return data + raise RemoteExecutionError( + "non-terminal job response is missing a job id " + f"(status: {data.get('status', 'UNKNOWN')})" + ) if time.monotonic() >= deadline: raise TimeoutError( f"job {job_id} did not complete within {timeout}s " @@ -572,6 +596,7 @@ def __init__( self._source_spec: Optional[ResourceSpec] = None # source hash the worker last confirmed; None forces a sync self._synced_hash: Optional[str] = None + self._sync_lock = asyncio.Lock() def attach_source( self, target: Any, resource: str, spec: Optional[ResourceSpec] = None @@ -595,22 +620,23 @@ async def _sync_source(self, timeout: float) -> None: from .serialization import get_function_source - source = get_function_source(self._source_target) - digest = hashlib.sha256(source.encode()).hexdigest() - if digest == self._synced_hash: - return - url = f"https://{self.endpoint_id}.{_lb_domain()}/_runpod/sync" - payload: Dict[str, Any] = { - "source": source, - "resource": self._source_resource, - } - if self._source_spec is not None: - payload["dependencies"] = self._source_spec.dependencies - payload["system_dependencies"] = ( - self._source_spec.system_dependencies - ) - await _post_json(url, payload, _headers(), timeout) - self._synced_hash = digest + async with self._sync_lock: + source = get_function_source(self._source_target) + digest = hashlib.sha256(source.encode()).hexdigest() + if digest == self._synced_hash: + return + url = f"https://{self.endpoint_id}.{_lb_domain()}/_runpod/sync" + payload: Dict[str, Any] = { + "source": source, + "resource": self._source_resource, + } + if self._source_spec is not None: + payload["dependencies"] = self._source_spec.dependencies + payload["system_dependencies"] = ( + self._source_spec.system_dependencies + ) + await _post_json(url, payload, _headers(), timeout) + self._synced_hash = digest def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict @@ -798,10 +824,13 @@ def __init__( spec: ResourceSpec, fn: Callable, events: Optional[object] = None, + *, + specs: Optional[List[ResourceSpec]] = None, ): self.spec = spec self.fn = fn self.events = events + self.specs = specs def build_payload( self, fn: Callable, spec: ResourceSpec, args: tuple, kwargs: dict @@ -837,7 +866,7 @@ async def invoke( start = time.monotonic() stream = None - execution = TaskExecution(self.spec) + execution = TaskExecution(self.spec, specs=self.specs) try: await execution.start() if execution.pod_id: @@ -864,9 +893,11 @@ async def invoke( ) raise finally: - if stream is not None: - await stream.stop() - await execution.terminate() + try: + if stream is not None: + await stream.stop() + finally: + await execution.terminate() result = unwrap_task_response(response) emit( self.events, @@ -879,8 +910,12 @@ async def invoke( async def submit(self, payload: Dict[str, Any]) -> Any: from .tasks import TaskExecution, TaskJob - execution = TaskExecution(self.spec) - await execution.start() - await execution.wait_ready() - await execution.submit(payload) + execution = TaskExecution(self.spec, specs=self.specs) + try: + await execution.start() + await execution.wait_ready() + await execution.submit(payload) + except BaseException: + await execution.terminate() + raise return TaskJob(execution) diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 934efa9b6..24d897f90 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -21,7 +21,7 @@ import aiohttp -from .api import AppsApiClient +from .api import AppsApiClient, is_capacity_error from .errors import RemoteExecutionError from .spec import ResourceSpec @@ -88,6 +88,7 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] runtime images have the runner baked in; custom images install and start the runtime package via dockerArgs. """ + spec.validate() terminate_after = ( datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME ).isoformat() @@ -126,9 +127,20 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] else: # spread across the storage-supported set; a null DC can pin # repeated deploys to one (possibly broken) machine - from .datacenter import CPU3_DATACENTERS, DataCenter - - pool = CPU3_DATACENTERS if spec.is_cpu else DataCenter.all() + from .datacenter import CPU3_DATACENTERS, CPU5_DATACENTERS, DataCenter + + if spec.is_cpu: + pool = dict.fromkeys( + dc + for instance in spec.cpu or [] + for dc in ( + CPU5_DATACENTERS + if instance.startswith("cpu5") + else CPU3_DATACENTERS + ) + ) + else: + pool = DataCenter.all() pod["dataCenterIds"] = [dc.value for dc in pool] if spec.is_cpu: pod["instanceIds"] = spec.cpu @@ -147,8 +159,15 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] class TaskExecution: """one pod running one function.""" - def __init__(self, spec: ResourceSpec, api: Optional[AppsApiClient] = None): + def __init__( + self, + spec: ResourceSpec, + api: Optional[AppsApiClient] = None, + *, + specs: Optional[List[ResourceSpec]] = None, + ): self.spec = spec + self.specs = specs if specs is not None else [spec] self.api = api or AppsApiClient() self.token = secrets.token_urlsafe(32) self.pod_id: Optional[str] = None @@ -179,7 +198,7 @@ async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: pod, is_cpu=self.spec.is_cpu ) except QueryError as exc: - if not self._is_capacity_error(exc): + if not is_capacity_error(exc): raise capacity_error = exc @@ -194,23 +213,10 @@ async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: candidate, is_cpu=self.spec.is_cpu ) except QueryError as exc: - if not self._is_capacity_error(exc): + if not is_capacity_error(exc): raise raise capacity_error - @staticmethod - def _is_capacity_error(exc: Exception) -> bool: - message = str(exc).lower() - return any( - phrase in message - for phrase in ( - "resources to deploy your pod", - "insufficient capacity", - "no available machine", - "out of stock", - ) - ) - async def _attach_volume(self, pod: Dict[str, Any]) -> Dict[str, Any]: """resolve the task's volume and pin the pod to its datacenter.""" from .volume import VolumeError, VolumeResolver, volume_list @@ -222,7 +228,16 @@ async def _attach_volume(self, pod: Dict[str, Any]) -> Dict[str, Any]: f"({len(volumes)} given)" ) resolver = VolumeResolver(self.api) - resolved = await resolver.resolve(volumes[0], [self.spec]) + sharing = [ + spec + for spec in self.specs + if any(ref.name == volumes[0].name for ref in volume_list(spec.volume)) + ] + if not any(spec is self.spec for spec in sharing): + sharing.append(self.spec) + for spec in sharing: + spec.validate() + resolved = await resolver.resolve(volumes[0], sharing) from .volume import POD_MOUNT_PATH pod["networkVolumeId"] = resolved["id"] @@ -330,13 +345,14 @@ async def terminate(self) -> None: try: await self.api.terminate_pod(self.pod_id) log.info("task pod %s terminated", self.pod_id) - except Exception as exc: # noqa: BLE001 - terminateAfter is the backstop + except Exception as exc: log.warning( "failed to terminate task pod %s (terminateAfter is the " "backstop): %s", self.pod_id, exc, ) + raise self.pod_id = None @@ -366,8 +382,7 @@ def pod_id(self) -> Optional[str]: return self._execution.pod_id async def wait(self, timeout: Optional[float] = None) -> Any: - """block until the task finishes; terminates the pod and returns - the result.""" + """wait for the result, terminating the pod on every exit.""" deadline = time.monotonic() + timeout if timeout is not None else None try: while not self._done: @@ -382,8 +397,7 @@ async def wait(self, timeout: Optional[float] = None) -> Any: break await asyncio.sleep(RESULT_POLL_INTERVAL) finally: - if self._done: - await self._execution.terminate() + await self._execution.terminate() return self._result async def cancel(self) -> None: diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index fcf3e8dc3..24f72333f 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -123,6 +123,8 @@ async def resolve( specs are all resource specs that attach this volume; they drive placement for creation and validate an existing DC. """ + for spec in specs: + spec.validate() cached = self._resolved.get(volume.name) if cached is not None: return cached diff --git a/runpod/rp_cli/main.py b/runpod/rp_cli/main.py index 9026d5380..bbcb72731 100644 --- a/runpod/rp_cli/main.py +++ b/runpod/rp_cli/main.py @@ -313,6 +313,7 @@ def dev(module, once): from runpod.apps.dev import DevSession from runpod.apps.discovery import DiscoveryError, discover_apps from runpod.apps.entrypoint import get_entrypoint, run_entrypoint + from runpod.apps.utils.names import dev_endpoint_name from runpod.apps.watch import FileWatcher from runpod.rp_cli import console as ui @@ -361,7 +362,7 @@ def _table_rows(session: "DevSession") -> list: spec = handle.spec hardware = ",".join(spec.cpu or spec.gpu or ["any"]) endpoint_id = session._endpoints.get( - f"dev-{a.name}-{spec.name}", "" + dev_endpoint_name(a.name, spec.name), "" ) or ("per-call" if spec.kind.value == "task" else "") rows.append((spec.name, spec.kind.value, hardware, endpoint_id)) return rows @@ -402,13 +403,10 @@ async def _wait_for_rerun(watcher: "FileWatcher") -> str: await asyncio.sleep(0.5) async def _run_entrypoint_cancellable(fn) -> None: - """drive the entrypoint on a daemon thread. + """run user code without blocking the event loop. - the entrypoint is user code full of blocking .remote() calls; - running it inline would pin the main loop and make ctrl-c - undeliverable (asyncio's sigint handler cancels the main task, - which needs an await point). a daemon thread keeps the loop - free, and cancellation simply abandons the in-flight call. + cancellation waits for the entrypoint to finish so its remote + calls cannot race with session teardown. """ loop = asyncio.get_running_loop() done: asyncio.Future[None] = loop.create_future() @@ -433,8 +431,12 @@ def _runner() -> None: else: loop.call_soon_threadsafe(_finish, None) - threading.Thread(target=_runner, daemon=True).start() - return await done + threading.Thread(target=_runner).start() + try: + return await asyncio.shield(done) + except asyncio.CancelledError: + await done + raise async def _session() -> int: nonlocal apps, entrypoint @@ -443,10 +445,9 @@ async def _session() -> int: watcher = FileWatcher([module.parent]) ui.dev_banner([a.name for a in apps], str(module.name)) - await session.start() - ui.resources_table(_table_rows(session)) - try: + await session.start() + ui.resources_table(_table_rows(session)) while True: ui.entrypoint_header(getattr(entrypoint, "__name__", "")) with ui.Timer() as t: @@ -469,8 +470,10 @@ async def _session() -> int: await session.refresh(apps) finally: events = ui.CleanupEvents() - await session.stop(events=events) - events.close() + try: + await session.stop(events=events) + finally: + events.close() try: sys.exit(asyncio.run(_session())) @@ -532,7 +535,8 @@ async def _run(): default=None, help="Store this API key directly (skips the browser flow).", ) -def login(no_open, api_key_opt): +@click.option("--profile", default="default", show_default=True, help="Credential profile.") +def login(no_open, api_key_opt, profile): """Authenticate with Runpod and store the API key. Opens the Runpod console for browser approval by default; pass @@ -544,7 +548,7 @@ def login(no_open, api_key_opt): if api_key_opt: try: - set_credentials(api_key_opt, overwrite=True) + set_credentials(api_key_opt, profile=profile, overwrite=True) except ValueError as exc: _fail(str(exc)) ui.success("credentials saved to [dim]~/.runpod/config.toml[/dim]") @@ -561,7 +565,7 @@ def _show_url(url: str) -> None: try: with ui.console.status("[dim]waiting for approval ...[/dim]"): api_key = asyncio.run(browser_login(on_url=_show_url)) - set_credentials(api_key, overwrite=True) + set_credentials(api_key, profile=profile, overwrite=True) except (LoginError, ValueError) as exc: _fail(str(exc)) ui.success("logged in, credentials saved to [dim]~/.runpod/config.toml[/dim]") diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py index 445d97b0a..3bdee6ab6 100644 --- a/tests/test_apps/test_api_client.py +++ b/tests/test_apps/test_api_client.py @@ -39,7 +39,7 @@ async def test_retries_transport_errors(self): patch("runpod.apps.api.run_graphql_query_async", transport), patch("asyncio.sleep", AsyncMock()), ): - assert await client._execute("query {}") == {"ok": 1} + assert await client._execute("query {}", retry=True) == {"ok": 1} assert transport.await_count == 3 async def test_exhausted_retries_raise(self): @@ -50,7 +50,7 @@ async def test_exhausted_retries_raise(self): patch("asyncio.sleep", AsyncMock()), ): with pytest.raises(aiohttp.ClientError): - await client._execute("query {}") + await client._execute("query {}", retry=True) assert transport.await_count == 4 async def test_graphql_errors_propagate_immediately(self): @@ -61,6 +61,18 @@ async def test_graphql_errors_propagate_immediately(self): await client._execute("query {}") assert transport.await_count == 1 + @pytest.mark.parametrize( + "failure", + [aiohttp.ClientError("response lost"), OSError("reset")], + ) + async def test_mutation_transport_failure_is_not_retried(self, failure): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock(side_effect=failure) + with patch("runpod.apps.api.run_graphql_query_async", transport): + with pytest.raises(type(failure)): + await client.create_app("demo") + assert transport.await_count == 1 + class TestEndpoints: async def test_save_endpoint(self): @@ -76,6 +88,11 @@ async def test_delete_endpoint(self): with patcher: assert await client.delete_endpoint("ep1") is True + async def test_delete_failure_is_not_reported_as_success(self): + client, patcher = _client_with({"deleteEndpoint": False}) + with patcher, pytest.raises(QueryError, match="ep1"): + await client.delete_endpoint("ep1") + async def test_list_my_endpoints(self): client, patcher = _client_with( {"myself": {"endpoints": [{"id": "ep1"}]}} @@ -96,17 +113,41 @@ async def test_deploy_gpu_pod(self): ) assert result["id"] == "pod1" - async def test_deploy_cpu_pod_converts_instance_ids(self): + async def test_cpu_alternative_can_deploy_in_volume_datacenter(self): client = AppsApiClient(api_key="test-key") - transport = _respond({"deployCpuPod": {"id": "pod2"}}) + transport = AsyncMock( + side_effect=[ + QueryError("insufficient capacity"), + {"data": {"deployCpuPod": {"id": "pod2"}}}, + ] + ) + pod = { + "instanceIds": ["cpu3c-2-4", "cpu3g-2-8"], + "dataCenterIds": ["US-CA-2"], + "networkVolumeId": "volume1", + } with patch("runpod.apps.api.run_graphql_query_async", transport): - result = await client.deploy_task_pod( - {"instanceIds": ["cpu3c-2-4", "cpu3g-2-8"]}, is_cpu=True - ) + result = await client.deploy_task_pod(pod, is_cpu=True) assert result["id"] == "pod2" - sent = transport.call_args[1]["variables"]["input"] - assert sent["instanceId"] == "cpu3c-2-4" - assert "instanceIds" not in sent + requests = [ + call.kwargs["variables"]["input"] for call in transport.await_args_list + ] + assert [request["instanceId"] for request in requests] == pod["instanceIds"] + assert all(request["dataCenterIds"] == ["US-CA-2"] for request in requests) + assert all(request["networkVolumeId"] == "volume1" for request in requests) + + @pytest.mark.parametrize( + "failure", [QueryError("invalid image"), aiohttp.ClientError("response lost")] + ) + async def test_cpu_alternatives_stop_after_ambiguous_or_invalid_failure(self, failure): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock(side_effect=failure) + with patch("runpod.apps.api.run_graphql_query_async", transport): + with pytest.raises(type(failure)): + await client.deploy_task_pod( + {"instanceIds": ["cpu3c-2-4", "cpu3g-2-8"]}, is_cpu=True + ) + assert transport.await_count == 1 async def test_terminate_pod(self): client = AppsApiClient(api_key="test-key") diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py index 803b30738..b8052b3a5 100644 --- a/tests/test_apps/test_app.py +++ b/tests/test_apps/test_app.py @@ -232,6 +232,46 @@ async def q(x): assert await q.local(1) == 2 +async def test_remote_rejects_mutated_invalid_spec(): + app = App("a") + + @app.queue(name="q") + def q(): + raise AssertionError("invalid specs must not execute") + + q.spec.workers = (9, 1) + with pytest.raises(InvalidResourceError): + await q.remote.aio() + + +@pytest.mark.parametrize("collection", [list, tuple]) +async def test_endpoint_volume_collections_reach_attachment(collection): + from unittest.mock import AsyncMock + + from runpod.apps.volume import Volume, VolumeResolver, attach_endpoint_volumes + + app = App("a") + + @app.api(name="web", cpu="cpu5c-2-4", volume=collection([Volume("models"), "data"])) + def web(): + return None + + api = AsyncMock() + api.list_network_volumes.return_value = [ + {"id": "nv-models", "name": "models", "dataCenterId": "EU-RO-1"}, + {"id": "nv-data", "name": "data", "dataCenterId": "EU-RO-1"}, + ] + api.cpu_stock_status.return_value = "HIGH" + payload = {} + await attach_endpoint_volumes(payload, web.spec, VolumeResolver(api), app) + assert payload["networkVolumeIds"] == [ + {"networkVolumeId": "nv-models"}, + {"networkVolumeId": "nv-data"}, + ] + assert payload["locations"] == "EU-RO-1" + assert web.spec.to_manifest()["networkVolumes"] == ["models", "data"] + + def test_manifest_serialization(): app = App("a") volume = "my-volume" diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index eebcf9588..9c1e8a8bf 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -1,5 +1,8 @@ """tests for deploy-time environment vendoring.""" +import subprocess +import sys +import threading from unittest.mock import patch import pytest @@ -159,8 +162,7 @@ def fake(cmd, **kwargs): class P: def __init__(self): - self.stdout = io.StringIO(stdout) - self.stderr = io.StringIO(stderr) + self.stdout = io.StringIO(stdout + stderr) self.returncode = returncode def wait(self): @@ -220,3 +222,49 @@ def test_progress_reports_collected_packages(self, tmp_path): progress=lambda c, n: seen.append((c, n)), ) assert seen == [(1, "numpy"), (2, "requests")] + + def test_large_stderr_does_not_block_progress_or_error(self, tmp_path): + original_popen = subprocess.Popen + processes = [] + expired = threading.Event() + + def launch(_cmd, **kwargs): + process = original_popen( + [ + sys.executable, + "-c", + "import sys; " + "sys.stderr.write('x' * 262144 + '\\n'); " + "sys.stderr.flush(); " + "print('Collecting numpy'); " + "print('resolver failed', file=sys.stderr); " + "sys.exit(1)", + ], + **kwargs, + ) + processes.append(process) + return process + + def stop_blocked_process(): + expired.set() + for process in processes: + process.kill() + + timer = threading.Timer(10, stop_blocked_process) + seen = [] + timer.start() + try: + with patch("runpod.apps.build.subprocess.Popen", side_effect=launch): + with pytest.raises(BuildError, match="resolver failed"): + vendor( + tmp_path, + ["numpy"], + "3.12", + progress=lambda count, name: seen.append((count, name)), + ) + finally: + timer.cancel() + for process in processes: + process.wait() + assert not expired.is_set() + assert seen == [(1, "numpy")] diff --git a/tests/test_apps/test_config_params.py b/tests/test_apps/test_config_params.py index 214a40c69..48d087f1c 100644 --- a/tests/test_apps/test_config_params.py +++ b/tests/test_apps/test_config_params.py @@ -46,6 +46,43 @@ def test_cuda_version_rejects_unknown(self): class TestSpecValidation: + @pytest.mark.parametrize( + "config", + [ + {"gpu_count": 0}, + {"gpu_count": -2}, + {"workers": (9, 1)}, + {"scaler_type": "BOGUS"}, + {"min_cuda_version": "99"}, + ], + ) + def test_direct_construction_rejects_invalid_configuration(self, config): + with pytest.raises(InvalidResourceError): + ResourceSpec(kind=ResourceKind.QUEUE, name="q", **config) + + def test_mutable_spec_revalidates_before_serialization(self): + spec = ResourceSpec(kind=ResourceKind.QUEUE, name="q") + spec.gpu_count = 0 + with pytest.raises(InvalidResourceError): + spec.to_manifest() + spec.gpu_count = 2 + spec.workers = 4 + spec.scaler_type = "request_count" + spec.gpu = "4090" + manifest = spec.to_manifest() + assert manifest["gpuCount"] == 2 + assert manifest["workersMax"] == 4 + assert manifest["scalerType"] == "REQUEST_COUNT" + assert manifest["gpus"] == ["NVIDIA GeForce RTX 4090"] + + def test_mutated_routes_reject_reserved_runtime_path(self): + from runpod.apps.spec import RouteSpec + + spec = ResourceSpec(kind=ResourceKind.API, name="web") + spec.routes.append(RouteSpec("POST", "/_runpod/sync", "sync")) + with pytest.raises(InvalidResourceError): + spec.to_manifest() + def test_max_concurrency_floor(self): with pytest.raises(InvalidResourceError): ResourceSpec( @@ -90,70 +127,7 @@ def test_effective_scaler_type_explicit_wins(self): assert spec.effective_scaler_type == "REQUEST_COUNT" -class TestDecoratorPlumbing: - def test_queue_accepts_all_params(self): - app = App("a") - - @app.queue( - name="q", - gpu="4090", - max_concurrency=4, - execution_timeout_ms=60000, - flashboot=False, - scaler_type="request_count", - scaler_value=8, - min_cuda_version="12.4", - accelerate_downloads=False, - container_disk_gb=50, - ) - def q(): - pass - - spec = q.spec - assert spec.max_concurrency == 4 - assert spec.execution_timeout_ms == 60000 - assert spec.flashboot is False - assert spec.scaler_type == "REQUEST_COUNT" - assert spec.scaler_value == 8 - assert spec.min_cuda_version == "12.4" - assert spec.accelerate_downloads is False - assert spec.container_disk_gb == 50 - - def test_api_accepts_params(self): - app = App("a") - - @app.api( - name="api", - cpu="cpu3c-1-2", - execution_timeout_ms=30000, - scaler_value=2, - container_disk_gb=20, - ) - class Api: - @runpod.post("/x") - def x(self, body: dict): - return body - - assert Api.spec.execution_timeout_ms == 30000 - assert Api.spec.scaler_value == 2 - assert Api.spec.container_disk_gb == 20 - - def test_task_accepts_params(self): - app = App("a") - - @app.task( - name="t", - gpu="4090", - min_cuda_version="12.8", - accelerate_downloads=False, - container_disk_gb=100, - ) - def t(): - pass - - assert t.spec.min_cuda_version == "12.8" - assert t.spec.accelerate_downloads is False - assert t.spec.container_disk_gb == 100 +class TestDecoratorValidation: def test_queue_invalid_scaler_fails_at_decoration(self): app = App("a") diff --git a/tests/test_apps/test_deploy.py b/tests/test_apps/test_deploy.py index 4de46f4d6..5af21d320 100644 --- a/tests/test_apps/test_deploy.py +++ b/tests/test_apps/test_deploy.py @@ -11,6 +11,7 @@ import runpod from runpod.apps import App from runpod.apps.app import _clear_registry +from runpod.apps.context import Context from runpod.apps.deploy import ( DeployResult, _deployed_endpoint_input, @@ -19,7 +20,7 @@ package_project, ) from runpod.apps.discovery import DiscoveryError, discover_apps -from runpod.apps.errors import ScheduleNotSupported +from runpod.apps.errors import InvalidResourceError, ScheduleNotSupported @pytest.fixture(autouse=True) @@ -240,6 +241,44 @@ async def test_deploy_reuses_existing_app_and_env(self, tmp_path): api.create_app.assert_not_awaited() api.create_environment.assert_not_awaited() + async def test_deploy_environment_is_used_for_nested_calls(self, tmp_path, monkeypatch): + _write_project(tmp_path) + (app,) = discover_apps(tmp_path) + handle = next(iter(app.resources.values())) + handle.spec.env = {"FLASH_ENVIRONMENT": "stale"} + api = AsyncMock() + api.get_app_by_name.return_value = { + "id": "app-1", + "flashEnvironments": [{"id": "env-prod", "name": "prod"}], + } + api.prepare_artifact_upload.return_value = { + "uploadUrl": "https://upload", "objectKey": "key-1", + } + api.finalize_artifact_upload.return_value = {"id": "build-1"} + api.save_endpoint.return_value = {"id": "ep-1"} + with _stub_build(tmp_path): + await deploy_app(app, tmp_path, env_name="prod", api=api) + payload = api.save_endpoint.await_args.args[0] + worker_env = { + entry["key"]: entry["value"] for entry in payload["template"]["env"] + } + monkeypatch.setenv("FLASH_ENVIRONMENT", worker_env["FLASH_ENVIRONMENT"]) + monkeypatch.setattr(runpod, "api_key", "test-key") + monkeypatch.delenv("RUNPOD_DEV_APP", raising=False) + with patch("runpod.apps.app.current_context", return_value=Context.WORKER): + target = await app._resolve(handle.spec) + assert target._sentinel_headers()["X-Flash-Environment"] == "prod" + + async def test_invalid_mutated_spec_fails_before_remote_deployment(self, tmp_path): + _write_project(tmp_path) + (app,) = discover_apps(tmp_path) + next(iter(app.resources.values())).spec.workers = (5, 1) + api = AsyncMock() + with pytest.raises(InvalidResourceError): + await deploy_app(app, tmp_path, api=api) + api.create_app.assert_not_awaited() + api.save_endpoint.assert_not_awaited() + class TestTolerantDiscovery: def test_broken_bystander_warns_but_discovers(self, tmp_path): diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index 4fb593e63..cf38d8bf2 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -8,6 +8,7 @@ from runpod.apps import App from runpod.apps.app import _clear_registry from runpod.apps.dev import DevSession, _endpoint_input, dev_endpoint_name +from runpod.apps.errors import AppError from runpod.apps.targets import LiveTarget @@ -180,6 +181,23 @@ def q(): "template" ]["env"] + async def test_worker_self_call_stays_local(self, monkeypatch): + app = App("self-call") + + @app.queue(name="q-basic", cpu="cpu3c-1-2") + def q(value): + return value + 1 + + payload = _endpoint_input(app, q.spec) + for entry in payload["template"]["env"]: + monkeypatch.setenv(entry["key"], entry["value"]) + monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-self") + monkeypatch.setattr( + app, "_resolve", AsyncMock(side_effect=AssertionError("remote self-call")) + ) + + assert await q.remote.aio(4) == 5 + class TestDevSession: async def test_start_provisions_and_registers_targets(self): @@ -246,6 +264,62 @@ def t(): api.save_endpoint.assert_not_awaited() + async def test_stop_retains_failed_ids_and_retries_only_failures(self): + app = App("cleanup") + for name in ("first-failed", "deleted", "second-failed"): + app.queue(name=name, cpu="cpu3c-1-2")(lambda: None) + api = _mock_api() + api.save_endpoint.side_effect = [ + {"id": "ep-first"}, {"id": "ep-deleted"}, {"id": "ep-second"} + ] + session = DevSession([app], api=api) + await session.start() + api.delete_endpoint.side_effect = [ + RuntimeError("unauthorized"), True, RuntimeError("unavailable") + ] + + with pytest.raises(AppError) as error: + await session.stop() + assert "ep-first" in str(error.value) + assert "ep-second" in str(error.value) + assert set(session._endpoint_ids) == {"ep-first", "ep-second"} + + api.delete_endpoint.reset_mock(side_effect=True) + await session.stop() + assert {call.args[0] for call in api.delete_endpoint.await_args_list} == { + "ep-first", "ep-second" + } + assert session._endpoint_ids == [] + + async def test_context_entry_failure_cleans_already_created_endpoints(self): + app = App("partial") + for name in ("created", "fails"): + app.queue(name=name, cpu="cpu3c-1-2")(lambda: None) + api = _mock_api() + api.save_endpoint.side_effect = [ + {"id": "ep-created"}, RuntimeError("capacity") + ] + + with pytest.raises(RuntimeError, match="capacity"): + async with DevSession([app], api=api): + pytest.fail("startup must fail") + api.delete_endpoint.assert_awaited_once_with("ep-created") + + async def test_failed_adoption_remains_tracked_for_cleanup(self): + app = App("adopt") + app.queue(name="q-basic", cpu="cpu3c-1-2")(lambda: None) + api = _mock_api() + api.list_my_endpoints.return_value = [ + {"id": "ep-adopted", "name": dev_endpoint_name("adopt", "q-basic")} + ] + api.save_endpoint.side_effect = RuntimeError("update failed") + session = DevSession([app], api=api) + + with pytest.raises(RuntimeError, match="update failed"): + async with session: + pytest.fail("startup must fail") + api.delete_endpoint.assert_awaited_once_with("ep-adopted") + class TestDevRefresh: async def test_refresh_bumps_generation_and_updates(self): @@ -330,6 +404,25 @@ def q2(): api.delete_endpoint.assert_awaited_once_with("ep-gone") + async def test_failed_removals_remain_tracked_for_stop(self): + app = App("remove") + for name in ("q-basic", "q-other"): + app.queue(name=name, cpu="cpu3c-1-2")(lambda: None) + api = _mock_api() + api.save_endpoint.side_effect = [{"id": "ep-basic"}, {"id": "ep-other"}] + session = DevSession([app], api=api) + await session.start() + api.delete_endpoint.side_effect = [RuntimeError("failed"), True] + + with pytest.raises(AppError, match="ep-basic"): + await session.refresh([]) + assert session._endpoint_ids == ["ep-basic"] + + api.delete_endpoint.reset_mock(side_effect=True) + await session.stop() + api.delete_endpoint.assert_awaited_once_with("ep-basic") + assert session._endpoint_ids == [] + class TestDevEvents: async def test_lifecycle_events_emitted(self): @@ -359,7 +452,7 @@ def deleted(self, name): app = App("ev-app") - @app.queue(name="q", cpu="cpu3c-1-2") + @app.queue(name="q-basic", cpu="cpu3c-1-2") def q(): pass @@ -380,9 +473,9 @@ def q(): # unchanged resource refreshes silently: no diff events assert "resource_changed" not in kinds assert "resource_added" not in kinds - assert ("provisioning", "q", "queue", "cpu3c-1-2") in events + assert ("provisioning", "q-basic", "queue", "cpu3c-1-2") in events # deleted reports the resource name, not the endpoint name - assert ("deleted", "q") in events + assert ("deleted", "q-basic") in events async def test_refresh_diff_events(self): events = [] @@ -403,7 +496,7 @@ def resource_removed(self, name): def stays(): pass - @app.queue(name="goes", cpu="cpu3c-1-2") + @app.queue(name="goes-away", cpu="cpu3c-1-2") def goes(): pass @@ -428,7 +521,7 @@ def fresh(): await session.refresh([app2]) - assert ("removed", "goes") in events + assert ("removed", "goes-away") in events added = [e for e in events if e[0] == "added"] assert ("added", "fresh", "queue", "cpu5c-2-4") in added changed = [e for e in events if e[0] == "changed"] diff --git a/tests/test_apps/test_discovery.py b/tests/test_apps/test_discovery.py new file mode 100644 index 000000000..f70783d94 --- /dev/null +++ b/tests/test_apps/test_discovery.py @@ -0,0 +1,119 @@ +import importlib +import os +import sys +from pathlib import Path + +import pytest + +from runpod.apps import App, get_registered_apps +from runpod.apps.app import _clear_registry +from runpod.apps.discovery import DiscoveryError, discover_apps +from runpod.apps.discovery_state import DISCOVERY_ENV + + +@pytest.fixture(autouse=True) +def clean_discovery_state(tmp_path): + _clear_registry() + yield + _clear_registry() + for name, module in list(sys.modules.items()): + source = getattr(module, "__file__", None) + if source and Path(source).resolve().is_relative_to(tmp_path): + sys.modules.pop(name, None) + + +def test_failed_module_does_not_return_partial_app(tmp_path): + existing = App("existing") + (tmp_path / "broken.py").write_text( + "from runpod import App\n" + "app = App('partial')\n" + "raise RuntimeError('incomplete initialization')\n" + ) + (tmp_path / "healthy.py").write_text( + "from runpod import App\napp = App('healthy')\n" + ) + + apps = discover_apps(tmp_path) + + assert [app.name for app in apps] == ["healthy"] + assert get_registered_apps() == [existing, apps[0]] + + +def test_failed_package_import_can_be_retried_without_orphaned_app(tmp_path): + package = tmp_path / "discovery_retry_package" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "helper.py").write_text( + "from runpod import App\napp = App('recovered')\n" + ) + target = package / "main.py" + target.write_text("from .helper import app\nraise RuntimeError('broken')\n") + existing = App("existing") + + with pytest.raises(DiscoveryError): + discover_apps(target) + assert get_registered_apps() == [existing] + + target.write_text("from .helper import app\n") + apps = discover_apps(target) + + assert [app.name for app in apps] == ["recovered"] + assert importlib.import_module("discovery_retry_package.helper").app is apps[0] + assert get_registered_apps() == [existing, apps[0]] + + +@pytest.mark.parametrize("single_file", [False, True]) +def test_package_relative_imports_preserve_local_handle_identity(tmp_path, single_file): + package = tmp_path / "discovery_identity_package" + package.mkdir() + (package / "__init__.py").write_text("from .main import app, double\n") + (package / "helpers.py").write_text("FACTOR = 2\n") + target = package / "main.py" + target.write_text( + "from runpod import App\n" + "from .helpers import FACTOR\n" + "app = App('package')\n" + "@app.queue(cpu='cpu3c-1-2')\n" + "def double(value):\n" + " return value * FACTOR\n" + ) + + apps = discover_apps(target if single_file else tmp_path) + module = importlib.import_module("discovery_identity_package.main") + + assert apps == [module.app] + assert get_registered_apps() == apps + assert apps[0].resources["double"] is module.double + assert module.double.local(3) == 6 + + +def test_discovery_restores_existing_invocation_guard(tmp_path, monkeypatch): + target = tmp_path / "main.py" + target.write_text("raise RuntimeError('broken')\n") + monkeypatch.setenv(DISCOVERY_ENV, "outer-scan") + + with pytest.raises(DiscoveryError): + discover_apps(target) + + assert os.environ[DISCOVERY_ENV] == "outer-scan" + + +def test_interrupt_aborts_discovery_and_rolls_back_partial_import(tmp_path, monkeypatch): + monkeypatch.delenv(DISCOVERY_ENV, raising=False) + existing = App("existing") + (tmp_path / "a_interrupted.py").write_text( + "from runpod import App\n" + "app = App('partial')\n" + "raise KeyboardInterrupt\n" + ) + (tmp_path / "b_later.py").write_text( + "from pathlib import Path\n" + "Path(__file__).with_suffix('.started').touch()\n" + ) + + with pytest.raises(KeyboardInterrupt): + discover_apps(tmp_path) + + assert get_registered_apps() == [existing] + assert not (tmp_path / "b_later.started").exists() + assert DISCOVERY_ENV not in os.environ diff --git a/tests/test_apps/test_markers.py b/tests/test_apps/test_markers.py new file mode 100644 index 000000000..6aeafb2ea --- /dev/null +++ b/tests/test_apps/test_markers.py @@ -0,0 +1,10 @@ +import pytest + +from runpod.apps.markers import delete, get, patch, post, put + + +@pytest.mark.parametrize("marker", [get, post, put, delete, patch]) +@pytest.mark.parametrize("path", ["/ping", "/execute", "/_runpod/sync"]) +def test_runtime_control_paths_are_rejected(marker, path): + with pytest.raises(ValueError): + marker(path) diff --git a/tests/test_apps/test_placement.py b/tests/test_apps/test_placement.py index 9c55c2101..8716c6a02 100644 --- a/tests/test_apps/test_placement.py +++ b/tests/test_apps/test_placement.py @@ -1,10 +1,14 @@ """placement solve: candidates, intersections, maximin ranking.""" import pytest +from unittest.mock import AsyncMock + +from runpod.apps.datacenter import DataCenter from runpod.apps.placement import ( PlacementError, StockMap, + _hardware_keys, candidates, solve_placement, ) @@ -25,8 +29,8 @@ def _stock(gpu=None, cpu=None): stock = StockMap(api=object()) # task specs query the pod plane; mirror into both tables so the # fixtures stay hardware-shaped rather than plane-shaped - stock._gpu = dict(gpu or {}) - stock._gpu_pod = dict(gpu or {}) + stock._gpu = {(gpu_id, 1, dc): score for (gpu_id, dc), score in (gpu or {}).items()} + stock._gpu_pod = dict(stock._gpu) stock._cpu = cpu or {} return stock @@ -61,6 +65,22 @@ def test_datacenter_pin_intersects(self): ) assert dcs == {"US-KS-2"} + @pytest.mark.parametrize( + "pin", + [DataCenter.EU_RO_1, [DataCenter.EU_RO_1], " eu_ro_1 ", ["eu-ro-1"]], + ) + def test_datacenter_variants_match_api_ids(self, pin): + spec = _spec(cpu="cpu5c-2-4", datacenter=pin) + stock = _stock(cpu={("cpu5c-2-4", "EU-RO-1"): 2}) + assert candidates(spec, stock) == {"EU-RO-1"} + assert spec.to_manifest()["locations"] == "EU-RO-1" + + def test_mutated_datacenter_pin_is_normalized(self): + spec = _spec(cpu="cpu5c-2-4") + spec.datacenter = [DataCenter.EU_RO_1] + stock = _stock(cpu={("cpu5c-2-4", "EU-RO-1"): 2}) + assert candidates(spec, stock) == {"EU-RO-1"} + def test_cpu_stock(self): stock = _stock(cpu={("cpu5c-2-4", "EU-RO-1"): 2}) dcs = candidates(_spec(cpu=["cpu5c-2-4"]), stock) @@ -166,3 +186,27 @@ def test_mixed_cpu_gpu_sharing(self): volume_name="shared", ) assert dc == "EU-RO-1" + + @pytest.mark.parametrize("kind", [ResourceKind.QUEUE, ResourceKind.TASK]) + async def test_shared_volume_requires_requested_gpu_count(self, kind): + async def gpu_stock(gpu_id, dc, gpu_count=1, pods=False): + if dc == "EU-RO-1" and gpu_count == 1: + return "HIGH" + if dc == "US-KS-2": + return "MEDIUM" + return None + + api = AsyncMock() + api.gpu_stock_status.side_effect = gpu_stock + stock = StockMap(api) + single = ResourceSpec(kind=kind, name="single", gpu="4090") + pair = ResourceSpec(kind=kind, name="pair", gpu="4090", gpu_count=2) + await stock.fetch(_hardware_keys(single)) + await stock.fetch(_hardware_keys(pair)) + + assert solve_placement([single], stock, volume_name="one") == "EU-RO-1" + assert solve_placement([single, pair], stock, volume_name="shared") == "US-KS-2" + with pytest.raises(PlacementError): + solve_placement( + [pair], stock, volume_name="fixed", existing_dc="EU-RO-1" + ) diff --git a/tests/test_apps/test_retry.py b/tests/test_apps/test_retry.py index f2a3ea549..6a7f159a1 100644 --- a/tests/test_apps/test_retry.py +++ b/tests/test_apps/test_retry.py @@ -1,6 +1,7 @@ """tests for transient-failure retry in the http layer.""" import json +import socket import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from unittest.mock import patch @@ -12,7 +13,7 @@ from runpod.apps.targets import ( JOB_PROPAGATION_ATTEMPTS, RETRY_ATTEMPTS, - RETRYABLE_STATUSES, + QueueClient, _post_json, _request_json, ) @@ -26,6 +27,10 @@ def flaky_server(): class Handler(BaseHTTPRequestHandler): def _respond(self): state["hits"] += 1 + if state.get("disconnect"): + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return if state["hits"] <= len(state["script"]): status = state["script"][state["hits"] - 1] self.send_response(status) @@ -51,12 +56,13 @@ def log_message(self, *args): server.url = f"http://127.0.0.1:{server.server_address[1]}/run" yield server server.shutdown() + server.server_close() async def test_retries_transient_5xx_then_succeeds(flaky_server): flaky_server.state["script"] = [520, 502] with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): - data = await _post_json(flaky_server.url, {"x": 1}, {}, timeout=10) + data = await _request_json("GET", flaky_server.url, {}, timeout=10) assert data["ok"] is True assert data["hits"] == 3 @@ -65,7 +71,7 @@ async def test_exhausted_retries_raise_last_error(flaky_server): flaky_server.state["script"] = [503] * (RETRY_ATTEMPTS + 2) with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): with pytest.raises(aiohttp.ClientResponseError) as exc_info: - await _post_json(flaky_server.url, {"x": 1}, {}, timeout=10) + await _request_json("GET", flaky_server.url, {}, timeout=10) assert exc_info.value.status == 503 assert flaky_server.state["hits"] == RETRY_ATTEMPTS @@ -120,13 +126,32 @@ async def test_client_4xx_not_retried(flaky_server): assert flaky_server.state["hits"] == 1 -def test_429_is_retryable(): - assert 429 in RETRYABLE_STATUSES - - async def test_connection_error_retried(unused_tcp_port): # nothing listening: pure connection failures, all attempts consumed url = f"http://127.0.0.1:{unused_tcp_port}/run" with patch("runpod.apps.targets.RETRY_BASE_DELAY", 0.01): with pytest.raises(aiohttp.ClientConnectionError): - await _post_json(url, {"x": 1}, {}, timeout=5) + await _request_json("GET", url, {}, timeout=5) + + +@pytest.mark.parametrize("operation", ["run", "runsync", "retry"]) +@pytest.mark.parametrize("failure", [503, "disconnect"]) +async def test_submission_is_not_repeated_after_ambiguous_failure( + flaky_server, monkeypatch, operation, failure +): + import runpod + + monkeypatch.setattr(runpod, "endpoint_url_base", flaky_server.url) + if failure == "disconnect": + flaky_server.state["disconnect"] = True + else: + flaky_server.state["script"] = [failure] + client = QueueClient("endpoint", lambda: {}) + with pytest.raises(aiohttp.ClientError): + if operation == "runsync": + await client.runsync({"input": {}}, timeout=10) + elif operation == "retry": + await client.retry("job") + else: + await client.run({"input": {}}) + assert flaky_server.state["hits"] == 1 diff --git a/tests/test_apps/test_shim.py b/tests/test_apps/test_shim.py index e961befd3..78fe667d2 100644 --- a/tests/test_apps/test_shim.py +++ b/tests/test_apps/test_shim.py @@ -9,23 +9,13 @@ from runpod.apps.shim import runtime_launcher -def test_no_inner_single_quotes(): - command = runtime_launcher("queue") - assert command.startswith("sh -c '") - assert command.endswith("'") - assert "'" not in command[len("sh -c '") : -1] - - -def test_posix_sh_not_bash(): - assert runtime_launcher("queue").startswith("sh -c ") - - def test_starts_an_installed_runtime(tmp_path): marker = tmp_path / "marker" package = tmp_path / "runpod_sdk_runtime" / "task" package.mkdir(parents=True) (package.parent / "__init__.py").write_text("") (package / "__init__.py").write_text("") + (tmp_path / "runpod.py").write_text("") (package / "runner.py").write_text( f"from pathlib import Path\nPath({str(marker)!r}).write_text('ran')\n" ) @@ -48,26 +38,70 @@ def test_starts_an_installed_runtime(tmp_path): assert marker.read_text() == "ran" -def test_probes_beyond_path(): - command = runtime_launcher("api") - assert "/opt/conda/bin/python" in command - assert "/opt/venv/bin/python" in command - - -def test_pythonless_image_fails_loudly(): - command = runtime_launcher("queue") - assert "FATAL" in command - assert "must include python3" in command - - -def test_supports_runtime_and_sdk_package_overrides(): - command = runtime_launcher("queue") - assert "RUNPOD_RUNTIME_PACKAGE_SPEC" in command - assert "RUNPOD_PACKAGE_SPEC" in command - assert "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_RUNPOD" in command - assert "runpod-sdk-runtime" in command - def test_rejects_unknown_runtime_kind(): with pytest.raises(ValueError, match="unknown runtime kind"): runtime_launcher("other") + + +@pytest.mark.parametrize( + ("installed", "runtime_override", "sdk_override", "expected_installs"), + [ + (False, "", "", ["runpod-sdk-runtime", "runpod"]), + (True, "", "", []), + (True, "runpod-sdk-runtime==2.1", "runpod==3.2", [ + "runpod-sdk-runtime==2.1", "runpod==3.2" + ]), + ], +) +def test_installs_missing_packages_and_honors_overrides( + tmp_path, installed, runtime_override, sdk_override, expected_installs +): + import json + import shlex + + state_path = tmp_path / "state.json" + state_path.write_text(json.dumps({ + "runtime": installed, + "sdk": installed, + "installs": [], + })) + interpreter = tmp_path / "python3" + interpreter.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "path = Path(os.environ['SHIM_STATE'])\n" + "state = json.loads(path.read_text())\n" + "args = sys.argv[1:]\n" + "if args[0] == '-c':\n" + " key = 'runtime' if args[1] == 'import runpod_sdk_runtime' else 'sdk'\n" + " sys.exit(0 if state[key] else 1)\n" + "if args[:2] == ['-m', 'pip']:\n" + " spec = args[-1]\n" + " state['installs'].append(spec)\n" + " state['runtime' if spec.startswith('runpod-sdk-runtime') else 'sdk'] = True\n" + "else:\n" + " if not (state['runtime'] and state['sdk']):\n" + " sys.exit(2)\n" + " state['started'] = args[1]\n" + "path.write_text(json.dumps(state))\n" + ) + interpreter.chmod(0o755) + result = subprocess.run( + shlex.split(runtime_launcher("queue")), + env={ + "PATH": f"{tmp_path}:/usr/bin:/bin", + "SHIM_STATE": str(state_path), + "RUNPOD_RUNTIME_PACKAGE_SPEC": runtime_override, + "RUNPOD_PACKAGE_SPEC": sdk_override, + }, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == 0, result.stderr + state = json.loads(state_path.read_text()) + assert state["installs"] == expected_installs + assert state["started"] == "runpod_sdk_runtime.bootstrap" diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py index b660c83e7..9f4753a17 100644 --- a/tests/test_apps/test_targets.py +++ b/tests/test_apps/test_targets.py @@ -1,5 +1,6 @@ """unit tests for invocation targets and their helpers.""" +import asyncio import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -78,6 +79,13 @@ def fn(a, b): assert args_to_input(fn, (1,), {"b": 2}) == {"a": 1, "b": 2} + def test_duplicate_argument_rejected(self): + def fn(a): + return a + + with pytest.raises(TypeError, match="multiple values.*a"): + args_to_input(fn, (1,), {"a": 2}) + def test_too_many_positional(self): def fn(a): pass @@ -108,6 +116,23 @@ def test_error_in_output(self): {"status": "COMPLETED", "output": {"error": "oops"}} ) + def test_failed_generator_preserves_error(self): + with pytest.raises(RemoteExecutionError, match="generator broke"): + unwrap_job_output( + { + "status": "FAILED", + "output": [ + {"success": True, "json_result": 1}, + {"success": False, "error": "generator broke"}, + ], + } + ) + + @pytest.mark.parametrize("output", [None, [], "partial", 3]) + def test_failed_non_mapping_output(self, output): + with pytest.raises(RemoteExecutionError): + unwrap_job_output({"status": "FAILED", "output": output}) + def test_missing_output_returns_data(self): data = {"status": "COMPLETED", "value": 7} assert unwrap_job_output(data) == data @@ -142,10 +167,11 @@ async def test_polls_to_completion(self): assert result["status"] == "COMPLETED" assert len(seen) == 3 - async def test_no_job_id_returns_data(self): - data = {"status": "IN_QUEUE"} - result = await _wait_terminal("http://x", data, {}, timeout=5) - assert result is data + async def test_no_job_id_raises(self): + with pytest.raises(RemoteExecutionError, match="missing a job id"): + await _wait_terminal( + "http://x", {"status": "IN_QUEUE"}, {}, timeout=5 + ) async def test_timeout(self): with ( @@ -304,6 +330,63 @@ def fn(prompt): assert payload == {"input": {"prompt": "hi"}} +@pytest.fixture +async def live_api_server(monkeypatch): + import aiohttp + from aiohttp import web + from yarl import URL + + state = {"uploads": 0, "calls": 0, "reject_sync": False, "handler": None} + entered = asyncio.Event() + release = asyncio.Event() + + async def sync(request): + state["uploads"] += 1 + entered.set() + await release.wait() + if state["reject_sync"]: + state["reject_sync"] = False + raise web.HTTPServiceUnavailable() + payload = await request.json() + namespace = {} + exec(payload["source"], namespace) + state["handler"] = namespace["calculate"] + return web.json_response({"status": "synced"}) + + async def calculate(request): + state["calls"] += 1 + if state["handler"] is None: + raise web.HTTPConflict() + body = await request.json() + return web.json_response({"result": state["handler"](body["value"])}) + + app = web.Application() + app.router.add_post("/_runpod/sync", sync) + app.router.add_post("/calculate", calculate) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + original_request = aiohttp.ClientSession._request + + async def local_request(session, method, url, **kwargs): + local_url = URL(url).with_scheme("http").with_host("127.0.0.1").with_port(port) + return await original_request(session, method, local_url, **kwargs) + + monkeypatch.setattr(aiohttp.ClientSession, "_request", local_request) + monkeypatch.setenv("RUNPOD_API_KEY", "test-key") + monkeypatch.setattr( + "runpod.apps.serialization.get_function_source", + lambda target: "def calculate(value):\n return value * 2\n", + ) + try: + yield state, entered, release + finally: + release.set() + await runner.cleanup() + + class TestLiveTarget: def _spec(self): return ResourceSpec(kind=ResourceKind.QUEUE, name="chat") @@ -416,22 +499,30 @@ async def test_job_operations(self, local_endpoint): assert (await target.cancel_job("j1"))["status"] == "CANCELLED" assert (await target.retry_job("j1"))["status"] == "IN_QUEUE" - async def test_sync_source_skips_unchanged(self, monkeypatch): - target = LiveTarget("ep123", "chat") - - def backing(): - return 1 - - target.attach_source(backing, "chat", self._spec()) - post = AsyncMock(return_value={}) - with patch("runpod.apps.targets._post_json", post): - await target._sync_source(timeout=10) - await target._sync_source(timeout=10) - assert post.await_count == 1 - - async def test_sync_source_noop_without_attachment(self): - target = LiveTarget("ep123", "chat") - post = AsyncMock() - with patch("runpod.apps.targets._post_json", post): - await target._sync_source(timeout=10) - post.assert_not_awaited() + async def test_concurrent_requests_share_completed_source_sync(self, live_api_server): + state, entered, release = live_api_server + target = LiveTarget("ep123", "calculate") + target.attach_source(self._spec, "calculate", self._spec()) + first = asyncio.create_task(target.request("POST", "/calculate", {"value": 3})) + await entered.wait() + second = asyncio.create_task(target.request("POST", "/calculate", {"value": 5})) + await asyncio.sleep(0) + assert state["calls"] == 0 + release.set() + assert await asyncio.gather(first, second) == [{"result": 6}, {"result": 10}] + assert state["uploads"] == 1 + + async def test_failed_source_sync_prevents_execution_and_can_retry(self, live_api_server): + import aiohttp + + state, _, release = live_api_server + state["reject_sync"] = True + release.set() + target = LiveTarget("ep123", "calculate") + target.attach_source(self._spec, "calculate", self._spec()) + with pytest.raises(aiohttp.ClientResponseError): + await target.request("POST", "/calculate", {"value": 7}) + assert state["calls"] == 0 + assert await target.request("POST", "/calculate", {"value": 7}) == {"result": 14} + assert state["uploads"] == 2 + assert state["calls"] == 1 diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 55ef3eac8..3e569186b 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -1,5 +1,6 @@ """tests for the task runner and pod task transport.""" +import asyncio import base64 import json from unittest.mock import AsyncMock, MagicMock, patch @@ -94,6 +95,12 @@ def test_datacenter_forwarded(self): ) assert pod["dataCenterIds"] == ["EU-RO-1"] + def test_cpu5_uses_supported_datacenters(self): + from runpod.apps.datacenter import CPU5_DATACENTERS + + pod = _pod_input(self._spec(cpu=["cpu5c-1-2"]), "tok", "t") + assert pod["dataCenterIds"] == [dc.value for dc in CPU5_DATACENTERS] + class TestUnwrapTaskResponse: def test_success_cloudpickle(self): @@ -363,6 +370,41 @@ async def test_start_does_not_retry_non_capacity_error(self): await execution.start() api.deploy_task_pod.assert_awaited_once() + async def test_shared_volume_placement_accounts_for_siblings(self): + from runpod.apps.tasks import TaskExecution + from runpod.apps.volume import Volume + + volume = Volume("shared") + spec = self._spec( + volume=volume, datacenter=["US-IL-1", "EU-RO-1"] + ) + sibling = ResourceSpec( + kind=ResourceKind.QUEUE, + name="sibling", + cpu=["cpu3c-1-2"], + volume=volume, + datacenter=["EU-RO-1"], + ) + unrelated = ResourceSpec( + kind=ResourceKind.QUEUE, + name="unrelated", + cpu=["cpu3c-1-2"], + volume=Volume("other"), + datacenter=["US-IL-1"], + ) + api = AsyncMock() + api.list_network_volumes.return_value = [] + api.cpu_stock_status.side_effect = ( + lambda instance, dc: "High" if dc == "US-IL-1" else "Low" + ) + api.create_network_volume.return_value = {"id": "volume-1"} + api.deploy_task_pod.return_value = {"id": "pod-9"} + execution = TaskExecution(spec, api=api, specs=[spec, sibling, unrelated]) + await execution.start() + assert api.create_network_volume.call_args.kwargs["data_center_id"] == "EU-RO-1" + assert api.deploy_task_pod.call_args.args[0]["dataCenterIds"] == ["EU-RO-1"] + + async def test_start_resolves_registry_auth(self): from runpod.apps.tasks import TaskExecution @@ -379,14 +421,17 @@ async def test_start_resolves_registry_auth(self): pod = api.deploy_task_pod.call_args[0][0] assert pod["containerRegistryAuthId"] == "auth-1" - async def test_terminate_swallows_api_errors(self): + async def test_failed_termination_retains_id_for_retry(self): from runpod.apps.tasks import TaskExecution api = MagicMock() - api.terminate_pod = AsyncMock(side_effect=RuntimeError("api down")) + api.terminate_pod = AsyncMock(side_effect=[RuntimeError("api down"), None]) execution = TaskExecution(self._spec(), api=api) execution.pod_id = "pod-9" - await execution.terminate() # must not raise + with pytest.raises(RuntimeError, match="api down"): + await execution.terminate() + assert execution.pod_id == "pod-9" + await execution.terminate() assert execution.pod_id is None async def test_terminate_noop_without_pod(self): @@ -447,7 +492,7 @@ async def test_wait_returns_result_and_terminates(self): assert result == 9 execution.terminate.assert_awaited_once() - async def test_wait_timeout_keeps_pod(self): + async def test_wait_timeout_terminates_pod(self): job, execution = self._job() execution.poll_result = AsyncMock(return_value=None) with ( @@ -458,7 +503,27 @@ async def test_wait_timeout_keeps_pod(self): ): with pytest.raises(TimeoutError): await job.wait(timeout=10) - execution.terminate.assert_not_awaited() + execution.terminate.assert_awaited_once() + + @pytest.mark.parametrize( + "failure", + [RuntimeError("poll failed"), asyncio.CancelledError()], + ) + async def test_poll_failure_terminates_pod(self, failure): + job, execution = self._job() + execution.poll_result = AsyncMock(side_effect=failure) + with pytest.raises(type(failure)): + await job.wait() + execution.terminate.assert_awaited_once() + + async def test_failed_result_terminates_pod(self): + job, execution = self._job() + execution.poll_result = AsyncMock( + return_value={"success": False, "error": "task failed"} + ) + with pytest.raises(RemoteExecutionError, match="task failed"): + await job.wait() + execution.terminate.assert_awaited_once() async def test_wait_after_done_returns_cached(self): job, execution = self._job() @@ -474,3 +539,38 @@ async def test_cancel_terminates(self): await job.cancel() execution.terminate.assert_awaited_once() assert job._done + + +class TestSpawnCleanup: + @pytest.mark.parametrize("stage", ["start", "wait_ready", "submit"]) + @pytest.mark.parametrize("cancelled", [False, True]) + async def test_setup_failure_terminates_pod(self, stage, cancelled): + from runpod.apps.tasks import TaskExecution + + spec = ResourceSpec(kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"]) + execution = TaskExecution(spec, api=MagicMock()) + pods = set() + failure = asyncio.CancelledError() if cancelled else RuntimeError("setup failed") + + async def start(): + execution.pod_id = "pod-9" + pods.add("pod-9") + if stage == "start": + raise failure + + async def terminate(pod_id): + pods.remove(pod_id) + + execution.start = start + execution.api.terminate_pod = terminate + execution.wait_ready = AsyncMock( + side_effect=failure if stage == "wait_ready" else None + ) + execution.submit = AsyncMock( + side_effect=failure if stage == "submit" else None + ) + with patch("runpod.apps.tasks.TaskExecution", return_value=execution): + with pytest.raises(type(failure)): + await PodTarget(spec, lambda: None).submit({}) + assert not pods + assert execution.pod_id is None diff --git a/tests/test_apps/test_volume.py b/tests/test_apps/test_volume.py index bda7cc445..e606028e2 100644 --- a/tests/test_apps/test_volume.py +++ b/tests/test_apps/test_volume.py @@ -5,6 +5,7 @@ import pytest +from runpod.apps.errors import InvalidResourceError from runpod.apps.spec import ResourceKind, ResourceSpec from runpod.apps.volume import ( Volume, @@ -148,17 +149,13 @@ def volume_created(self, name, size, dc): class TestTaskVolume: def test_single_volume_only(self): - from runpod.apps.tasks import TaskExecution - - spec = ResourceSpec( - kind=ResourceKind.TASK, - name="t", - cpu=["cpu3c-1-2"], - volume=[Volume("a"), Volume("b")], - ) - execution = TaskExecution(spec, api=_api()) - with pytest.raises(VolumeError, match="exactly one volume"): - asyncio.run(execution._attach_volume({})) + with pytest.raises(InvalidResourceError): + ResourceSpec( + kind=ResourceKind.TASK, + name="t", + cpu=["cpu3c-1-2"], + volume=[Volume("a"), Volume("b")], + ) def test_pod_pins_to_volume_dc(self): from runpod.apps.tasks import TaskExecution diff --git a/tests/test_cli/test_deploy_command.py b/tests/test_cli/test_deploy_command.py index 6a6a17359..2fe5942e7 100644 --- a/tests/test_cli/test_deploy_command.py +++ b/tests/test_cli/test_deploy_command.py @@ -7,6 +7,7 @@ from runpod.apps.deploy import DeployResult from runpod.rp_cli.main import cli +from runpod.apps.utils.names import dev_endpoint_name @pytest.fixture(autouse=True) @@ -166,7 +167,7 @@ def test_once_runs_entrypoint_and_stops(self, tmp_path, monkeypatch): session.start = AsyncMock() session.stop = AsyncMock() session.refresh = AsyncMock() - session._endpoints = {"dev-demo-chat": "ep1"} + session._endpoints = {dev_endpoint_name("demo", "chat"): "ep1"} def make_session(apps, events=None): session.apps = apps @@ -177,6 +178,7 @@ def make_session(apps, events=None): assert result.exit_code == 0, result.output session.start.assert_awaited_once() session.stop.assert_awaited_once() + assert "ep1" in result.output def test_once_entrypoint_failure_exits_nonzero(self, tmp_path, monkeypatch): module = tmp_path / "main.py" @@ -201,6 +203,93 @@ def make_session(apps, events=None): assert result.exit_code == 1 session.stop.assert_awaited_once() + def test_partial_start_failure_deletes_created_endpoints(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text( + ENTRYPOINT_SOURCE + + '\n@app.queue(name="later", cpu="cpu3c-1-2")\n' + + "def later():\n return 1\n" + ) + monkeypatch.chdir(tmp_path) + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.side_effect = [ + {"id": "ep-created"}, RuntimeError("no capacity") + ] + api.delete_endpoint.return_value = True + + with patch("runpod.apps.dev.AppsApiClient", return_value=api): + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) + + assert result.exit_code == 1, result.output + assert "no capacity" in result.output + api.delete_endpoint.assert_awaited_once_with("ep-created") + + def test_cleanup_failure_reports_billable_endpoint(self, tmp_path, monkeypatch): + module = tmp_path / "main.py" + module.write_text(ENTRYPOINT_SOURCE) + monkeypatch.chdir(tmp_path) + api = AsyncMock() + api.list_my_endpoints.return_value = [] + api.save_endpoint.return_value = {"id": "ep-billed"} + api.delete_endpoint.side_effect = RuntimeError("permission denied") + + with patch("runpod.apps.dev.AppsApiClient", return_value=api): + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) + + assert result.exit_code == 1, result.output + assert "ep-billed" in result.output + assert "permission denied" in result.output + + def test_cancellation_waits_for_entrypoint_before_cleanup(self, tmp_path, monkeypatch): + import asyncio + import threading + + module = tmp_path / "main.py" + module.write_text(ENTRYPOINT_SOURCE) + monkeypatch.chdir(tmp_path) + release = threading.Event() + events = [] + api = AsyncMock() + api.list_my_endpoints.return_value = [] + loop = None + session_task = None + + async def provision(payload): + nonlocal loop, session_task + loop = asyncio.get_running_loop() + session_task = asyncio.current_task() + return {"id": "ep-active"} + + def cancel(): + events.append("cancelled") + session_task.cancel() + loop.call_later(0.05, release.set) + + def entrypoint(fn): + loop.call_soon_threadsafe(cancel) + if not release.wait(5): + raise TimeoutError("entrypoint was not released") + events.append("finished") + + async def delete(endpoint_id): + events.append("deleted") + return True + + api.save_endpoint.side_effect = provision + api.delete_endpoint.side_effect = delete + try: + with ( + patch("runpod.apps.dev.AppsApiClient", return_value=api), + patch("runpod.apps.entrypoint.run_entrypoint", side_effect=entrypoint), + ): + result = _runner().invoke(cli, ["flash", "dev", str(module), "--once"]) + finally: + release.set() + + assert result.exit_code == 0, result.output + assert events == ["cancelled", "finished", "deleted"] + def test_module_without_entrypoint(self, tmp_path, monkeypatch): module = tmp_path / "main.py" module.write_text(APP_SOURCE) diff --git a/tests/test_cli/test_init_command.py b/tests/test_cli/test_init_command.py index 4bad4c4b3..42be0898c 100644 --- a/tests/test_cli/test_init_command.py +++ b/tests/test_cli/test_init_command.py @@ -1,7 +1,5 @@ """rp flash init: project scaffolding.""" -from pathlib import Path - from click.testing import CliRunner from runpod.apps.init import create_project, detect_conflicts @@ -13,25 +11,18 @@ def test_writes_skeleton(self, tmp_path): written = create_project(tmp_path, "my-app") names = {p.name for p in written} assert names == {"main.py", "requirements.txt", ".runpodignore"} - main = (tmp_path / "main.py").read_text() - assert 'App("my-app")' in main - assert "@runpod.local_entrypoint" in main - - def test_skeleton_module_is_valid_python(self, tmp_path): - create_project(tmp_path, "my-app") - compile((tmp_path / "main.py").read_text(), "main.py", "exec") def test_existing_files_kept_without_overwrite(self, tmp_path): (tmp_path / "main.py").write_text("original") written = create_project(tmp_path, "my-app") assert (tmp_path / "main.py").read_text() == "original" - assert Path(tmp_path / "requirements.txt") not in written or True + assert tmp_path / "requirements.txt" in written assert (tmp_path / "requirements.txt").exists() def test_overwrite_replaces_files(self, tmp_path): (tmp_path / "main.py").write_text("original") create_project(tmp_path, "my-app", overwrite=True) - assert 'App("my-app")' in (tmp_path / "main.py").read_text() + assert (tmp_path / "main.py").read_text() != "original" def test_creates_directory(self, tmp_path): target = tmp_path / "new-project" @@ -62,7 +53,6 @@ def test_init_current_directory(self, tmp_path, monkeypatch): result = runner.invoke(cli, ["flash", "init", "."]) assert result.exit_code == 0, result.output assert (tmp_path / "main.py").exists() - assert tmp_path.name in (tmp_path / "main.py").read_text() def test_init_conflicts_fail_without_force(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) @@ -79,4 +69,4 @@ def test_init_force_overwrites(self, tmp_path, monkeypatch): runner = CliRunner() result = runner.invoke(cli, ["flash", "init", ".", "--force"]) assert result.exit_code == 0, result.output - assert "App(" in (tmp_path / "main.py").read_text() + assert (tmp_path / "main.py").read_text() != "old" diff --git a/tests/test_cli/test_management_commands.py b/tests/test_cli/test_management_commands.py index b81b9a744..6e57f3eb7 100644 --- a/tests/test_cli/test_management_commands.py +++ b/tests/test_cli/test_management_commands.py @@ -2,6 +2,8 @@ from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from click.testing import CliRunner from runpod.apps.manage import UndeployResult @@ -345,29 +347,34 @@ async def fake_stream(pod_id, **kwargs): class TestLoginCommand: - def test_api_key_flag(self): + @pytest.mark.parametrize("browser", [False, True]) + @pytest.mark.parametrize("profile", ["default", "staging"]) + def test_login_updates_only_selected_profile( + self, tmp_path, monkeypatch, browser, profile + ): + from runpod.cli.groups.config import functions + + credentials = tmp_path / "config.toml" + credentials.write_text( + '[default]\napi_key = "old-default"\n' + '[staging]\napi_key = "old-staging"\n' + '[production]\napi_key = "keep-production"\n' + ) + monkeypatch.setattr(functions, "CREDENTIAL_FILE", str(credentials)) + args = ["login"] + if profile != "default": + args.extend(["--profile", profile]) + args.extend(["--no-open"] if browser else ["--api-key", "new-key"]) with patch( - "runpod.cli.groups.config.functions.set_credentials" - ) as creds: - result = _runner().invoke( - cli, ["login", "--api-key", "sk-test"] - ) - assert result.exit_code == 0 - creds.assert_called_once_with("sk-test", overwrite=True) - - def test_browser_flow(self): - with ( - patch( - "runpod.apps.auth.browser_login", - AsyncMock(return_value="sk-granted"), - ), - patch( - "runpod.cli.groups.config.functions.set_credentials" - ) as creds, + "runpod.apps.auth.browser_login", AsyncMock(return_value="new-key") ): - result = _runner().invoke(cli, ["login", "--no-open"]) - assert result.exit_code == 0 - creds.assert_called_once_with("sk-granted", overwrite=True) + result = _runner().invoke(cli, args) + + assert result.exit_code == 0, result.output + assert functions.get_credentials(profile)["api_key"] == "new-key" + untouched = "staging" if profile == "default" else "default" + assert functions.get_credentials(untouched)["api_key"] == f"old-{untouched}" + assert functions.get_credentials("production")["api_key"] == "keep-production" def test_browser_flow_error(self): from runpod.apps.auth import LoginError From a42f5b4877e99b796ad16773e8aee3640f1eb562 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:41:02 -0700 Subject: [PATCH 106/120] fix: preserve apps capabilities across rest integration --- runpod/api/mutations/apps.py | 39 +-- runpod/api/queries/apps.py | 50 +--- runpod/api/rest.py | 65 +++-- runpod/apps/api.py | 355 +++++++++++++++++++------- runpod/apps/build.py | 1 + runpod/apps/dev.py | 14 +- runpod/apps/discovery.py | 15 +- runpod/apps/handles.py | 5 + runpod/apps/logs.py | 39 +-- runpod/apps/monitor.py | 50 ++-- runpod/apps/placement.py | 26 +- runpod/apps/targets.py | 7 +- runpod/apps/volume.py | 2 +- tests/test_api/test_rest.py | 273 +++++++++++++------- tests/test_apps/test_api_client.py | 286 ++++++++++++++------- tests/test_apps/test_app.py | 4 +- tests/test_apps/test_build.py | 41 +-- tests/test_apps/test_config_params.py | 1 - tests/test_apps/test_dev.py | 3 + tests/test_apps/test_logs.py | 107 +++++--- tests/test_apps/test_monitor.py | 44 ++-- tests/test_apps/test_placement.py | 24 +- tests/test_apps/test_targets.py | 25 +- tests/test_apps/test_tasks.py | 4 +- tests/test_apps/test_volume.py | 14 +- 25 files changed, 949 insertions(+), 545 deletions(-) diff --git a/runpod/api/mutations/apps.py b/runpod/api/mutations/apps.py index ff7287f5e..2e3c73a44 100644 --- a/runpod/api/mutations/apps.py +++ b/runpod/api/mutations/apps.py @@ -1,5 +1,6 @@ -"""graphql mutations for the apps control-plane client.""" +"""GraphQL mutations for capabilities absent from the resource REST API.""" +# endpoint saves retain flash bindings, cached models, and schedules atomically. MUTATION_SAVE_ENDPOINT = """ mutation saveEndpoint($input: EndpointInput!) { saveEndpoint(input: $input) { @@ -16,12 +17,7 @@ } """ -MUTATION_DELETE_ENDPOINT = """ -mutation deleteEndpoint($id: String!) { - deleteEndpoint(id: $id) -} -""" - +# task provisioning requires terminateAfter and supportPublicIp. MUTATION_DEPLOY_CPU_POD = """ mutation deployCpuPod($input: deployCpuPodInput!) { deployCpuPod(input: $input) { id desiredStatus } @@ -34,12 +30,6 @@ } """ -MUTATION_TERMINATE_POD = """ -mutation terminatePod($input: PodTerminateInput!) { - podTerminate(input: $input) -} -""" - MUTATION_CREATE_FLASH_APP = """ mutation createFlashApp($input: CreateFlashAppInput!) { createFlashApp(input: $input) { id name } @@ -52,29 +42,6 @@ } """ -MUTATION_CREATE_NETWORK_VOLUME = """ -mutation createNetworkVolume($input: CreateNetworkVolumeInput!) { - createNetworkVolume(input: $input) { - id - name - size - dataCenterId - } -} -""" - -MUTATION_SAVE_REGISTRY_AUTH = """ -mutation SaveRegistryAuth($input: SaveRegistryAuthInput!) { - saveRegistryAuth(input: $input) { id name } -} -""" - -MUTATION_DELETE_REGISTRY_AUTH = """ -mutation DeleteRegistryAuth($registryAuthId: String!) { - deleteRegistryAuth(registryAuthId: $registryAuthId) -} -""" - MUTATION_CREATE_SECRET = """ mutation secretCreate($input: SecretCreateInput!) { secretCreate(input: $input) { id name } diff --git a/runpod/api/queries/apps.py b/runpod/api/queries/apps.py index 8089aa253..9da1b2cde 100644 --- a/runpod/api/queries/apps.py +++ b/runpod/api/queries/apps.py @@ -1,16 +1,10 @@ -"""graphql queries for the apps control-plane client.""" +"""GraphQL queries for capabilities absent from the resource REST API.""" -QUERY_MY_ENDPOINTS = """ -query myEndpoints { - myself { - endpoints { - id - name - templateId - workersMin - workersMax - template { env { key value } } - } +# rest stock filters require a power-of-two count of at least two. +QUERY_CPU_STOCK = """ +query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { + cpuFlavors(input: $cpuFlavorInput) { + specifics(input: $specificsInput) { stockStatus } } } """ @@ -31,38 +25,6 @@ } """ -QUERY_GPU_STOCK = """ -query GpuStock($gpuTypesInput: GpuTypeFilter, $lowestPriceInput: GpuLowestPriceInput) { - gpuTypes(input: $gpuTypesInput) { - lowestPrice(input: $lowestPriceInput) { stockStatus } - } -} -""" - -QUERY_CPU_STOCK = """ -query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { - cpuFlavors(input: $cpuFlavorInput) { - specifics(input: $specificsInput) { stockStatus } - } -} -""" - -QUERY_NETWORK_VOLUMES = """ -query myVolumes { - myself { - networkVolumes { id name size dataCenterId } - } -} -""" - -QUERY_REGISTRY_AUTHS = """ -query myRegistryCreds { - myself { - containerRegistryCreds { id name } - } -} -""" - QUERY_SECRETS = """ query mySecrets { myself { diff --git a/runpod/api/rest.py b/runpod/api/rest.py index 987eb1439..22e519c72 100644 --- a/runpod/api/rest.py +++ b/runpod/api/rest.py @@ -1,8 +1,10 @@ """Runpod REST API transport.""" +import json as json_module import os from typing import Any, Mapping, Optional +import aiohttp import requests from runpod import error @@ -37,34 +39,31 @@ def _build_headers(api_key: str) -> dict[str, str]: } -def _response_json(response: requests.Response) -> dict[str, Any]: - try: - payload = response.json() - except ValueError: - return {} - return payload if isinstance(payload, dict) else {} - - def _raise_for_error( - response: requests.Response, method: str, path: str + status_code: int, method: str, path: str, text: str = "" ) -> None: - if response.status_code == HTTP_STATUS_UNAUTHORIZED: + if status_code == HTTP_STATUS_UNAUTHORIZED: raise error.AuthenticationError( "Unauthorized request, please check your API key." ) - if response.status_code < HTTP_STATUS_BAD_REQUEST: + if status_code < HTTP_STATUS_BAD_REQUEST: return - payload = _response_json(response) + try: + payload = json_module.loads(text) + except ValueError: + payload = {} + if not isinstance(payload, dict): + payload = {} message = payload.get("detail") or payload.get("title") if not message: - message = response.text or f"Request failed with status {response.status_code}" + message = text or f"Request failed with status {status_code}" raise error.QueryError( str(message), f"{method.upper()} {path}", - status_code=response.status_code, + status_code=status_code, errors=payload.get("errors"), ) @@ -87,8 +86,44 @@ def run_rest_request( json=json, timeout=timeout, ) - _raise_for_error(response, method, path) + if response.status_code >= HTTP_STATUS_BAD_REQUEST: + _raise_for_error(response.status_code, method, path, response.text) if response.status_code == HTTP_STATUS_NO_CONTENT or not response.content: return None return response.json() + + +async def run_rest_request_async( + method: str, + path: str, + *, + api_key: Optional[str] = None, + params: Optional[Mapping[str, Any]] = None, + json: Optional[Mapping[str, Any]] = None, + timeout: float = 30, +) -> Optional[dict[str, Any]]: + """Send an authenticated REST request without blocking the event loop.""" + headers = _build_headers(_resolve_api_key(api_key)) + client_timeout = aiohttp.ClientTimeout(total=timeout) + async with aiohttp.ClientSession(timeout=client_timeout) as session: + # aiohttp otherwise replays put/delete requests after an ambiguous disconnect. + session._retry_connection = False # pylint: disable=protected-access + async with session.request( + method, + _build_url(path), + headers=headers, + params=params, + json=json, + ) as response: + if response.status >= HTTP_STATUS_BAD_REQUEST: + text = ( + "" + if response.status == HTTP_STATUS_UNAUTHORIZED + else await response.text(errors="replace") + ) + _raise_for_error(response.status, method, path, text) + + if response.status == HTTP_STATUS_NO_CONTENT or not await response.read(): + return None + return await response.json(content_type=None) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index fb6104ee6..b25ee5320 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -1,8 +1,7 @@ """control-plane calls for app provisioning. -a thin wrapper over runpod.api.graphql's shared async transport, -scoped to what the apps surface needs: endpoint save/delete for dev -sessions, and the app / build / environment lifecycle for deploys. +rest handles equivalent resource operations; graphql handles flash lifecycle, +auth, secrets, task provisioning, and endpoint capabilities absent from rest. management verbs for the wider sdk stay in runpod.api.ctl_commands. """ @@ -10,15 +9,142 @@ import aiohttp +from ..api.ctl_commands import _cpu_config, _path_segment, _split_values from ..api.graphql import run_graphql_query_async from ..api.mutations import apps as app_mutations from ..api.queries import apps as app_queries +from ..api.rest import run_rest_request_async from ..error import QueryError - _TRANSPORT_RETRIES = 4 +def _rest_endpoint_input(endpoint: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """translate only saves whose capabilities rest can preserve atomically.""" + fields = { + "id", + "name", + "type", + "template", + "templateId", + "workersMin", + "workersMax", + "idleTimeout", + "scalerType", + "scalerValue", + "executionTimeoutMs", + "flashBootType", + "locations", + "instanceIds", + "gpuIds", + "gpuCount", + "minCudaVersion", + "allowedCudaVersions", + "networkVolumeIds", + } + template_fields = { + "imageName": "image", + "containerDiskInGb": "disk", + "dockerArgs": "args", + "containerRegistryAuthId": "registry", + } + template = endpoint.get("template") or {} + # flash bindings, cached models, schedules, and template metadata need gql. + if endpoint.keys() - fields or template.keys() - ( + template_fields.keys() | {"env", "ports"} + ): + return None + if endpoint.get("type", "QB") not in {"QB", "LB"}: + return None + scaler = endpoint.get("scalerType") + if scaler not in {None, "QUEUE_DELAY", "REQUEST_COUNT"}: + return None + if ("scalerType" in endpoint) != ("scalerValue" in endpoint): + return None + if not endpoint.get("id") and ( + scaler is None or not (template.get("imageName") or endpoint.get("templateId")) + ): + return None + + body = { + rest: template[gql] for gql, rest in template_fields.items() if gql in template + } + if "env" in template: + body["env"] = {item["key"]: item["value"] for item in template["env"]} + if "ports" in template: + body["ports"] = _split_values(template["ports"]) + for gql, rest in ( + ("name", "name"), + ("templateId", "templateId"), + ("executionTimeoutMs", "timeout"), + ("flashBootType", "flashboot"), + ): + if gql in endpoint: + body[rest] = endpoint[gql] + if "networkVolumeIds" in endpoint: + volumes = endpoint["networkVolumeIds"] + if any(set(volume) != {"networkVolumeId"} for volume in volumes): + return None + body["networkVolumes"] = [volume["networkVolumeId"] for volume in volumes] + if not endpoint.get("id"): + body["type"] = "LOAD_BALANCER" if endpoint.get("type") == "LB" else "QUEUE" + if "locations" in endpoint: + body["dataCenterIds"] = _split_values(endpoint["locations"]) + if "instanceIds" in endpoint: + body["cpu"] = [_cpu_config(instance) for instance in endpoint["instanceIds"]] + # one-vcpu endpoints require graphql. + if any(cpu["vcpuCount"] < 2 for cpu in body["cpu"]): + return None + gpu = {} + if "gpuIds" in endpoint: + tokens = _split_values(endpoint["gpuIds"]) + gpu["pools"] = [token for token in tokens if not token.startswith("-")] + gpu["excludedTypes"] = [token[1:] for token in tokens if token.startswith("-")] + for key in ("gpuCount", "minCudaVersion", "allowedCudaVersions"): + if key in endpoint: + gpu["count" if key == "gpuCount" else key] = ( + _split_values(endpoint[key]) + if key == "allowedCudaVersions" + else endpoint[key] + ) + if gpu: + body["gpu"] = gpu + workers = { + rest: endpoint[gql] + for gql, rest in ( + ("workersMin", "min"), + ("workersMax", "max"), + ("idleTimeout", "idleTimeout"), + ) + if gql in endpoint + } + # queue request-count scaling ignores idle timeout upstream and rest rejects it. + if ( + not endpoint.get("id") + and endpoint.get("type", "QB") == "QB" + and scaler == "REQUEST_COUNT" + ): + workers.pop("idleTimeout", None) + if workers: + body["workers"] = workers + if scaler is not None: + key = "queueDelay" if scaler == "QUEUE_DELAY" else "requestCount" + body["scaling"] = {"type": scaler, key: endpoint["scalerValue"]} + return body + + +def _stock_in_datacenter(data: Dict[str, Any], data_center_id: str) -> str: + """catalog omits datacenters where the requested configuration is unavailable.""" + return next( + ( + dc["availability"] + for dc in data.get("dataCenters", []) + if dc["id"] == data_center_id + ), + "NONE", + ) + + def is_capacity_error(exc: Exception) -> bool: message = str(exc).lower() return any( @@ -49,7 +175,7 @@ async def _execute( import asyncio attempts = _TRANSPORT_RETRIES if retry else 1 - for attempt in range(attempts): + for attempt in range(attempts - 1): try: response = await run_graphql_query_async( query, @@ -59,32 +185,92 @@ async def _execute( ) return response["data"] except (aiohttp.ClientError, OSError, asyncio.TimeoutError): - if attempt == attempts - 1: - raise await asyncio.sleep(2**attempt) + response = await run_graphql_query_async( + query, + api_key=self._api_key, + variables=variables, + anonymous=anonymous, + ) + return response["data"] async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: - """create or update a serverless endpoint. include id to update.""" + """save via rest unless the input requires an unsupported capability.""" + body = _rest_endpoint_input(endpoint_input) + endpoint_id = endpoint_input.get("id") + if body is not None and endpoint_id: + current = await run_rest_request_async( + "GET", + f"/v2/serverless/{_path_segment(endpoint_id)}", + api_key=self._api_key, + ) + requested_type = endpoint_input.get("type") + expected = ( + ("LOAD_BALANCER" if requested_type == "LB" else "QUEUE") + if requested_type is not None + else current.get("type") + ) + # rest updates cannot change routing type or compute family. + if ( + current.get("type") != expected + or ("cpu" in body and not current.get("cpu")) + or ("gpu" in body and current.get("cpu")) + ): + body = None + elif ( + current.get("type") == "QUEUE" + and body.get("scaling", current.get("scaling", {})).get("type") + == "REQUEST_COUNT" + ): + body.get("workers", {}).pop("idleTimeout", None) + if body is not None: + path = "/v2/serverless" + if endpoint_id: + path += f"/{_path_segment(endpoint_id)}" + return await run_rest_request_async( + "PATCH" if endpoint_id else "POST", + path, + api_key=self._api_key, + json=body, + ) + template = endpoint_input.get("template") + if template is not None and "name" not in template and "name" in endpoint_input: + # graphql requires a template name; rest names the bound template itself. + endpoint_input = { + **endpoint_input, + "template": {"name": endpoint_input["name"], **template}, + } mutation = app_mutations.MUTATION_SAVE_ENDPOINT data = await self._execute(mutation, {"input": endpoint_input}) return data["saveEndpoint"] async def delete_endpoint(self, endpoint_id: str) -> bool: - mutation = app_mutations.MUTATION_DELETE_ENDPOINT - data = await self._execute(mutation, {"id": endpoint_id}) - if not data.get("deleteEndpoint"): - raise QueryError(f"endpoint {endpoint_id!r} was not deleted", mutation) + await run_rest_request_async( + "DELETE", + f"/v2/serverless/{_path_segment(endpoint_id)}", + api_key=self._api_key, + ) return True async def list_my_endpoints(self) -> List[Dict[str, Any]]: - query = app_queries.QUERY_MY_ENDPOINTS - data = await self._execute(query, retry=True) - return data["myself"]["endpoints"] + data = await run_rest_request_async( + "GET", "/v2/serverless", api_key=self._api_key + ) + return data["endpoints"] + + async def endpoint_workers(self, endpoint_id: str) -> Dict[str, Any]: + """worker state uses the user key.""" + return await run_rest_request_async( + "GET", + f"/v2/serverless/{_path_segment(endpoint_id)}/workers", + api_key=self._api_key, + timeout=10, + ) async def deploy_task_pod( self, pod_input: Dict[str, Any], *, is_cpu: bool ) -> Dict[str, Any]: - """deploy an on-demand pod for a task run.""" + """provision with a termination deadline and public-ip requirement.""" pod_input = dict(pod_input) if is_cpu: instance_ids = pod_input.pop("instanceIds", None) or [ @@ -105,8 +291,9 @@ async def deploy_task_pod( return data["podFindAndDeployOnDemand"] async def terminate_pod(self, pod_id: str) -> None: - mutation = app_mutations.MUTATION_TERMINATE_POD - await self._execute(mutation, {"input": {"podId": pod_id}}) + await run_rest_request_async( + "DELETE", f"/v2/pods/{_path_segment(pod_id)}", api_key=self._api_key + ) async def get_app_by_name(self, name: str) -> Optional[Dict[str, Any]]: query = app_queries.QUERY_FLASH_APP_BY_NAME @@ -137,98 +324,94 @@ async def gpu_stock_status( gpu_count: int = 1, pods: bool = False, ) -> Optional[str]: - """stock signal for a gpu device in one datacenter. - - the serverless plane (includeAiApi) and the pod plane have - different availability; pods=True queries pod stock (tasks). - """ - query = app_queries.QUERY_GPU_STOCK - data = await self._execute( - query, - { - "gpuTypesInput": {"ids": [gpu_id]}, - "lowestPriceInput": { - "dataCenterId": data_center_id, - "gpuCount": gpu_count, - "secureCloud": True, - "includeAiApi": not pods, - }, + """secure-cloud stock scoped to GPU count and pod/serverless product.""" + data = await run_rest_request_async( + "GET", + f"/v2/catalog/gpus/{_path_segment(gpu_id)}", + api_key=self._api_key, + params={ + "include": "AVAILABILITY", + "product": "POD" if pods else "SERVERLESS", + "count": gpu_count, + "cloud": "SECURE", }, - retry=True, ) - gpu_types = data.get("gpuTypes") or [] - first = gpu_types[0] if gpu_types else {} - price = first.get("lowestPrice") if isinstance(first, dict) else None - return (price or {}).get("stockStatus") + return _stock_in_datacenter(data, data_center_id) async def cpu_stock_status( - self, instance_id: str, data_center_id: str + self, instance_id: str, data_center_id: str, *, pods: bool = False ) -> Optional[str]: - """stock signal for a cpu flavor in one datacenter.""" - flavor = instance_id.split("-", 1)[0] - query = app_queries.QUERY_CPU_STOCK - data = await self._execute( - query, - { - "cpuFlavorInput": {"id": flavor}, - "specificsInput": { - "dataCenterId": data_center_id, - "instanceId": instance_id, + """stock scoped to CPU flavor, vCPU count, and pod/serverless product.""" + cpu = _cpu_config(instance_id) + # rest stock filters require a power-of-two count of at least two. + count = cpu["vcpuCount"] + if count < 2 or count & (count - 1): + data = await self._execute( + app_queries.QUERY_CPU_STOCK, + { + "cpuFlavorInput": {"id": cpu["id"], "isSls": not pods}, + "specificsInput": { + "dataCenterId": data_center_id, + "instanceId": instance_id, + "isSls": not pods, + }, }, + retry=True, + ) + flavors = data.get("cpuFlavors") or [] + specifics = (flavors[0].get("specifics") or {}) if flavors else {} + return specifics.get("stockStatus") + data = await run_rest_request_async( + "GET", + f"/v2/catalog/cpus/{_path_segment(cpu['id'])}", + api_key=self._api_key, + params={ + "include": "AVAILABILITY", + "product": "POD" if pods else "SERVERLESS", + "vcpuCount": cpu["vcpuCount"], }, - retry=True, ) - flavors = data.get("cpuFlavors") or [] - first = flavors[0] if flavors else {} - specifics = first.get("specifics") if isinstance(first, dict) else None - return (specifics or {}).get("stockStatus") + return _stock_in_datacenter(data, data_center_id) async def list_network_volumes(self) -> List[Dict[str, Any]]: - query = app_queries.QUERY_NETWORK_VOLUMES - data = await self._execute(query, retry=True) - return data["myself"].get("networkVolumes") or [] + data = await run_rest_request_async( + "GET", "/v2/network-volumes", api_key=self._api_key + ) + return data["networkVolumes"] async def create_network_volume( self, name: str, size: int, data_center_id: str ) -> Dict[str, Any]: - mutation = app_mutations.MUTATION_CREATE_NETWORK_VOLUME - data = await self._execute( - mutation, - { - "input": { - "name": name, - "size": size, - "dataCenterId": data_center_id, - } - }, + return await run_rest_request_async( + "POST", + "/v2/network-volumes", + api_key=self._api_key, + json={"name": name, "size": size, "dataCenter": data_center_id}, ) - return data["createNetworkVolume"] async def list_registry_auths(self) -> List[Dict[str, Any]]: - query = app_queries.QUERY_REGISTRY_AUTHS - data = await self._execute(query, retry=True) - return data["myself"].get("containerRegistryCreds") or [] + data = await run_rest_request_async( + "GET", "/v2/registries", api_key=self._api_key + ) + return data["registries"] async def create_registry_auth( self, name: str, username: str, password: str ) -> Dict[str, Any]: - mutation = app_mutations.MUTATION_SAVE_REGISTRY_AUTH - data = await self._execute( - mutation, - { - "input": { - "name": name, - "username": username, - "password": password, - } - }, + return await run_rest_request_async( + "POST", + "/v2/registries", + api_key=self._api_key, + json={"name": name, "username": username, "password": password}, ) - return data["saveRegistryAuth"] async def delete_registry_auth(self, auth_id: str) -> bool: - mutation = app_mutations.MUTATION_DELETE_REGISTRY_AUTH - data = await self._execute(mutation, {"registryAuthId": auth_id}) - return bool(data.get("deleteRegistryAuth")) + await run_rest_request_async( + "DELETE", + f"/v2/registries/{_path_segment(auth_id)}", + api_key=self._api_key, + ) + return True async def list_secrets(self) -> List[Dict[str, Any]]: query = app_queries.QUERY_SECRETS diff --git a/runpod/apps/build.py b/runpod/apps/build.py index 7be2b2fee..b438f0ac6 100644 --- a/runpod/apps/build.py +++ b/runpod/apps/build.py @@ -256,6 +256,7 @@ def vendor( """ if not requirements: return + _ensure_pip() cmd = [ sys.executable, "-m", diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index ef12f1d9e..222b66a8e 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -108,16 +108,11 @@ def _cpu_locations(instance_ids: List[str]) -> str: def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: - """build the saveEndpoint payload for a live dev endpoint. - - the template is nested in the saveEndpoint input so it is bound to - the endpoint and cascades on deleteEndpoint. - """ + """build a live endpoint payload with an atomically bound container config.""" spec.validate() resource_env = _render_env(spec.env) - resource_env["FLASH_RESOURCE_NAME"] = spec.name - resource_env["RUNPOD_RESOURCE_NAME"] = spec.name + resource_env["RUNPOD_DEV_RESOURCE"] = spec.name if spec.kind is ResourceKind.API: resource_env["PORT"] = "80" resource_env["PORT_HEALTH"] = "80" @@ -131,7 +126,6 @@ def _endpoint_input(app: App, spec: ResourceSpec, generation: int = 1) -> Dict: "scalerValue": spec.scaler_value, "executionTimeoutMs": spec.execution_timeout_ms, "template": { - "name": f"{dev_endpoint_name(app.name, spec.name)}-template", "imageName": _image_for(spec), "containerDiskInGb": spec.container_disk_gb or (10 if spec.is_cpu else 30), "dockerArgs": "", @@ -308,7 +302,7 @@ async def start(self) -> None: endpoint_id, spec.name, events=self.events, - metrics_key=result.get("aiKey"), + api=self.api, ) self._emit("session_started") @@ -379,7 +373,7 @@ async def refresh(self, apps: List[App]) -> None: endpoint_id, handle.spec.name, events=self.events, - metrics_key=result.get("aiKey"), + api=self.api, ) spec = handle.spec diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 848017d81..2da87dd23 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -133,12 +133,17 @@ def discover_apps(target: Path) -> List[App]: registered = get_registered_apps() seen = {id(a) for a in registered} modules_before = set(sys.modules) + imported = False try: - _import_module(path) - except BaseException as exc: - _restore_registry(registered) - _rollback_modules(modules_before, root) - if strict or not isinstance(exc, DiscoveryError): + try: + _import_module(path) + imported = True + finally: + if not imported: + _restore_registry(registered) + _rollback_modules(modules_before, root) + except DiscoveryError as exc: + if strict: raise failures.append(str(exc)) log.warning("%s", exc) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index b2f006204..a6e6666bc 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -208,6 +208,11 @@ def _guard_discovery(self) -> None: def _is_current_worker(self) -> bool: import os + if dev_app := os.getenv("RUNPOD_DEV_APP"): + return ( + dev_app == self._app.name + and os.getenv("RUNPOD_DEV_RESOURCE") == self.spec.name + ) current = os.getenv("FLASH_RESOURCE_NAME") or os.getenv("RUNPOD_RESOURCE_NAME") return current is not None and current == self.spec.name diff --git a/runpod/apps/logs.py b/runpod/apps/logs.py index 912bc0b6a..c6ec89991 100644 --- a/runpod/apps/logs.py +++ b/runpod/apps/logs.py @@ -1,12 +1,7 @@ -"""pod log access via the host api (hapi). +"""pod logs with rest streams and host-api snapshots. -hosts expose per-pod container and system logs; this is the fastest -way to see why a worker is stuck (image pull progress, region errors, -crash output) without ssh. snapshot and sse streaming supported. - - GET https://hapi.runpod.net/v1/pod/{podId}/logs - ?type=all|container|system - GET .../logs?stream=true&type=...&tail=N&since=RFC3339 +single-source streams use rest pod or serverless-worker routes. combined +streams and finite snapshots use the host api, which supports both. """ import json @@ -15,6 +10,9 @@ import aiohttp +from ..api.ctl_commands import _path_segment +from ..api.rest import _build_url + HAPI_BASE = os.environ.get("RUNPOD_HAPI_URL", "https://hapi.runpod.net") STREAM_TIMEOUT_SECONDS = 3600.0 @@ -47,6 +45,7 @@ async def pod_logs( async def stream_pod_logs( pod_id: str, *, + endpoint_id: Optional[str] = None, log_type: str = "all", tail: int = 100, since: Optional[str] = None, @@ -54,15 +53,23 @@ async def stream_pod_logs( """follow a pod's logs as they arrive. yields {"source": "container"|"system", "line": str, "ts": str} - parsed from the host's sse stream. ends when the host closes the - stream (1h server-side cap) or the caller breaks out. + parsed from the sse stream. pass endpoint_id for serverless workers. + ends when the server closes the stream or the caller breaks out. """ - url = f"{HAPI_BASE}/v1/pod/{pod_id}/logs" - params: Dict[str, Any] = { - "stream": "true", - "type": log_type, - "tail": str(tail), - } + params: Dict[str, Any] = {"tail": str(tail)} + if log_type == "all": + url = f"{HAPI_BASE}/v1/pod/{_path_segment(pod_id)}/logs" + params.update(stream="true", type=log_type) + else: + if endpoint_id is None: + path = f"/v2/pods/{_path_segment(pod_id)}/logs" + else: + path = ( + f"/v2/serverless/{_path_segment(endpoint_id)}" + f"/workers/{_path_segment(pod_id)}/logs" + ) + url = _build_url(path) + params["source"] = log_type if since: params["since"] = since diff --git a/runpod/apps/monitor.py b/runpod/apps/monitor.py index 3affb28e9..eb10e48c3 100644 --- a/runpod/apps/monitor.py +++ b/runpod/apps/monitor.py @@ -1,9 +1,9 @@ """request-time worker observability for dev sessions. while a dev call is in flight, a WorkerMonitor watches the endpoint's -worker metrics for state transitions (initializing, throttled, ready) +rest worker summary for state transitions (initializing, throttled, ready) and, once the job is assigned a worker, follows that worker's container -logs over the hapi sse stream. everything surfaces through a duck-typed +logs over the rest sse stream. everything surfaces through a duck-typed event sink; missing handlers are silently skipped. """ @@ -69,10 +69,18 @@ class PodLogStream: lines dedup across reconnects and sdk runtime frames are filtered. """ - def __init__(self, pod_id: str, resource_name: str, events: object): + def __init__( + self, + pod_id: str, + resource_name: str, + events: object, + *, + endpoint_id: Optional[str] = None, + ): self.pod_id = pod_id self.name = resource_name self.events = events + self.endpoint_id = endpoint_id self._task: Optional[asyncio.Task] = None self._since = datetime.now(timezone.utc).isoformat() self._lines_emitted = 0 @@ -131,6 +139,7 @@ async def _follow(self) -> None: # from the backfill instead of dropped async for event in stream_pod_logs( self.pod_id, + endpoint_id=self.endpoint_id, log_type="container", tail=1000, since=self._since, @@ -179,28 +188,32 @@ def __init__( endpoint_id: str, resource_name: str, events: object, - metrics_key: Optional[str] = None, + api=None, ): self.endpoint_id = endpoint_id self.name = resource_name self.events = events - # /metrics is served on the data plane behind the endpoint's own - # ai key; the user api key is rejected there - self.metrics_key = metrics_key + from .utils.client import default_client + + self.api = default_client(api) self._tasks: List[asyncio.Task] = [] self._streams: Dict[str, PodLogStream] = {} self._last_counts: Optional[Dict[str, int]] = None async def start(self) -> None: - if self.metrics_key: - self._tasks.append(asyncio.ensure_future(self._poll_metrics())) + self._tasks.append(asyncio.ensure_future(self._poll_metrics())) def on_status(self, data: Dict[str, Any]) -> None: """inspect a job-status payload for the assigned worker.""" worker_id = data.get("workerId") if worker_id and worker_id not in self._streams: emit(self.events, "worker_ready", self.name, str(worker_id)) - stream = PodLogStream(str(worker_id), self.name, self.events) + stream = PodLogStream( + str(worker_id), + self.name, + self.events, + endpoint_id=self.endpoint_id, + ) stream.attach() self._streams[str(worker_id)] = stream @@ -214,13 +227,9 @@ async def stop(self) -> None: await stream.stop() async def _poll_metrics(self) -> None: - from .utils.network import endpoint_url_base, get_json - - url = f"{endpoint_url_base()}/{self.endpoint_id}/metrics" - headers = {"Authorization": f"Bearer {self.metrics_key}"} while True: try: - data = await get_json(url, headers, 10.0) + data = await self.api.endpoint_workers(self.endpoint_id) except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - observability is best-effort @@ -233,11 +242,12 @@ def _report_counts(self, data: Dict[str, Any]) -> None: # once a worker picked the job up, pool churn is noise if self._streams: return - workers = data.get("workers") + workers = data.get("summary") if not isinstance(workers, dict): return counts = { - state: workers.get(state, 0) or 0 for state in _TRACKED_STATES + state: workers.get("idle" if state == "ready" else state, 0) or 0 + for state in _TRACKED_STATES } if counts == self._last_counts: return @@ -270,9 +280,5 @@ def _line_before(raw: str, since_iso: str) -> bool: def format_worker_counts(counts: Dict[str, int]) -> str: """human summary like '1 initializing, 2 ready'.""" - parts = [ - f"{count} {state}" - for state, count in counts.items() - if count - ] + parts = [f"{count} {state}" for state, count in counts.items() if count] return ", ".join(parts) if parts else "no workers" diff --git a/runpod/apps/placement.py b/runpod/apps/placement.py index aa1b227e4..931bece77 100644 --- a/runpod/apps/placement.py +++ b/runpod/apps/placement.py @@ -44,7 +44,8 @@ def _hardware_keys(spec) -> List[StockKey]: from .spec import ResourceKind if spec.is_cpu: - return [("cpu", c, 0) for c in spec.cpu or []] + cpu_kind = "cpu-pod" if spec.kind is ResourceKind.TASK else "cpu" + return [(cpu_kind, c, 0) for c in spec.cpu or []] gpu_kind = "gpu-pod" if spec.kind is ResourceKind.TASK else "gpu" gpu = spec.gpu if not gpu or any(str(g).lower() == "any" for g in gpu): @@ -66,10 +67,10 @@ def __init__(self, api=None): self._api = api self._gpu: Dict[Tuple[str, int, str], int] = {} self._gpu_pod: Dict[Tuple[str, int, str], int] = {} - self._cpu: Dict[Tuple[str, str], int] = {} + self._cpu: Dict[Tuple[str, str, bool], int] = {} self._fetched_gpu: Set[Tuple[str, int]] = set() self._fetched_gpu_pod: Set[Tuple[str, int]] = set() - self._fetched_cpu: Set[str] = set() + self._fetched_cpu: Set[Tuple[str, bool]] = set() async def _client(self): self._api = default_client(self._api) @@ -93,7 +94,10 @@ async def fetch(self, keys: Iterable[StockKey]) -> None: and (k[1], k[2]) not in self._fetched_gpu_pod } cpu_ids = { - k[1] for k in keys if k[0] == "cpu" and k[1] not in self._fetched_cpu + (k[1], k[0] == "cpu-pod") + for k in keys + if k[0] in ("cpu", "cpu-pod") + and (k[1], k[0] == "cpu-pod") not in self._fetched_cpu } client = await self._client() jobs = [] @@ -105,10 +109,10 @@ async def fetch(self, keys: Iterable[StockKey]) -> None: self._fetched_gpu_pod.add((gpu_id, count)) for dc in DataCenter.all(): jobs.append(self._fetch_gpu(client, gpu_id, count, dc.value, True)) - for cpu_id in cpu_ids: - self._fetched_cpu.add(cpu_id) + for cpu_id, pods in cpu_ids: + self._fetched_cpu.add((cpu_id, pods)) for dc in DataCenter.all(): - jobs.append(self._fetch_cpu(client, cpu_id, dc.value)) + jobs.append(self._fetch_cpu(client, cpu_id, dc.value, pods)) if jobs: await asyncio.gather(*jobs) @@ -125,13 +129,13 @@ async def _fetch_gpu( target = self._gpu_pod if pods else self._gpu target[(gpu_id, gpu_count, dc)] = _score(status) - async def _fetch_cpu(self, client, instance_id: str, dc: str) -> None: + async def _fetch_cpu(self, client, instance_id: str, dc: str, pods: bool) -> None: try: - status = await client.cpu_stock_status(instance_id, dc) + status = await client.cpu_stock_status(instance_id, dc, pods=pods) except Exception: # noqa: BLE001 - stock is advisory log.debug("cpu stock query failed for %s@%s", instance_id, dc, exc_info=True) status = None - self._cpu[(instance_id, dc)] = _score(status) + self._cpu[(instance_id, dc, pods)] = _score(status) def score(self, key: StockKey, dc: str) -> int: kind, hw, count = key @@ -146,7 +150,7 @@ def score(self, key: StockKey, dc: str) -> int: ] return max(scores, default=1) return table.get((hw, count, dc), 0) - return self._cpu.get((hw, dc), 0) + return self._cpu.get((hw, dc, kind == "cpu-pod"), 0) def candidates(spec, stock: StockMap) -> Set[str]: diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 8ff7b9b30..3f4c3ff67 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -328,7 +328,6 @@ async def _wait_terminal( window, typically on cold starts; polling covers the rest. on_status, when given, sees every payload (observability hooks read workerId). """ - import asyncio import time if on_status is not None: @@ -584,12 +583,12 @@ def __init__( endpoint_id: str, resource_name: str = "", events: Optional[object] = None, - metrics_key: Optional[str] = None, + api=None, ): self.endpoint_id = endpoint_id self.resource_name = resource_name self.events = events - self.metrics_key = metrics_key + self.api = api self._client = QueueClient(endpoint_id, _headers) self._source_target: Optional[Any] = None self._source_resource: str = "" @@ -697,7 +696,7 @@ def _monitor(self): self.endpoint_id, self.resource_name, self.events, - metrics_key=self.metrics_key, + api=self.api, ) async def invoke( diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index 24f72333f..90bb26d88 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -151,7 +151,7 @@ async def resolve( specs, self._stock, volume_name=volume.name, - existing_dc=record["dataCenterId"], + existing_dc=record["dataCenter"], ) resolved = {"id": record["id"], "dataCenterId": dc} self._resolved[volume.name] = resolved diff --git a/tests/test_api/test_rest.py b/tests/test_api/test_rest.py index d00274687..38a7c5836 100644 --- a/tests/test_api/test_rest.py +++ b/tests/test_api/test_rest.py @@ -1,117 +1,210 @@ -"""Tests for the REST API transport.""" +"""Network-level tests for the synchronous and asynchronous REST transports.""" -from unittest.mock import Mock, patch +import asyncio +import json +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import aiohttp import pytest +import requests import runpod -from runpod.api.rest import run_rest_request +from runpod.api.rest import run_rest_request, run_rest_request_async from runpod.error import AuthenticationError, QueryError -from runpod.user_agent import USER_AGENT - - -def _response(status_code=200, payload=None, content=b"{}", text=""): - response = Mock() - response.status_code = status_code - response.content = content - response.text = text - response.json.return_value = payload if payload is not None else {} - return response - - -def test_request_uses_explicit_api_key(): - response = _response(payload={"pods": []}) - with patch("runpod.api.rest.requests.request", return_value=response) as request: - result = run_rest_request( - "POST", - "/v2/pods", - api_key="key", - params={"include": "all"}, - json={"name": "pod"}, - ) - assert result == {"pods": []} - request.assert_called_once_with( - "POST", - "https://api.runpod.io/v2/pods", - headers={ - "Accept": "application/json", - "Content-Type": "application/json", - "User-Agent": USER_AGENT, - "Authorization": "Bearer key", - }, - params={"include": "all"}, - json={"name": "pod"}, - timeout=30, + +@pytest.fixture +def rest_server(monkeypatch): + state = { + "hits": 0, + "status": 200, + "body": b'{"id":"pod-1"}', + "content_type": "application/json", + "started": threading.Event(), + "disconnected": threading.Event(), + } + + class Handler(BaseHTTPRequestHandler): + def _respond(self): + state["hits"] += 1 + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + state["started"].set() + if state.get("stall"): + self.connection.settimeout(3) + if self.connection.recv(1) == b"": + state["disconnected"].set() + return + if state.get("disconnect"): + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + status, response_body = state["status"], state["body"] + if self.headers.get("Authorization") != "Bearer key": + status, response_body = 401, b"invalid key" + elif self.path != state.get("path", "/prefix/v2/pods"): + status, response_body = 404, b"wrong path" + elif "expected_body" in state and json.loads(body) != state["expected_body"]: + status, response_body = 400, b"wrong body" + self.send_response(status) + self.send_header("Content-Type", state["content_type"]) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + do_GET = do_POST = do_PUT = do_DELETE = _respond + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + monkeypatch.setenv( + "RUNPOD_API_BASE_URL", f"http://127.0.0.1:{server.server_port}/prefix/" ) + monkeypatch.setattr(runpod, "api_key", None) + try: + yield state + finally: + server.shutdown() + server.server_close() + thread.join() + +@pytest.fixture(params=["sync", "async"]) +def rest_request(request): + if request.param == "sync": + return run_rest_request -def test_request_uses_global_api_key_and_custom_base_url(): - response = _response(payload={"gpus": []}) - with ( - patch.object(runpod, "api_key", "global-key"), - patch.dict("os.environ", {"RUNPOD_API_BASE_URL": "https://example.test/"}), - patch("runpod.api.rest.requests.request", return_value=response) as request, - ): - run_rest_request("GET", "/v2/catalog/gpus") + def request_async(*args, **kwargs): + return asyncio.run(run_rest_request_async(*args, **kwargs)) - assert request.call_args.args[1] == "https://example.test/v2/catalog/gpus" - assert request.call_args.kwargs["headers"]["Authorization"] == "Bearer global-key" + return request_async -def test_request_requires_api_key(): - with ( - patch.object(runpod, "api_key", None), - patch("runpod.api.rest.requests.request") as request, - pytest.raises(AuthenticationError, match="No API key provided"), - ): - run_rest_request("GET", "/v2/pods") +@pytest.mark.parametrize("explicit_key", ["key", None]) +def test_authenticated_mutation_at_custom_base_url( + rest_server, rest_request, monkeypatch, explicit_key +): + monkeypatch.setattr(runpod, "api_key", "wrong-key" if explicit_key else "key") + rest_server.update( + path="/prefix/v2/pods?include=all", expected_body={"name": "pod"} + ) + result = rest_request( + "POST", + "/v2/pods", + api_key=explicit_key, + params={"include": "all"}, + json={"name": "pod"}, + ) + assert result == {"id": "pod-1"} - request.assert_not_called() +def test_missing_key_fails_before_network_request(rest_server, rest_request): + with pytest.raises(AuthenticationError): + rest_request("POST", "/v2/pods", json={"name": "pod"}) + assert rest_server["hits"] == 0 -def test_request_returns_none_for_no_content(): - response = _response(status_code=204, content=b"") - with patch("runpod.api.rest.requests.request", return_value=response): - assert run_rest_request("DELETE", "/v2/pods/pod", api_key="key") is None +def test_invalid_key_raises_authentication_error(rest_server, rest_request): + with pytest.raises(AuthenticationError): + rest_request("GET", "/v2/pods", api_key="invalid-key") -def test_request_raises_authentication_error_for_unauthorized_response(): - response = _response(status_code=401, payload={"detail": "invalid key"}) - with ( - patch("runpod.api.rest.requests.request", return_value=response), - pytest.raises(AuthenticationError, match="Unauthorized request"), - ): - run_rest_request("GET", "/v2/pods", api_key="key") +@pytest.mark.parametrize("status", [204, 200]) +def test_empty_response_returns_none(rest_server, rest_request, status): + rest_server.update(status=status, body=b"") + assert rest_request("DELETE", "/v2/pods", api_key="key") is None -def test_request_raises_query_error_from_problem_response(): - response = _response( - status_code=422, - payload={ - "title": "Unprocessable Entity", - "detail": "request validation failed", - "errors": ["$.name is required"], - }, - ) - with ( - patch("runpod.api.rest.requests.request", return_value=response), - pytest.raises(QueryError, match="request validation failed") as raised, - ): - run_rest_request("POST", "/v2/pods", api_key="key", json={}) +def test_problem_json_preserves_validation_error(rest_server, rest_request): + rest_server.update( + status=422, + content_type="application/problem+json", + body=json.dumps( + { + "title": "Unprocessable Entity", + "detail": "request validation failed", + "errors": ["$.name is required"], + } + ).encode(), + ) + with pytest.raises(QueryError) as raised: + rest_request("post", "/v2/pods", api_key="key", json={}) + assert str(raised.value) == "request validation failed" assert raised.value.query == "POST /v2/pods" assert raised.value.status_code == 422 assert raised.value.errors == ["$.name is required"] -def test_request_uses_text_for_non_json_error(): - response = _response(status_code=500, content=b"failure", text="upstream failure") - response.json.side_effect = ValueError - with ( - patch("runpod.api.rest.requests.request", return_value=response), - pytest.raises(QueryError, match="upstream failure") as raised, - ): - run_rest_request("GET", "/v2/pods", api_key="key") +@pytest.mark.parametrize( + ("body", "message"), + [ + (b'{"title":"service unavailable"}', "service unavailable"), + (b"upstream failure", "upstream failure"), + (b'["unexpected array"]', '["unexpected array"]'), + (b"\xff", "\ufffd"), + (b"", "Request failed with status 503"), + ], +) +def test_error_body_fallbacks_do_not_mask_http_failure( + rest_server, rest_request, body, message +): + rest_server.update( + status=503, content_type="application/problem+json; charset=utf-8", body=body + ) + with pytest.raises(QueryError) as raised: + rest_request("POST", "/v2/pods", api_key="key", json={"name": "pod"}) + assert str(raised.value) == message + assert raised.value.status_code == 503 + assert raised.value.errors == [] + assert rest_server["hits"] == 1 + + +@pytest.mark.parametrize("method", ["POST", "PUT", "DELETE"]) +def test_mutation_is_not_repeated_after_disconnect(rest_server, rest_request, method): + rest_server["disconnect"] = True + with pytest.raises((requests.ConnectionError, aiohttp.ClientConnectionError)): + rest_request(method, "/v2/pods", api_key="key", json={"name": "pod"}) + assert rest_server["hits"] == 1 + + +async def _wait_for_event(event): + async def poll(): + while not event.is_set(): + await asyncio.sleep(0.01) - assert raised.value.status_code == 500 + await asyncio.wait_for(poll(), timeout=2) + + +@pytest.mark.asyncio +async def test_cancellation_closes_pending_request_without_repeating_mutation(rest_server): + rest_server["stall"] = True + pending = asyncio.create_task( + run_rest_request_async( + "POST", "/v2/pods", api_key="key", json={"name": "pod"}, timeout=5 + ) + ) + try: + await _wait_for_event(rest_server["started"]) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(pending, timeout=1) + await _wait_for_event(rest_server["disconnected"]) + assert rest_server["hits"] == 1 + finally: + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_timeout_closes_pending_request_without_repeating_mutation(rest_server): + rest_server["stall"] = True + with pytest.raises(asyncio.TimeoutError): + await run_rest_request_async( + "POST", "/v2/pods", api_key="key", json={"name": "pod"}, timeout=0.5 + ) + await _wait_for_event(rest_server["disconnected"]) + assert rest_server["hits"] == 1 diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py index 3bdee6ab6..2e14c965e 100644 --- a/tests/test_apps/test_api_client.py +++ b/tests/test_apps/test_api_client.py @@ -75,30 +75,151 @@ async def test_mutation_transport_failure_is_not_retried(self, failure): class TestEndpoints: - async def test_save_endpoint(self): - client, patcher = _client_with( - {"saveEndpoint": {"id": "ep1", "name": "chat"}} + async def test_dev_endpoint_preserves_runtime_and_placement(self, monkeypatch): + from runpod.apps import App + from runpod.apps.dev import _endpoint_input + from runpod.apps.spec import ResourceKind, ResourceSpec + monkeypatch.setattr("runpod.apps.app._REGISTRY", []) + + spec = ResourceSpec( + kind=ResourceKind.QUEUE, name="chat", gpu="4090", gpu_count=2, + datacenter="EU-RO-1", workers=(1, 3), idle_timeout=60, ) - with patcher: - result = await client.save_endpoint({"name": "chat"}) - assert result["id"] == "ep1" + payload = _endpoint_input(App("demo"), spec) + payload["template"]["containerRegistryAuthId"] = "registry1" + payload["networkVolumeIds"] = [{"networkVolumeId": "volume1"}] + + async def create(method, path, *, api_key, json): + assert (method, path) == ("POST", "/v2/serverless") + assert json["gpu"]["count"] == 2 + assert json["dataCenterIds"] == ["EU-RO-1"] + assert json["networkVolumes"] == ["volume1"] + assert json["registry"] == "registry1" + assert json["workers"] == {"min": 1, "max": 3, "idleTimeout": 60} + assert json["env"]["RUNPOD_DEV_RESOURCE"] == "chat" + assert "FLASH_RESOURCE_NAME" not in json["env"] + assert "RUNPOD_RESOURCE_NAME" not in json["env"] + return {"id": "ep1", "name": json["name"]} - async def test_delete_endpoint(self): - client, patcher = _client_with({"deleteEndpoint": True}) - with patcher: - assert await client.delete_endpoint("ep1") is True + client = AppsApiClient(api_key="test-key") + with ( + patch("runpod.apps.api.run_rest_request_async", side_effect=create), + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + ): + assert (await client.save_endpoint(payload))["id"] == "ep1" + graphql.assert_not_awaited() + + @pytest.mark.parametrize("capability", [ + {"flashEnvironmentId": "env1"}, + {"modelReferences": [{"name": "model"}]}, + {"schedule": {"cron": "* * * * *"}}, + {"template": {"name": "user-template", "imageName": "image"}}, + {"instanceIds": ["cpu3c-1-2"]}, + {"instanceIds": ["cpu3c-2-4", "cpu3c-1-2"]}, + ]) + async def test_unsupported_capability_remains_one_atomic_graphql_save(self, capability): + payload = { + "name": "chat", "template": {"imageName": "image"}, + "scalerType": "QUEUE_DELAY", "scalerValue": 4, + **capability, + } + client = AppsApiClient(api_key="test-key") + graphql = _respond({"saveEndpoint": {"id": "ep1"}}) + with ( + patch("runpod.apps.api.run_graphql_query_async", graphql), + patch("runpod.apps.api.run_rest_request_async", AsyncMock()) as rest, + ): + await client.save_endpoint(payload) + rest.assert_not_awaited() + expected_template = {"name": payload["name"], **payload["template"]} + assert graphql.call_args.kwargs["variables"]["input"] == { + **payload, "template": expected_template, + } + assert payload["template"] == capability.get("template", {"imageName": "image"}) - async def test_delete_failure_is_not_reported_as_success(self): - client, patcher = _client_with({"deleteEndpoint": False}) - with patcher, pytest.raises(QueryError, match="ep1"): - await client.delete_endpoint("ep1") + @pytest.mark.parametrize("routing", ["QB", "LB"]) + async def test_request_count_preserves_only_effective_idle_timeout(self, routing): + client = AppsApiClient(api_key="test-key") + transport = AsyncMock(return_value={"id": "ep1"}) + with patch("runpod.apps.api.run_rest_request_async", transport): + await client.save_endpoint({ + "name": "chat", "type": routing, "template": {"imageName": "image"}, + "scalerType": "REQUEST_COUNT", "scalerValue": 7, "idleTimeout": 60, + "workersMin": 0, "workersMax": 2, + }) + body = transport.call_args.kwargs["json"] + assert body["scaling"] == {"type": "REQUEST_COUNT", "requestCount": 7} + assert body["workers"].get("idleTimeout") == (60 if routing == "LB" else None) + + async def test_rest_update_preserves_cpu_alternatives_and_load_balancer_idle_timeout(self): + client = AppsApiClient(api_key="test-key") + rest = AsyncMock(side_effect=[ + {"id": "ep1", "type": "LOAD_BALANCER", "cpu": [{"id": "cpu5c"}]}, + {"id": "ep1"}, + ]) + with ( + patch("runpod.apps.api.run_rest_request_async", rest), + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + ): + await client.save_endpoint({ + "id": "ep1", "type": "LB", + "instanceIds": ["cpu5c-2-4", "cpu5g-4-16"], + "scalerType": "REQUEST_COUNT", "scalerValue": 7, "idleTimeout": 60, + "template": {"env": [{"key": "RUNPOD_DEV_GENERATION", "value": "2"}]}, + }) + patch_request = rest.call_args + assert patch_request.args == ("PATCH", "/v2/serverless/ep1") + body = patch_request.kwargs["json"] + assert body["cpu"] == [ + {"id": "cpu5c", "vcpuCount": 2}, {"id": "cpu5g", "vcpuCount": 4}, + ] + assert body["workers"]["idleTimeout"] == 60 + assert body["env"] == {"RUNPOD_DEV_GENERATION": "2"} + graphql.assert_not_awaited() - async def test_list_my_endpoints(self): - client, patcher = _client_with( - {"myself": {"endpoints": [{"id": "ep1"}]}} - ) - with patcher: - assert await client.list_my_endpoints() == [{"id": "ep1"}] + async def test_routing_change_remains_atomic_graphql_operation(self): + client = AppsApiClient(api_key="test-key") + rest = AsyncMock(return_value={"id": "ep1", "type": "QUEUE"}) + graphql = _respond({"saveEndpoint": {"id": "ep1"}}) + with ( + patch("runpod.apps.api.run_rest_request_async", rest), + patch("runpod.apps.api.run_graphql_query_async", graphql), + ): + await client.save_endpoint({"id": "ep1", "type": "LB"}) + assert rest.call_args.args == ("GET", "/v2/serverless/ep1") + assert rest.await_count == 1 + assert graphql.call_args.kwargs["variables"]["input"]["type"] == "LB" + + @pytest.mark.parametrize("failure", [ + aiohttp.ClientError("response lost"), QueryError("denied", status_code=403), + ]) + async def test_rest_save_failure_never_retries_or_falls_back(self, failure): + rest = AsyncMock(side_effect=failure) + client = AppsApiClient(api_key="test-key") + with ( + patch("runpod.apps.api.run_rest_request_async", rest), + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + pytest.raises(type(failure)), + ): + await client.save_endpoint({ + "name": "chat", "template": {"imageName": "image"}, + "scalerType": "QUEUE_DELAY", "scalerValue": 4, + }) + assert rest.await_count == 1 + graphql.assert_not_awaited() + + @pytest.mark.parametrize("method", ["delete_endpoint", "delete_registry_auth", "terminate_pod"]) + async def test_delete_failure_is_not_reported_as_success(self, method): + client = AppsApiClient(api_key="test-key") + rest = AsyncMock(side_effect=QueryError("denied", status_code=403)) + with ( + patch("runpod.apps.api.run_rest_request_async", rest), + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + pytest.raises(QueryError), + ): + await getattr(client, method)("resource1") + assert rest.await_count == 1 + graphql.assert_not_awaited() class TestTaskPods: @@ -109,9 +230,16 @@ async def test_deploy_gpu_pod(self): ) with patch("runpod.apps.api.run_graphql_query_async", transport): result = await client.deploy_task_pod( - {"gpuTypeIdList": ["NVIDIA GeForce RTX 4090"]}, is_cpu=False + { + "gpuTypeIdList": ["NVIDIA GeForce RTX 4090"], + "terminateAfter": "2026-09-15T12:30:00Z", + "supportPublicIp": True, + }, is_cpu=False ) assert result["id"] == "pod1" + sent = transport.call_args.kwargs["variables"]["input"] + assert sent["terminateAfter"] == "2026-09-15T12:30:00Z" + assert sent["supportPublicIp"] is True async def test_cpu_alternative_can_deploy_in_volume_datacenter(self): client = AppsApiClient(api_key="test-key") @@ -149,14 +277,6 @@ async def test_cpu_alternatives_stop_after_ambiguous_or_invalid_failure(self, fa ) assert transport.await_count == 1 - async def test_terminate_pod(self): - client = AppsApiClient(api_key="test-key") - transport = _respond({"podTerminate": None}) - with patch("runpod.apps.api.run_graphql_query_async", transport): - await client.terminate_pod("pod1") - sent = transport.call_args[1]["variables"]["input"] - assert sent == {"podId": "pod1"} - class TestAppLifecycle: async def test_get_app_by_name(self): @@ -234,74 +354,66 @@ async def test_get_environment_not_found(self): class TestStock: - async def test_gpu_stock_status(self): - client, patcher = _client_with( - {"gpuTypes": [{"lowestPrice": {"stockStatus": "High"}}]} - ) - with patcher: - status = await client.gpu_stock_status( - "NVIDIA GeForce RTX 4090", "US-KS-2" + async def test_gpu_availability_respects_count_product_and_datacenter(self): + async def catalog(method, path, *, api_key, params): + available = ( + params["product"] == "SERVERLESS" and params["count"] == 1 + and params["cloud"] == "SECURE" and params["include"] == "AVAILABILITY" ) - assert status == "High" + return {"dataCenters": [ + {"id": "EU-RO-1", "availability": "HIGH" if available else "LOW"} + ]} - async def test_gpu_stock_no_data(self): - client, patcher = _client_with({"gpuTypes": []}) - with patcher: - assert ( - await client.gpu_stock_status("X", "US-KS-2") is None + client = AppsApiClient(api_key="test-key") + with patch("runpod.apps.api.run_rest_request_async", side_effect=catalog): + assert await client.gpu_stock_status("4090", "EU-RO-1") == "HIGH" + assert await client.gpu_stock_status("4090", "EU-RO-1", 2) == "LOW" + assert await client.gpu_stock_status("4090", "EU-RO-1", pods=True) == "LOW" + assert await client.gpu_stock_status("4090", "US-KS-2") == "NONE" + + async def test_cpu_availability_respects_flavor_size_and_product(self): + async def catalog(method, path, *, api_key, params): + available = ( + path == "/v2/catalog/cpus/cpu3c" and params["vcpuCount"] == 2 + and params["product"] == "POD" and params["include"] == "AVAILABILITY" ) + return {"dataCenters": [ + {"id": "EU-RO-1", "availability": "MEDIUM" if available else "NONE"} + ]} - async def test_gpu_stock_pods_flag(self): client = AppsApiClient(api_key="test-key") - transport = _respond({"gpuTypes": []}) - with patch("runpod.apps.api.run_graphql_query_async", transport): - await client.gpu_stock_status("X", "US-KS-2", pods=True) - sent = transport.call_args[1]["variables"]["lowestPriceInput"] - assert sent["includeAiApi"] is False + with patch("runpod.apps.api.run_rest_request_async", side_effect=catalog): + assert await client.cpu_stock_status("cpu3c-2-4", "EU-RO-1", pods=True) == "MEDIUM" + assert await client.cpu_stock_status("cpu3c-4-8", "EU-RO-1", pods=True) == "NONE" + assert await client.cpu_stock_status("cpu3c-2-4", "EU-RO-1") == "NONE" + + @pytest.mark.parametrize("pods", [False, True]) + @pytest.mark.parametrize("instance_id", ["cpu3c-1-2", "cpu3c-3-6"]) + async def test_legacy_cpu_stock_keeps_exact_configuration(self, pods, instance_id): + async def graphql(query, *, api_key, variables, anonymous): + flavor = variables["cpuFlavorInput"] + specifics = variables["specificsInput"] + available = ( + flavor == {"id": "cpu3c", "isSls": not pods} + and specifics == { + "dataCenterId": "EU-RO-1", "instanceId": instance_id, + "isSls": not pods, + } + ) + return {"data": {"cpuFlavors": [ + {"specifics": {"stockStatus": "High" if available else "None"}} + ]}} - async def test_cpu_stock_status(self): client = AppsApiClient(api_key="test-key") - transport = _respond( - {"cpuFlavors": [{"specifics": {"stockStatus": "Low"}}]} - ) - with patch("runpod.apps.api.run_graphql_query_async", transport): - status = await client.cpu_stock_status("cpu3c-2-4", "US-KS-2") - assert status == "Low" - sent = transport.call_args[1]["variables"] - assert sent["cpuFlavorInput"] == {"id": "cpu3c"} + with ( + patch("runpod.apps.api.run_graphql_query_async", side_effect=graphql), + patch("runpod.apps.api.run_rest_request_async", AsyncMock()) as rest, + ): + assert await client.cpu_stock_status(instance_id, "EU-RO-1", pods=pods) == "High" + rest.assert_not_awaited() class TestVolumesRegistrySecrets: - async def test_list_network_volumes(self): - client, patcher = _client_with( - {"myself": {"networkVolumes": [{"id": "v1"}]}} - ) - with patcher: - assert await client.list_network_volumes() == [{"id": "v1"}] - - async def test_create_network_volume(self): - client, patcher = _client_with( - {"createNetworkVolume": {"id": "v1", "name": "data"}} - ) - with patcher: - result = await client.create_network_volume("data", 10, "US-KS-2") - assert result["id"] == "v1" - - async def test_registry_auth_crud(self): - client, patcher = _client_with( - { - "myself": {"containerRegistryCreds": [{"id": "r1"}]}, - "saveRegistryAuth": {"id": "r1", "name": "dh"}, - "deleteRegistryAuth": True, - } - ) - with patcher: - assert await client.list_registry_auths() == [{"id": "r1"}] - assert ( - await client.create_registry_auth("dh", "user", "pass") - )["id"] == "r1" - assert await client.delete_registry_auth("r1") is True - async def test_secret_crud(self): client, patcher = _client_with( { diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py index b8052b3a5..b202622f6 100644 --- a/tests/test_apps/test_app.py +++ b/tests/test_apps/test_app.py @@ -258,8 +258,8 @@ def web(): api = AsyncMock() api.list_network_volumes.return_value = [ - {"id": "nv-models", "name": "models", "dataCenterId": "EU-RO-1"}, - {"id": "nv-data", "name": "data", "dataCenterId": "EU-RO-1"}, + {"id": "nv-models", "name": "models", "dataCenter": "EU-RO-1"}, + {"id": "nv-data", "name": "data", "dataCenter": "EU-RO-1"}, ] api.cpu_stock_status.return_value = "HIGH" payload = {} diff --git a/tests/test_apps/test_build.py b/tests/test_apps/test_build.py index 9c1e8a8bf..b757867e1 100644 --- a/tests/test_apps/test_build.py +++ b/tests/test_apps/test_build.py @@ -174,24 +174,10 @@ def wait(self): class TestVendor: - def test_targets_worker_platform(self, tmp_path): - captured = {} - with patch( - "runpod.apps.build.subprocess.Popen", - side_effect=_fake_popen(captured), - ): - vendor(tmp_path, ["numpy"], "3.12") - - cmd = captured["cmd"] - assert "--target" in cmd - assert "--python-version" in cmd and "3.12" in cmd - assert "--only-binary" in cmd - assert "manylinux_2_28_x86_64" in cmd - - def test_empty_requirements_no_op(self, tmp_path): - with patch("runpod.apps.build.subprocess.Popen") as popen: - vendor(tmp_path, [], "3.12") - popen.assert_not_called() + @pytest.fixture(autouse=True) + def pip_available(self): + with patch("runpod.apps.build._ensure_pip"): + yield def test_failure_raises_build_error(self, tmp_path): with patch( @@ -227,19 +213,18 @@ def test_large_stderr_does_not_block_progress_or_error(self, tmp_path): original_popen = subprocess.Popen processes = [] expired = threading.Event() + script = ( + "import sys; " + "sys.stderr.write('x' * 262144 + '\\n'); " + "sys.stderr.flush(); " + "print('Collecting numpy'); " + "print('resolver failed', file=sys.stderr); " + "sys.exit(1)" + ) def launch(_cmd, **kwargs): process = original_popen( - [ - sys.executable, - "-c", - "import sys; " - "sys.stderr.write('x' * 262144 + '\\n'); " - "sys.stderr.flush(); " - "print('Collecting numpy'); " - "print('resolver failed', file=sys.stderr); " - "sys.exit(1)", - ], + [sys.executable, "-c", script], **kwargs, ) processes.append(process) diff --git a/tests/test_apps/test_config_params.py b/tests/test_apps/test_config_params.py index 48d087f1c..f92d4f4fa 100644 --- a/tests/test_apps/test_config_params.py +++ b/tests/test_apps/test_config_params.py @@ -2,7 +2,6 @@ import pytest -import runpod from runpod.apps import App from runpod.apps.app import _clear_registry from runpod.apps.dev import _endpoint_input diff --git a/tests/test_apps/test_dev.py b/tests/test_apps/test_dev.py index cf38d8bf2..52a82e7b0 100644 --- a/tests/test_apps/test_dev.py +++ b/tests/test_apps/test_dev.py @@ -72,6 +72,9 @@ def q(): assert env["RUNPOD_DEV_GENERATION"] == "1" # nested .remote() support: dev-session marker always present assert env["RUNPOD_DEV_APP"] == app.name + assert env["RUNPOD_DEV_RESOURCE"] == q.spec.name + assert "FLASH_RESOURCE_NAME" not in env + assert "RUNPOD_RESOURCE_NAME" not in env def test_api_key_forwarded_when_configured(self, monkeypatch): monkeypatch.setenv("RUNPOD_API_KEY", "rpa_test123") diff --git a/tests/test_apps/test_logs.py b/tests/test_apps/test_logs.py index fffc9df02..a49510f4a 100644 --- a/tests/test_apps/test_logs.py +++ b/tests/test_apps/test_logs.py @@ -3,15 +3,16 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +import pytest +from aiohttp import web + from runpod.apps.logs import pod_logs, stream_pod_logs, tail_summary class TestTailSummary: def test_empty(self): assert tail_summary({}) == "(no logs available)" - assert tail_summary({"container": [], "system": []}) == ( - "(no logs available)" - ) + assert tail_summary({"container": [], "system": []}) == ("(no logs available)") def test_container_only(self): out = tail_summary({"container": ["a", "b"], "system": []}) @@ -49,9 +50,7 @@ async def test_snapshot_normalizes_nulls(self, monkeypatch): monkeypatch.setenv("RUNPOD_API_KEY", "test-key") response = MagicMock() response.raise_for_status = MagicMock() - response.json = AsyncMock( - return_value={"container": ["hello"], "system": None} - ) + response.json = AsyncMock(return_value={"container": ["hello"], "system": None}) session_cm, session = _mock_session(response) with patch("aiohttp.ClientSession", return_value=session_cm): @@ -63,18 +62,6 @@ async def test_snapshot_normalizes_nulls(self, monkeypatch): headers = session.get.call_args[1]["headers"] assert headers["Authorization"] == "Bearer test-key" - async def test_log_type_param(self, monkeypatch): - monkeypatch.setenv("RUNPOD_API_KEY", "test-key") - response = MagicMock() - response.raise_for_status = MagicMock() - response.json = AsyncMock(return_value={}) - session_cm, session = _mock_session(response) - - with patch("aiohttp.ClientSession", return_value=session_cm): - await pod_logs("pod123", log_type="system") - - assert session.get.call_args[1]["params"] == {"type": "system"} - class _FakeContent: def __init__(self, lines): @@ -94,14 +81,14 @@ async def test_parses_sse_frames(self, monkeypatch): monkeypatch.setenv("RUNPOD_API_KEY", "test-key") frames = [ b": heartbeat\n", - b"data: " + json.dumps( - {"source": "container", "line": "hi", "ts": "t1"} - ).encode() + b"\n", + b"data: " + + json.dumps({"source": "container", "line": "hi", "ts": "t1"}).encode() + + b"\n", b"garbage\n", b"data: not-json\n", - b"data: " + json.dumps( - {"source": "system", "line": "boot", "ts": "t2"} - ).encode() + b"\n", + b"data: " + + json.dumps({"source": "system", "line": "boot", "ts": "t2"}).encode() + + b"\n", ] response = MagicMock() response.raise_for_status = MagicMock() @@ -119,19 +106,63 @@ async def test_parses_sse_frames(self, monkeypatch): assert params["stream"] == "true" assert params["tail"] == "100" - async def test_since_param(self, monkeypatch): + @pytest.mark.parametrize( + "endpoint_id,log_type,owner", + [(None, "system", "pod"), ("ep1", "container", "worker")], + ) + async def test_rest_stream_selects_resource_source_and_window( + self, monkeypatch, endpoint_id, log_type, owner + ): monkeypatch.setenv("RUNPOD_API_KEY", "test-key") - response = MagicMock() - response.raise_for_status = MagicMock() - response.content = _FakeContent([]) - session_cm, session = _mock_session(response) - - with patch("aiohttp.ClientSession", return_value=session_cm): - async for _ in stream_pod_logs( - "pod123", since="2026-01-01T00:00:00Z", tail=5 - ): - pass - params = session.get.call_args[1]["params"] - assert params["since"] == "2026-01-01T00:00:00Z" - assert params["tail"] == "5" + async def logs(request): + if request.headers.get("Authorization") != "Bearer test-key": + raise web.HTTPUnauthorized() + resource = "worker" if "serverless" in request.path else "pod" + entries = [ + {"source": source, "line": f"{resource} {index}", "ts": timestamp} + for source in ("container", "system") + for index, timestamp in enumerate( + ( + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + "2026-01-01T00:00:02Z", + ) + ) + if source == request.query["source"] + and timestamp >= request.query["since"] + ][-int(request.query["tail"]) :] + return web.Response( + text="".join(f"data: {json.dumps(event)}\n\n" for event in entries), + content_type="text/event-stream", + ) + + app = web.Application() + app.router.add_get("/v2/pods/pod1/logs", logs) + app.router.add_get("/v2/serverless/ep1/workers/pod1/logs", logs) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + monkeypatch.setenv("RUNPOD_API_BASE_URL", f"http://127.0.0.1:{port}") + try: + events = [ + event + async for event in stream_pod_logs( + "pod1", + endpoint_id=endpoint_id, + log_type=log_type, + since="2026-01-01T00:00:01Z", + tail=1, + ) + ] + finally: + await runner.cleanup() + assert events == [ + { + "source": log_type, + "line": f"{owner} 2", + "ts": "2026-01-01T00:00:02Z", + } + ] diff --git a/tests/test_apps/test_monitor.py b/tests/test_apps/test_monitor.py index f8546dfa6..3de8321e3 100644 --- a/tests/test_apps/test_monitor.py +++ b/tests/test_apps/test_monitor.py @@ -81,29 +81,29 @@ class TestReportCounts: def test_first_steady_snapshot_suppressed(self): sink = Sink() monitor = WorkerMonitor("ep1", "calc", sink) - monitor._report_counts({"workers": {"ready": 2, "running": 1}}) + monitor._report_counts({"summary": {"idle": 2, "running": 1}}) assert sink.events == [] def test_first_snapshot_with_initializing_reported(self): sink = Sink() monitor = WorkerMonitor("ep1", "calc", sink) - monitor._report_counts({"workers": {"initializing": 1}}) + monitor._report_counts({"summary": {"initializing": 1}}) assert len(sink.events) == 1 assert sink.events[0][2]["initializing"] == 1 def test_transition_reported_once(self): sink = Sink() monitor = WorkerMonitor("ep1", "calc", sink) - monitor._report_counts({"workers": {"initializing": 1}}) - monitor._report_counts({"workers": {"initializing": 1}}) - monitor._report_counts({"workers": {"ready": 1}}) + monitor._report_counts({"summary": {"initializing": 1}}) + monitor._report_counts({"summary": {"initializing": 1}}) + monitor._report_counts({"summary": {"idle": 1}}) assert len(sink.events) == 2 assert sink.events[1][2]["ready"] == 1 def test_malformed_payload_ignored(self): sink = Sink() monitor = WorkerMonitor("ep1", "calc", sink) - monitor._report_counts({"workers": None}) + monitor._report_counts({"summary": None}) monitor._report_counts({}) assert sink.events == [] @@ -243,27 +243,23 @@ async def parked(pod_id, **kwargs): class TestMonitorLifecycle: - async def test_start_without_metrics_key_is_noop(self): - monitor = WorkerMonitor("ep1", "calc", Sink()) - await monitor.start() - assert monitor._tasks == [] - await monitor.stop() - async def test_metrics_polling_reports_counts(self): sink = Sink() - monitor = WorkerMonitor("ep1", "calc", sink, metrics_key="mk") - payload = {"workers": {"initializing": 1, "ready": 0}} + api = AsyncMock() + polled = asyncio.Event() - async def fake_get(url, headers, timeout): - assert headers["Authorization"] == "Bearer mk" - return payload + async def workers(endpoint_id): + polled.set() + return {"summary": {"initializing": 1, "idle": 2}} - with patch("runpod.apps.utils.network.get_json", fake_get): - await monitor.start() - await asyncio.sleep(0.05) - await monitor.stop() + api.endpoint_workers.side_effect = workers + monitor = WorkerMonitor("ep1", "calc", sink, api=api) + await monitor.start() + await asyncio.wait_for(polled.wait(), timeout=1) + await monitor.stop() - statuses = [e for e in sink.events if e[0] == "worker_status"] - assert statuses - assert statuses[0][2]["initializing"] == 1 + assert sink.events == [("worker_status", "calc", { + "initializing": 1, "ready": 2, "running": 0, + "throttled": 0, "unhealthy": 0, + })] diff --git a/tests/test_apps/test_placement.py b/tests/test_apps/test_placement.py index 8716c6a02..7bfe4824c 100644 --- a/tests/test_apps/test_placement.py +++ b/tests/test_apps/test_placement.py @@ -31,7 +31,11 @@ def _stock(gpu=None, cpu=None): # fixtures stay hardware-shaped rather than plane-shaped stock._gpu = {(gpu_id, 1, dc): score for (gpu_id, dc), score in (gpu or {}).items()} stock._gpu_pod = dict(stock._gpu) - stock._cpu = cpu or {} + stock._cpu = { + (cpu_id, dc, pods): score + for (cpu_id, dc), score in (cpu or {}).items() + for pods in (False, True) + } return stock @@ -210,3 +214,21 @@ async def gpu_stock(gpu_id, dc, gpu_count=1, pods=False): solve_placement( [pair], stock, volume_name="fixed", existing_dc="EU-RO-1" ) + + async def test_cpu_task_and_endpoint_require_shared_product_stock(self): + async def cpu_stock(instance_id, dc, *, pods=False): + if dc == "EU-RO-1": + return "HIGH" if not pods else "NONE" + return "MEDIUM" if dc == "US-KS-2" else "NONE" + + api = AsyncMock() + api.cpu_stock_status.side_effect = cpu_stock + stock = StockMap(api) + endpoint = ResourceSpec(kind=ResourceKind.QUEUE, name="serve", cpu="cpu5c-2-4") + task = ResourceSpec(kind=ResourceKind.TASK, name="task", cpu="cpu5c-2-4") + await stock.fetch(_hardware_keys(endpoint)) + await stock.fetch(_hardware_keys(task)) + assert solve_placement([endpoint], stock, volume_name="one") == "EU-RO-1" + assert solve_placement([endpoint, task], stock, volume_name="shared") == "US-KS-2" + with pytest.raises(PlacementError): + solve_placement([task], stock, volume_name="fixed", existing_dc="EU-RO-1") diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py index 9f4753a17..118c521d8 100644 --- a/tests/test_apps/test_targets.py +++ b/tests/test_apps/test_targets.py @@ -22,11 +22,6 @@ unwrap_job_output, ) -# http.server's per-request sockets are collected lazily; the unraisable -# checker flags them as ResourceWarnings non-deterministically -pytestmark = pytest.mark.filterwarnings( - "ignore::pytest.PytestUnraisableExceptionWarning" -) class TestApiKey: @@ -236,8 +231,12 @@ def log_message(self, *args): f"http://127.0.0.1:{server.server_address[1]}", ) server.state = state - yield server - server.shutdown() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join() class TestSentinelTarget: @@ -336,7 +335,7 @@ async def live_api_server(monkeypatch): from aiohttp import web from yarl import URL - state = {"uploads": 0, "calls": 0, "reject_sync": False, "handler": None} + state = {"uploads": 0, "calls": 0, "reject_sync": False, "ready": False} entered = asyncio.Event() release = asyncio.Event() @@ -347,18 +346,16 @@ async def sync(request): if state["reject_sync"]: state["reject_sync"] = False raise web.HTTPServiceUnavailable() - payload = await request.json() - namespace = {} - exec(payload["source"], namespace) - state["handler"] = namespace["calculate"] + await request.read() + state["ready"] = True return web.json_response({"status": "synced"}) async def calculate(request): state["calls"] += 1 - if state["handler"] is None: + if not state["ready"]: raise web.HTTPConflict() body = await request.json() - return web.json_response({"result": state["handler"](body["value"])}) + return web.json_response({"result": body["value"] * 2}) app = web.Application() app.router.add_post("/_runpod/sync", sync) diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 3e569186b..df2370327 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -228,8 +228,6 @@ def log_message(self, *args): return httpd, hits def _poll(self, url): - import asyncio - from runpod.apps.tasks import TaskExecution spec = ResourceSpec(kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"]) @@ -395,7 +393,7 @@ async def test_shared_volume_placement_accounts_for_siblings(self): api = AsyncMock() api.list_network_volumes.return_value = [] api.cpu_stock_status.side_effect = ( - lambda instance, dc: "High" if dc == "US-IL-1" else "Low" + lambda instance, dc, *, pods=False: "High" if dc == "US-IL-1" else "Low" ) api.create_network_volume.return_value = {"id": "volume-1"} api.deploy_task_pod.return_value = {"id": "pod-9"} diff --git a/tests/test_apps/test_volume.py b/tests/test_apps/test_volume.py index e606028e2..ed346a175 100644 --- a/tests/test_apps/test_volume.py +++ b/tests/test_apps/test_volume.py @@ -28,7 +28,7 @@ def _api(volumes=None, created=None): "id": "nv-new", "name": "models", "size": 50, - "dataCenterId": "EU-RO-1", + "dataCenter": "EU-RO-1", } # stock queries: everything available everywhere api.gpu_stock_status.return_value = "High" @@ -62,7 +62,7 @@ class TestVolumeResolver: def test_existing_by_name(self): api = _api( volumes=[ - {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"} ] ) resolver = VolumeResolver(api) @@ -75,7 +75,7 @@ def test_existing_by_name(self): def test_existing_by_id(self): api = _api( volumes=[ - {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"} ] ) resolver = VolumeResolver(api) @@ -106,8 +106,8 @@ def test_missing_no_create_raises(self): def test_duplicate_names_raise(self): api = _api( volumes=[ - {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"}, - {"id": "nv-2", "name": "models", "size": 50, "dataCenterId": "US-KS-2"}, + {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"}, + {"id": "nv-2", "name": "models", "size": 50, "dataCenter": "US-KS-2"}, ] ) resolver = VolumeResolver(api) @@ -119,7 +119,7 @@ def test_duplicate_names_raise(self): def test_resolution_cached_per_name(self): api = _api( volumes=[ - {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"} ] ) resolver = VolumeResolver(api) @@ -162,7 +162,7 @@ def test_pod_pins_to_volume_dc(self): api = _api( volumes=[ - {"id": "nv-1", "name": "models", "size": 50, "dataCenterId": "EU-RO-1"} + {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"} ] ) spec = ResourceSpec( From e058478ef0ee0cb65fd39a5dbd040398f80f6dbf Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:26:37 -0700 Subject: [PATCH 107/120] fix: keep worker-local sync calls off the event loop --- runpod/apps/handles.py | 49 ++++++++++++++++++++++++-------- runpod/apps/invoker.py | 11 +++++-- tests/test_apps/test_dispatch.py | 37 +++++++++++++++++++++++- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/runpod/apps/handles.py b/runpod/apps/handles.py index a6e6666bc..b66af1d94 100644 --- a/runpod/apps/handles.py +++ b/runpod/apps/handles.py @@ -18,6 +18,7 @@ async def transcribe(audio_url: str): ... def load_model(): ... # worker startup hook """ +import asyncio import inspect from typing import ( TYPE_CHECKING, @@ -30,7 +31,7 @@ def load_model(): ... # worker startup hook Type, ) -from .context import Context, current_context +from .context import Context, block, current_context from .discovery_state import DiscoveryInvocationError, in_discovery from .errors import InvalidResourceError from .invoker import Invoker, StreamInvoker @@ -64,7 +65,7 @@ def __init__( if stamped and not spec.schedule: spec.schedule = stamped - self.remote = Invoker(self._remote_async) + self.remote = Invoker(self._remote_async, sync_factory=self._remote_sync) self.stream = StreamInvoker(self._stream_async) self.spawn = Invoker(self._spawn_async) self.job = Invoker(self._job_async) @@ -117,22 +118,46 @@ def local(self, *args: Any, **kwargs: Any) -> Any: function is async.""" return self._fn(*args, **kwargs) + def _call_local_sync(self, *args: Any, **kwargs: Any) -> Any: + result = self._fn(*args, **kwargs) + return list(result) if inspect.isgenerator(result) else result + + @staticmethod + async def _await_local_result(result: Any) -> Any: + if inspect.isawaitable(result): + result = await result + if inspect.isasyncgen(result): + return [chunk async for chunk in result] + return list(result) if inspect.isgenerator(result) else result + + def _remote_sync(self, *args: Any, **kwargs: Any) -> Any: + if ( + current_context() is Context.WORKER + and self._is_current_worker() + and not inspect.iscoroutinefunction(self._fn) + and not inspect.isasyncgenfunction(self._fn) + ): + self._guard_discovery() + self.spec.validate() + result = self._call_local_sync(*args, **kwargs) + if inspect.isawaitable(result) or inspect.isasyncgen(result): + return block(self._await_local_result(result)) + return result + return block(self._remote_async(*args, **kwargs)) + async def _remote_async(self, *args: Any, **kwargs: Any) -> Any: self._guard_discovery() self.spec.validate() ctx = current_context() if ctx is Context.WORKER and self._is_current_worker(): - result = self._fn(*args, **kwargs) - if inspect.isawaitable(result): - result = await result - # generators aggregate, matching the deployed worker's - # return_aggregate_stream output for .remote() - if inspect.isasyncgen(result): - return [chunk async for chunk in result] - if inspect.isgenerator(result): - return list(result) - return result + if inspect.iscoroutinefunction(self._fn) or inspect.isasyncgenfunction( + self._fn + ): + result = self._fn(*args, **kwargs) + else: + result = await asyncio.to_thread(self._call_local_sync, *args, **kwargs) + return await self._await_local_result(result) target = await self._app._resolve(self.spec) payload = target.build_payload(self._fn, self.spec, args, kwargs) diff --git a/runpod/apps/invoker.py b/runpod/apps/invoker.py index f44c6585d..d67533425 100644 --- a/runpod/apps/invoker.py +++ b/runpod/apps/invoker.py @@ -10,7 +10,7 @@ returns a sync iterator, `.aio(...)` returns the async iterator. """ -from typing import Any, AsyncIterator, Callable, Coroutine, Iterator +from typing import Any, AsyncIterator, Callable, Coroutine, Iterator, Optional from .context import block @@ -19,12 +19,17 @@ class Invoker: - __slots__ = ("_factory",) + __slots__ = ("_factory", "_sync_factory") - def __init__(self, factory: CoroFactory): + def __init__( + self, factory: CoroFactory, *, sync_factory: Optional[Callable] = None + ): self._factory = factory + self._sync_factory = sync_factory def __call__(self, *args: Any, **kwargs: Any) -> Any: + if self._sync_factory is not None: + return self._sync_factory(*args, **kwargs) return block(self._factory(*args, **kwargs)) def aio(self, *args: Any, **kwargs: Any) -> Coroutine[Any, Any, Any]: diff --git a/tests/test_apps/test_dispatch.py b/tests/test_apps/test_dispatch.py index f091c825c..5e1080ab8 100644 --- a/tests/test_apps/test_dispatch.py +++ b/tests/test_apps/test_dispatch.py @@ -1,5 +1,8 @@ """tests for context detection and remote dispatch.""" +import os +import subprocess +import sys from unittest.mock import AsyncMock, patch import pytest @@ -210,6 +213,39 @@ def q(x): assert q.remote(4) == 40 + def test_worker_local_recursion_from_sync_and_async_callers(self): + script = """ +import asyncio +import runpod + +app = runpod.App("self-smoke") + +@app.queue(cpu="cpu3c-2-4") +def countdown(n): + return 0 if n == 0 else 1 + countdown.remote(n - 1) + +assert countdown.remote(3) == 3 + +async def main(): + results = await asyncio.gather(*(countdown.remote.aio(4) for _ in range(40))) + assert results == [4] * 40 + +asyncio.run(main()) +""" + subprocess.run( + [sys.executable, "-c", script], + env={ + **os.environ, + "RUNPOD_POD_ID": "local-worker", + "RUNPOD_DEV_APP": "self-smoke", + "RUNPOD_DEV_RESOURCE": "countdown", + }, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + def test_worker_calls_sibling_via_sentinel(self, monkeypatch): monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") monkeypatch.setenv("FLASH_RESOURCE_NAME", "other") @@ -409,7 +445,6 @@ async def caller(): class TestModuleSourceShipping: def test_whole_module_ships(self, tmp_path): import importlib.util - import sys mod_file = tmp_path / "shipmod.py" mod_file.write_text( From bb6dfcb629499115c57925084d193c75c04bd8ee Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:03:01 -0700 Subject: [PATCH 108/120] fix: preserve queued calls through worker cold starts --- runpod/apps/targets.py | 66 ++++++-------------- tests/test_apps/test_retry.py | 6 +- tests/test_apps/test_targets.py | 107 ++++++++++++++++++-------------- 3 files changed, 81 insertions(+), 98 deletions(-) diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 3f4c3ff67..08a69c2ae 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -82,7 +82,7 @@ def build_job_options( ) -> Dict[str, Any]: """shape per-job queue options into the wire payload. - these ride alongside `input` on the run/runsync request, matching + these ride alongside `input` on a queue job request, matching the raw data-plane api's job payload. """ options: Dict[str, Any] = {} @@ -102,9 +102,7 @@ def build_job_options( return options -def merge_job_options( - base: Dict[str, Any], new: Dict[str, Any] -) -> Dict[str, Any]: +def merge_job_options(base: Dict[str, Any], new: Dict[str, Any]) -> Dict[str, Any]: """combine two option sets; policy fields accumulate rather than overwrite so chained with_options calls compose.""" merged = dict(base) @@ -117,7 +115,7 @@ def merge_job_options( def unwrap_job_output(data: Dict[str, Any]) -> Any: - """extract output from a runsync-style response, raising on failure.""" + """extract output from a queue response, raising on failure.""" output = data.get("output", data) if data.get("status") == "FAILED" or data.get("error"): err = data.get("error") @@ -165,9 +163,7 @@ async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: async def job_status(self, job_id: str) -> Dict[str, Any]: """fetch the current state of a submitted job.""" - raise NotImplementedError( - f"{type(self).__name__} does not support job status" - ) + raise NotImplementedError(f"{type(self).__name__} does not support job status") async def cancel_job(self, job_id: str) -> Dict[str, Any]: """cancel a submitted job.""" @@ -177,13 +173,9 @@ async def cancel_job(self, job_id: str) -> Dict[str, Any]: async def retry_job(self, job_id: str) -> Dict[str, Any]: """retry a failed or timed-out job.""" - raise NotImplementedError( - f"{type(self).__name__} does not support job retries" - ) + raise NotImplementedError(f"{type(self).__name__} does not support job retries") - def stream_job( - self, job_id: str, *, timeout: float - ) -> AsyncIterator[Any]: + def stream_job(self, job_id: str, *, timeout: float) -> AsyncIterator[Any]: """yield partial outputs of a generator job as they arrive.""" raise NotImplementedError( f"{type(self).__name__} does not support job streaming" @@ -197,9 +189,7 @@ async def wait( on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: """wait for a submitted job to finish and return its output.""" - raise NotImplementedError( - f"{type(self).__name__} does not support job polling" - ) + raise NotImplementedError(f"{type(self).__name__} does not support job polling") async def request( self, method: str, path: str, body: Any = None, *, timeout: float @@ -324,9 +314,8 @@ async def _wait_terminal( ) -> Dict[str, Any]: """poll /status until the job reaches a terminal state. - runsync returns early (e.g. IN_QUEUE) when the job outlives the sync - window, typically on cold starts; polling covers the rest. on_status, - when given, sees every payload (observability hooks read workerId). + on_status receives every payload, including admission and completion, + so observers can attach to the assigned worker. """ import time @@ -367,7 +356,7 @@ async def _wait_terminal( class QueueClient: """queue data-plane client for a single endpoint id. - owns the run/runsync/status/cancel/retry routes and the lb subdomain + owns the run/status/cancel/retry routes and the lb subdomain for http resources. targets compose it with their own routing headers: the sentinel client carries flash headers, a live client carries plain auth headers. @@ -410,13 +399,10 @@ async def _call( ), ) - async def run(self, payload: Dict[str, Any]) -> Dict[str, Any]: - return await self._call("POST", "run", payload) - - async def runsync( - self, payload: Dict[str, Any], *, timeout: float + async def run( + self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Dict[str, Any]: - return await self._call("POST", "runsync", payload, timeout) + return await self._call("POST", "run", payload, timeout) async def status(self, job_id: str) -> Dict[str, Any]: return await self._call("GET", f"status/{job_id}") @@ -427,9 +413,7 @@ async def cancel(self, job_id: str) -> Dict[str, Any]: async def retry(self, job_id: str) -> Dict[str, Any]: return await self._call("POST", f"retry/{job_id}") - async def stream( - self, job_id: str, *, timeout: float - ) -> AsyncIterator[Any]: + async def stream(self, job_id: str, *, timeout: float) -> AsyncIterator[Any]: """yield partial outputs from /stream until the job is terminal. the route long-polls (wait=) and drains buffered chunks, so each @@ -524,7 +508,7 @@ def _sentinel_headers(self) -> Dict[str, str]: async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: - data = await self._client.runsync(payload, timeout=timeout) + data = await self._client.run(payload, timeout=timeout) data = await self._client.wait(data, timeout=timeout) return self.unwrap(data) @@ -552,9 +536,7 @@ async def wait( timeout: float = DEFAULT_TIMEOUT_SECONDS, on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: - data = await self._client.wait( - job_data, timeout=timeout, on_status=on_status - ) + data = await self._client.wait(job_data, timeout=timeout, on_status=on_status) return self.unwrap(data) async def request( @@ -631,9 +613,7 @@ async def _sync_source(self, timeout: float) -> None: } if self._source_spec is not None: payload["dependencies"] = self._source_spec.dependencies - payload["system_dependencies"] = ( - self._source_spec.system_dependencies - ) + payload["system_dependencies"] = self._source_spec.system_dependencies await _post_json(url, payload, _headers(), timeout) self._synced_hash = digest @@ -710,13 +690,7 @@ async def invoke( if monitor is not None: await monitor.start() try: - if monitor is not None: - # async submit + status polling: runsync would hold the - # connection, hiding the workerId until completion and - # starving the live worker feed - data = await self._client.run(payload) - else: - data = await self._client.runsync(payload, timeout=timeout) + data = await self._client.run(payload, timeout=timeout) data = await self._client.wait( data, timeout=timeout, @@ -769,9 +743,7 @@ async def wait( timeout: float = DEFAULT_TIMEOUT_SECONDS, on_status: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> Any: - data = await self._client.wait( - job_data, timeout=timeout, on_status=on_status - ) + data = await self._client.wait(job_data, timeout=timeout, on_status=on_status) return self.unwrap(data) async def request( diff --git a/tests/test_apps/test_retry.py b/tests/test_apps/test_retry.py index 6a7f159a1..e56b28427 100644 --- a/tests/test_apps/test_retry.py +++ b/tests/test_apps/test_retry.py @@ -134,7 +134,7 @@ async def test_connection_error_retried(unused_tcp_port): await _request_json("GET", url, {}, timeout=5) -@pytest.mark.parametrize("operation", ["run", "runsync", "retry"]) +@pytest.mark.parametrize("operation", ["run", "retry"]) @pytest.mark.parametrize("failure", [503, "disconnect"]) async def test_submission_is_not_repeated_after_ambiguous_failure( flaky_server, monkeypatch, operation, failure @@ -148,9 +148,7 @@ async def test_submission_is_not_repeated_after_ambiguous_failure( flaky_server.state["script"] = [failure] client = QueueClient("endpoint", lambda: {}) with pytest.raises(aiohttp.ClientError): - if operation == "runsync": - await client.runsync({"input": {}}, timeout=10) - elif operation == "retry": + if operation == "retry": await client.retry("job") else: await client.run({"input": {}}) diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py index 118c521d8..9cd948107 100644 --- a/tests/test_apps/test_targets.py +++ b/tests/test_apps/test_targets.py @@ -23,7 +23,6 @@ ) - class TestApiKey: def test_env_var(self, monkeypatch): monkeypatch.setenv("RUNPOD_API_KEY", "sk-env") @@ -49,9 +48,7 @@ class TestUrlHelpers: def test_lb_domain(self, monkeypatch): import runpod - monkeypatch.setattr( - runpod, "endpoint_url_base", "https://api.runpod.ai/v2" - ) + monkeypatch.setattr(runpod, "endpoint_url_base", "https://api.runpod.ai/v2") assert _lb_domain() == "api.runpod.ai" def test_headers(self, monkeypatch): @@ -97,9 +94,9 @@ def fn(): class TestUnwrapJobOutput: def test_completed(self): - assert unwrap_job_output( - {"status": "COMPLETED", "output": {"x": 1}} - ) == {"x": 1} + assert unwrap_job_output({"status": "COMPLETED", "output": {"x": 1}}) == { + "x": 1 + } def test_failed_status(self): with pytest.raises(RemoteExecutionError, match="boom"): @@ -107,9 +104,7 @@ def test_failed_status(self): def test_error_in_output(self): with pytest.raises(RemoteExecutionError, match="oops"): - unwrap_job_output( - {"status": "COMPLETED", "output": {"error": "oops"}} - ) + unwrap_job_output({"status": "COMPLETED", "output": {"error": "oops"}}) def test_failed_generator_preserves_error(self): with pytest.raises(RemoteExecutionError, match="generator broke"): @@ -164,9 +159,7 @@ async def test_polls_to_completion(self): async def test_no_job_id_raises(self): with pytest.raises(RemoteExecutionError, match="missing a job id"): - await _wait_terminal( - "http://x", {"status": "IN_QUEUE"}, {}, timeout=5 - ) + await _wait_terminal("http://x", {"status": "IN_QUEUE"}, {}, timeout=5) async def test_timeout(self): with ( @@ -189,7 +182,7 @@ async def test_timeout(self): @pytest.fixture def local_endpoint(monkeypatch): """local http server standing in for the serverless data plane.""" - state = {"requests": [], "responses": {}} + state = {"requests": [], "responses": {}, "status_codes": {}, "headers": {}} class Handler(BaseHTTPRequestHandler): def _respond(self): @@ -207,7 +200,13 @@ def _respond(self): self.path, {"status": "COMPLETED", "output": {"ok": True}} ) payload = json.dumps(reply).encode() - self.send_response(200) + status = state["status_codes"].get(self.path, 200) + if any( + self.headers.get(key) != value + for key, value in state["headers"].items() + ): + status = 404 + self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() @@ -239,17 +238,38 @@ def log_message(self, *args): thread.join() -class TestSentinelTarget: - async def test_invoke_routes_through_sentinel(self, local_endpoint): - target = SentinelTarget("demo", "default", "chat") - result = await target.invoke({"input": {"prompt": "hi"}}, timeout=10) - assert result == {"ok": True} +@pytest.mark.parametrize("kind", ["sentinel", "live"]) +async def test_invocations_keep_queued_jobs_addressable( + local_endpoint, monkeypatch, kind +): + if kind == "sentinel": + target = SentinelTarget("demo", "production", "chat") + endpoint = SENTINEL_ID + local_endpoint.state["headers"] = { + "X-Flash-App": "demo", + "X-Flash-Environment": "production", + "X-Flash-Endpoint": "chat", + } + else: + target = LiveTarget("ep123", "chat") + endpoint = "ep123" + responses = local_endpoint.state["responses"] + responses[f"/{endpoint}/run"] = {"id": "durable", "status": "IN_QUEUE"} + responses[f"/{endpoint}/runsync"] = {"id": "ephemeral", "status": "IN_QUEUE"} + responses[f"/{endpoint}/status/durable"] = { + "id": "durable", + "status": "COMPLETED", + "output": {"answer": 42}, + } + local_endpoint.state["status_codes"][f"/{endpoint}/status/ephemeral"] = 404 + monkeypatch.setattr("runpod.apps.targets.JOB_PROPAGATION_ATTEMPTS", 1) - request = local_endpoint.state["requests"][0] - assert request["path"] == f"/{SENTINEL_ID}/runsync" - assert request["headers"]["X-Flash-App"] == "demo" - assert request["headers"]["X-Flash-Environment"] == "default" - assert request["headers"]["X-Flash-Endpoint"] == "chat" + result = await target.invoke({"input": {"prompt": "hello"}}, timeout=10) + + assert result == {"answer": 42} + + +class TestSentinelTarget: async def test_submit_and_wait(self, local_endpoint): local_endpoint.state["responses"][f"/{SENTINEL_ID}/run"] = { @@ -423,9 +443,7 @@ def fn(x): def test_unwrap_success_response(self): target = LiveTarget("ep123", "chat") output = {"success": True, "result": None, "json_result": {"x": 1}} - assert target.unwrap( - {"status": "COMPLETED", "output": output} - ) == {"x": 1} + assert target.unwrap({"status": "COMPLETED", "output": output}) == {"x": 1} def test_unwrap_failure_raises(self): target = LiveTarget("ep123", "chat") @@ -435,17 +453,15 @@ def test_unwrap_failure_raises(self): def test_unwrap_passthrough(self): target = LiveTarget("ep123", "chat") - assert target.unwrap( - {"status": "COMPLETED", "output": {"plain": 1}} - ) == {"plain": 1} + assert target.unwrap({"status": "COMPLETED", "output": {"plain": 1}}) == { + "plain": 1 + } def test_unwrap_aggregated_plain_function(self): # live handlers aggregate; a plain function is one unmarked envelope target = LiveTarget("ep123", "chat") output = [{"success": True, "json_result": {"x": 1}}] - assert target.unwrap( - {"status": "COMPLETED", "output": output} - ) == {"x": 1} + assert target.unwrap({"status": "COMPLETED", "output": output}) == {"x": 1} def test_unwrap_aggregated_generator(self): target = LiveTarget("ep123", "chat") @@ -453,9 +469,7 @@ def test_unwrap_aggregated_generator(self): {"success": True, "__stream__": True, "json_result": "a"}, {"success": True, "__stream__": True, "json_result": "b"}, ] - assert target.unwrap( - {"status": "COMPLETED", "output": output} - ) == ["a", "b"] + assert target.unwrap({"status": "COMPLETED", "output": output}) == ["a", "b"] async def test_stream_job_unwraps_chunks(self, local_endpoint): local_endpoint.state["responses"]["/ep123/stream/j1"] = { @@ -470,13 +484,6 @@ async def test_stream_job_unwraps_chunks(self, local_endpoint): chunks = [c async for c in target.stream_job("j1", timeout=10)] assert chunks == ["a", "b"] - async def test_invoke(self, local_endpoint): - target = LiveTarget("ep123", "chat") - data = await target.invoke({"input": {"x": 1}}, timeout=10) - assert data == {"ok": True} - request = local_endpoint.state["requests"][0] - assert request["path"] == "/ep123/runsync" - async def test_job_operations(self, local_endpoint): local_endpoint.state["responses"]["/ep123/status/j1"] = { "id": "j1", @@ -496,7 +503,9 @@ async def test_job_operations(self, local_endpoint): assert (await target.cancel_job("j1"))["status"] == "CANCELLED" assert (await target.retry_job("j1"))["status"] == "IN_QUEUE" - async def test_concurrent_requests_share_completed_source_sync(self, live_api_server): + async def test_concurrent_requests_share_completed_source_sync( + self, live_api_server + ): state, entered, release = live_api_server target = LiveTarget("ep123", "calculate") target.attach_source(self._spec, "calculate", self._spec()) @@ -509,7 +518,9 @@ async def test_concurrent_requests_share_completed_source_sync(self, live_api_se assert await asyncio.gather(first, second) == [{"result": 6}, {"result": 10}] assert state["uploads"] == 1 - async def test_failed_source_sync_prevents_execution_and_can_retry(self, live_api_server): + async def test_failed_source_sync_prevents_execution_and_can_retry( + self, live_api_server + ): import aiohttp state, _, release = live_api_server @@ -520,6 +531,8 @@ async def test_failed_source_sync_prevents_execution_and_can_retry(self, live_ap with pytest.raises(aiohttp.ClientResponseError): await target.request("POST", "/calculate", {"value": 7}) assert state["calls"] == 0 - assert await target.request("POST", "/calculate", {"value": 7}) == {"result": 14} + assert await target.request("POST", "/calculate", {"value": 7}) == { + "result": 14 + } assert state["uploads"] == 2 assert state["calls"] == 1 From b5050491e335d7dec184a744438e4eaad6bfa8c6 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:51:23 -0700 Subject: [PATCH 109/120] fix: use sync-first queue invocation with polling handoff --- README.md | 2 ++ runpod/apps/targets.py | 31 ++++++++++++++---- tests/test_apps/test_retry.py | 28 +++++++++++++++-- tests/test_apps/test_targets.py | 56 +++++++++++++++++++++++---------- 4 files changed, 92 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 132596e05..82a5f46b4 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ rp flash deploy # deploy production endpoints Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for runnable examples and [docs.runpod.io](https://docs.runpod.io) for the full guide. +Queue `.remote()` calls request a synchronous result and poll the same job if the server's wait window expires. Fast results need no client-side polling. Dev sessions use a short sync window to keep worker logs responsive. Use `.spawn()` for an asynchronous job handle. A transport failure before receiving a job ID raises an error without resubmitting the work. + ## Contributing Pull requests and issues are welcome — see the [contributing guide](CONTRIBUTING.md) to get started. diff --git a/runpod/apps/targets.py b/runpod/apps/targets.py index 08a69c2ae..40769eae0 100644 --- a/runpod/apps/targets.py +++ b/runpod/apps/targets.py @@ -356,7 +356,7 @@ async def _wait_terminal( class QueueClient: """queue data-plane client for a single endpoint id. - owns the run/status/cancel/retry routes and the lb subdomain + owns the run/runsync/status/cancel/retry routes and the lb subdomain for http resources. targets compose it with their own routing headers: the sentinel client carries flash headers, a live client carries plain auth headers. @@ -404,6 +404,27 @@ async def run( ) -> Dict[str, Any]: return await self._call("POST", "run", payload, timeout) + async def invoke( + self, + payload: Dict[str, Any], + *, + timeout: float, + on_status: Optional[Callable[[Dict[str, Any]], None]] = None, + ) -> Dict[str, Any]: + import time + + start = time.monotonic() + # leave transport headroom for the server to return a pollable job id. + wait_ms = int(min(90.0, max(1.0, timeout - 1.0)) * 1000) + if on_status is not None: + wait_ms = 1000 + data = await self._call("POST", f"runsync?wait={wait_ms}", payload, timeout) + return await self.wait( + data, + timeout=max(0.0, timeout - (time.monotonic() - start)), + on_status=on_status, + ) + async def status(self, job_id: str) -> Dict[str, Any]: return await self._call("GET", f"status/{job_id}") @@ -508,8 +529,7 @@ def _sentinel_headers(self) -> Dict[str, str]: async def invoke( self, payload: Dict[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS ) -> Any: - data = await self._client.run(payload, timeout=timeout) - data = await self._client.wait(data, timeout=timeout) + data = await self._client.invoke(payload, timeout=timeout) return self.unwrap(data) async def submit(self, payload: Dict[str, Any]) -> Dict[str, Any]: @@ -690,9 +710,8 @@ async def invoke( if monitor is not None: await monitor.start() try: - data = await self._client.run(payload, timeout=timeout) - data = await self._client.wait( - data, + data = await self._client.invoke( + payload, timeout=timeout, on_status=monitor.on_status if monitor else None, ) diff --git a/tests/test_apps/test_retry.py b/tests/test_apps/test_retry.py index e56b28427..abd732d5f 100644 --- a/tests/test_apps/test_retry.py +++ b/tests/test_apps/test_retry.py @@ -1,5 +1,6 @@ """tests for transient-failure retry in the http layer.""" +import asyncio import json import socket import threading @@ -22,11 +23,15 @@ @pytest.fixture def flaky_server(): """local server whose first responses are scripted status codes.""" - state = {"script": [], "hits": 0} + state = {"script": [], "hits": 0, "release": threading.Event()} class Handler(BaseHTTPRequestHandler): def _respond(self): state["hits"] += 1 + if state.get("stall"): + state["release"].wait(5) + self.close_connection = True + return if state.get("disconnect"): self.connection.shutdown(socket.SHUT_RDWR) self.connection.close() @@ -55,8 +60,10 @@ def log_message(self, *args): server.state = state server.url = f"http://127.0.0.1:{server.server_address[1]}/run" yield server + state["release"].set() server.shutdown() server.server_close() + thread.join() async def test_retries_transient_5xx_then_succeeds(flaky_server): @@ -134,7 +141,7 @@ async def test_connection_error_retried(unused_tcp_port): await _request_json("GET", url, {}, timeout=5) -@pytest.mark.parametrize("operation", ["run", "retry"]) +@pytest.mark.parametrize("operation", ["run", "invoke", "retry"]) @pytest.mark.parametrize("failure", [503, "disconnect"]) async def test_submission_is_not_repeated_after_ambiguous_failure( flaky_server, monkeypatch, operation, failure @@ -148,8 +155,23 @@ async def test_submission_is_not_repeated_after_ambiguous_failure( flaky_server.state["script"] = [failure] client = QueueClient("endpoint", lambda: {}) with pytest.raises(aiohttp.ClientError): - if operation == "retry": + if operation == "invoke": + await client.invoke({"input": {}}, timeout=10) + elif operation == "retry": await client.retry("job") else: await client.run({"input": {}}) assert flaky_server.state["hits"] == 1 + + +async def test_sync_transport_timeout_does_not_resubmit(flaky_server, monkeypatch): + import runpod + + monkeypatch.setattr(runpod, "endpoint_url_base", flaky_server.url) + flaky_server.state["stall"] = True + client = QueueClient("endpoint", lambda: {}) + + with pytest.raises(asyncio.TimeoutError): + await client.invoke({"input": {}}, timeout=0.2) + + assert flaky_server.state["hits"] == 1 diff --git a/tests/test_apps/test_targets.py b/tests/test_apps/test_targets.py index 9cd948107..dfe2cbd69 100644 --- a/tests/test_apps/test_targets.py +++ b/tests/test_apps/test_targets.py @@ -182,7 +182,7 @@ async def test_timeout(self): @pytest.fixture def local_endpoint(monkeypatch): """local http server standing in for the serverless data plane.""" - state = {"requests": [], "responses": {}, "status_codes": {}, "headers": {}} + state = {"requests": [], "responses": {}, "headers": {}} class Handler(BaseHTTPRequestHandler): def _respond(self): @@ -196,11 +196,12 @@ def _respond(self): "body": json.loads(body) if body else None, } ) + path = self.path.split("?", 1)[0] reply = state["responses"].get( - self.path, {"status": "COMPLETED", "output": {"ok": True}} + path, {"status": "COMPLETED", "output": {"ok": True}} ) payload = json.dumps(reply).encode() - status = state["status_codes"].get(self.path, 200) + status = 200 if any( self.headers.get(key) != value for key, value in state["headers"].items() @@ -238,10 +239,9 @@ def log_message(self, *args): thread.join() -@pytest.mark.parametrize("kind", ["sentinel", "live"]) -async def test_invocations_keep_queued_jobs_addressable( - local_endpoint, monkeypatch, kind -): +@pytest.fixture(params=["sentinel", "live"]) +def invocation_target(local_endpoint, request): + kind = request.param if kind == "sentinel": target = SentinelTarget("demo", "production", "chat") endpoint = SENTINEL_ID @@ -253,20 +253,44 @@ async def test_invocations_keep_queued_jobs_addressable( else: target = LiveTarget("ep123", "chat") endpoint = "ep123" - responses = local_endpoint.state["responses"] - responses[f"/{endpoint}/run"] = {"id": "durable", "status": "IN_QUEUE"} - responses[f"/{endpoint}/runsync"] = {"id": "ephemeral", "status": "IN_QUEUE"} - responses[f"/{endpoint}/status/durable"] = { - "id": "durable", + return target, endpoint + + +async def test_fast_invocation_returns_without_polling( + local_endpoint, invocation_target +): + target, endpoint = invocation_target + local_endpoint.state["responses"][f"/{endpoint}/runsync"] = { + "id": "sync-job", "status": "COMPLETED", "output": {"answer": 42}, } - local_endpoint.state["status_codes"][f"/{endpoint}/status/ephemeral"] = 404 - monkeypatch.setattr("runpod.apps.targets.JOB_PROPAGATION_ATTEMPTS", 1) + local_endpoint.state["responses"][f"/{endpoint}/run"] = { + "id": "async-job", + "status": "IN_QUEUE", + } - result = await target.invoke({"input": {"prompt": "hello"}}, timeout=10) + assert await target.invoke({"input": {}}, timeout=10) == {"answer": 42} + assert len(local_endpoint.state["requests"]) == 1 + + +async def test_sync_wait_hands_off_the_same_job(local_endpoint, invocation_target): + target, endpoint = invocation_target + responses = local_endpoint.state["responses"] + responses[f"/{endpoint}/runsync"] = {"id": "sync-job", "status": "IN_QUEUE"} + responses[f"/{endpoint}/status/sync-job"] = { + "id": "sync-job", + "status": "COMPLETED", + "output": {"answer": 42}, + } - assert result == {"answer": 42} + assert await target.invoke({"input": {}}, timeout=10) == {"answer": 42} + submissions = [ + request + for request in local_endpoint.state["requests"] + if request["method"] == "POST" + ] + assert len(submissions) == 1 class TestSentinelTarget: From b83d219949b3589f81827a2517623a24bf8fab8b Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:34:42 -0700 Subject: [PATCH 110/120] fix: enforce provisioning and discovery invariants --- runpod/apps/api.py | 20 +- runpod/apps/app.py | 2 +- runpod/apps/discovery.py | 31 ++- runpod/apps/gpu.py | 8 +- runpod/apps/spec.py | 28 ++- runpod/apps/tasks.py | 29 +-- runpod/cli/utils/rp_info.py | 4 +- tests/test_apps/test_api_client.py | 231 +++++++++++++++------ tests/test_apps/test_app.py | 28 +++ tests/test_apps/test_discovery.py | 31 +-- tests/test_apps/test_tasks.py | 63 +++--- tests/test_cli/test_cli_utils/test_info.py | 45 ++-- 12 files changed, 346 insertions(+), 174 deletions(-) diff --git a/runpod/apps/api.py b/runpod/apps/api.py index b25ee5320..1988b9d1f 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -204,6 +204,11 @@ async def save_endpoint(self, endpoint_input: Dict[str, Any]) -> Dict[str, Any]: f"/v2/serverless/{_path_segment(endpoint_id)}", api_key=self._api_key, ) + if not current: + raise QueryError( + f"endpoint '{endpoint_id}' returned an empty response", + f"GET /v2/serverless/{_path_segment(endpoint_id)}", + ) requested_type = endpoint_input.get("type") expected = ( ("LOAD_BALANCER" if requested_type == "LB" else "QUEUE") @@ -273,9 +278,18 @@ async def deploy_task_pod( """provision with a termination deadline and public-ip requirement.""" pod_input = dict(pod_input) if is_cpu: - instance_ids = pod_input.pop("instanceIds", None) or [ - pod_input.get("instanceId") - ] + instance_ids = pod_input.pop("instanceIds", None) + if instance_ids is None: + instance_ids = [pod_input.pop("instanceId", None)] + if ( + not isinstance(instance_ids, list) + or not instance_ids + or any( + not isinstance(instance_id, str) or not instance_id.strip() + for instance_id in instance_ids + ) + ): + raise ValueError("cpu tasks require non-empty instance ids") mutation = app_mutations.MUTATION_DEPLOY_CPU_POD for index, instance_id in enumerate(instance_ids): candidate = dict(pod_input, instanceId=instance_id) diff --git a/runpod/apps/app.py b/runpod/apps/app.py index f0cc9cb93..855112325 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -44,7 +44,7 @@ def _clear_registry() -> None: def _restore_registry(apps: List["App"]) -> None: - """restore registrations after a failed discovery import.""" + """restore the registry from a discovery snapshot.""" _REGISTRY[:] = apps diff --git a/runpod/apps/discovery.py b/runpod/apps/discovery.py index 2da87dd23..2287e5ede 100644 --- a/runpod/apps/discovery.py +++ b/runpod/apps/discovery.py @@ -64,8 +64,15 @@ def _rollback_modules(before: set, root: Path) -> None: for name in set(sys.modules) - before: module = sys.modules.get(name) source = getattr(module, "__file__", None) - if not source or not Path(source).resolve().is_relative_to(root): + if not source: continue + source_path = Path(source).resolve() + if not source_path.is_relative_to(root) or any( + part in _SKIP_DIRS for part in source_path.relative_to(root).parts + ): + continue + if cached := getattr(module, "__cached__", None): + Path(cached).unlink(missing_ok=True) sys.modules.pop(name, None) parent_name, _, child_name = name.rpartition(".") parent = sys.modules.get(parent_name) @@ -109,14 +116,34 @@ def discover_apps(target: Path) -> List[App]: collected as warnings, and discovery only fails outright when no app was found anywhere (the failures are then the likely cause and are included in the error). + + each scan imports fresh project modules while preserving sdk and + unrelated application registrations. """ target = target.resolve() - before = set(id(a) for a in get_registered_apps()) local_root = target if target.is_dir() else target.parent root = local_root + project_root = local_root while (root / "__init__.py").is_file(): + project_root = root root = root.parent + protected_modules = { + name for name in sys.modules if name == "runpod" or name.startswith("runpod.") + } + _rollback_modules(protected_modules, project_root) + _restore_registry( + [ + app + for app in get_registered_apps() + if not ( + (source := getattr(app, "_source_file", None)) + and Path(source).resolve().is_relative_to(project_root) + ) + ] + ) + before = {id(app) for app in get_registered_apps()} + importlib.invalidate_caches() added_paths = [] for import_root in (local_root, root): root_str = str(import_root) diff --git a/runpod/apps/gpu.py b/runpod/apps/gpu.py index 21a261fb0..79048b229 100644 --- a/runpod/apps/gpu.py +++ b/runpod/apps/gpu.py @@ -77,6 +77,8 @@ class GpuType(Enum): "NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition" ) NVIDIA_H100_80GB_HBM3 = "NVIDIA H100 80GB HBM3" + NVIDIA_H100_PCIe = "NVIDIA H100 PCIe" + NVIDIA_H100_NVL = "NVIDIA H100 NVL" NVIDIA_RTX_A4000 = "NVIDIA RTX A4000" NVIDIA_RTX_A4500 = "NVIDIA RTX A4500" NVIDIA_RTX_4000_ADA_GENERATION = "NVIDIA RTX 4000 Ada Generation" @@ -101,7 +103,11 @@ def all(cls) -> List["GpuType"]: GpuGroup.ADA_24: [GpuType.NVIDIA_GEFORCE_RTX_4090], GpuGroup.ADA_32_PRO: [GpuType.NVIDIA_GEFORCE_RTX_5090], GpuGroup.ADA_48_PRO: [GpuType.NVIDIA_RTX_6000_ADA_GENERATION], - GpuGroup.ADA_80_PRO: [GpuType.NVIDIA_H100_80GB_HBM3], + GpuGroup.ADA_80_PRO: [ + GpuType.NVIDIA_H100_80GB_HBM3, + GpuType.NVIDIA_H100_PCIe, + GpuType.NVIDIA_H100_NVL, + ], GpuGroup.AMPERE_16: [ GpuType.NVIDIA_RTX_A4000, GpuType.NVIDIA_RTX_A4500, diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index 881233e63..c7a186088 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -73,9 +73,7 @@ def normalize_workers(workers: Union[int, Tuple[int, int], None]) -> Tuple[int, if min_w < 0 or max_w < 1: raise InvalidResourceError(f"invalid worker range ({min_w}, {max_w})") if min_w > max_w: - raise InvalidResourceError( - f"workers min ({min_w}) cannot exceed max ({max_w})" - ) + raise InvalidResourceError(f"workers min ({min_w}) cannot exceed max ({max_w})") return (min_w, max_w) @@ -112,12 +110,14 @@ def normalize_cpu( if cpu is None: return None if isinstance(cpu, str): - return [cpu] - if isinstance(cpu, list): - return [str(c) for c in cpu] - raise InvalidResourceError( - f"cpu must be an instance id string or list, got {type(cpu).__name__}" - ) + cpu = [cpu] + if not isinstance(cpu, list): + raise InvalidResourceError( + f"cpu must be an instance id string or list, got {type(cpu).__name__}" + ) + if not cpu or any(not isinstance(c, str) or not c.strip() for c in cpu): + raise InvalidResourceError("cpu must contain non-empty instance ids") + return list(cpu) def normalize_scaler_type(scaler_type: Optional[str]) -> Optional[str]: @@ -302,9 +302,7 @@ def effective_scaler_type(self) -> str: """the scaler the endpoint deploys with: explicit or kind default.""" if self.scaler_type: return self.scaler_type - return ( - "REQUEST_COUNT" if self.kind is ResourceKind.API else "QUEUE_DELAY" - ) + return "REQUEST_COUNT" if self.kind is ResourceKind.API else "QUEUE_DELAY" def to_manifest(self) -> Dict[str, Any]: """serialize for the deploy manifest.""" @@ -332,9 +330,9 @@ def to_manifest(self) -> Dict[str, Any]: for volume in self.volume ] else: - data["networkVolume"] = getattr( - self.volume, "name", None - ) or str(self.volume) + data["networkVolume"] = getattr(self.volume, "name", None) or str( + self.volume + ) if self.env: from .secret import render_env diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 24d897f90..8d98164f9 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -53,9 +53,7 @@ def _key(version: str): return (int(major), int(minor)) floor = _key(minimum) - return sorted( - (v for v in CUDA_VERSIONS if _key(v) >= floor), key=_key - ) + return sorted((v for v in CUDA_VERSIONS if _key(v) >= floor), key=_key) def _device_names(gpu: Optional[List[str]]) -> List[str]: @@ -65,10 +63,8 @@ def _device_names(gpu: Optional[List[str]]) -> List[str]: only understands as the full device list.""" from .gpu import POOLS_TO_TYPES, GpuGroup - if not gpu: - return sorted( - {t.value for types in POOLS_TO_TYPES.values() for t in types} - ) + if not gpu or GpuGroup.ANY.value in gpu: + return sorted({t.value for types in POOLS_TO_TYPES.values() for t in types}) names: List[str] = [] for entry in gpu: try: @@ -89,9 +85,7 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] start the runtime package via dockerArgs. """ spec.validate() - terminate_after = ( - datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME - ).isoformat() + terminate_after = (datetime.now(timezone.utc) + DEFAULT_MAX_LIFETIME).isoformat() from .secret import render_env @@ -105,12 +99,9 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] pod: Dict[str, Any] = { "name": f"task-{task_name}-{secrets.token_hex(4)}", - "imageName": image_for_spec( - spec, python_version=local_python_version() - ), + "imageName": image_for_spec(spec, python_version=local_python_version()), "ports": f"{TASK_PORT}/http", - "containerDiskInGb": spec.container_disk_gb - or (10 if spec.is_cpu else 30), + "containerDiskInGb": spec.container_disk_gb or (10 if spec.is_cpu else 30), "terminateAfter": terminate_after, "supportPublicIp": True, } @@ -150,9 +141,7 @@ def _pod_input(spec: ResourceSpec, token: str, task_name: str) -> Dict[str, Any] if spec.min_cuda_version: # pods have no min-version filter; allow every version at # or above the requested floor - pod["allowedCudaVersions"] = cuda_versions_at_least( - spec.min_cuda_version - ) + pod["allowedCudaVersions"] = cuda_versions_at_least(spec.min_cuda_version) return pod @@ -194,9 +183,7 @@ async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: from ..error import QueryError try: - return await self.api.deploy_task_pod( - pod, is_cpu=self.spec.is_cpu - ) + return await self.api.deploy_task_pod(pod, is_cpu=self.spec.is_cpu) except QueryError as exc: if not is_capacity_error(exc): raise diff --git a/runpod/cli/utils/rp_info.py b/runpod/cli/utils/rp_info.py index c238e185b..ecdd70117 100644 --- a/runpod/cli/utils/rp_info.py +++ b/runpod/cli/utils/rp_info.py @@ -18,9 +18,7 @@ def get_pod_ssh_ip_port(pod_id, timeout=300): status = None while time.time() - start_time < timeout and (pod_ip is None or pod_port is None): - pod = get_pod(pod_id) - if not pod: - raise ValueError(f"pod '{pod_id}' not found") + pod = get_pod(pod_id) or {} status = pod.get("status") direct_ssh = (pod.get("ssh") or {}).get("direct") diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py index 2e14c965e..f66beadfe 100644 --- a/tests/test_apps/test_api_client.py +++ b/tests/test_apps/test_api_client.py @@ -15,9 +15,7 @@ def _respond(data): def _client_with(data): client = AppsApiClient(api_key="test-key") - return client, patch( - "runpod.apps.api.run_graphql_query_async", _respond(data) - ) + return client, patch("runpod.apps.api.run_graphql_query_async", _respond(data)) class TestExecuteRetry: @@ -79,11 +77,17 @@ async def test_dev_endpoint_preserves_runtime_and_placement(self, monkeypatch): from runpod.apps import App from runpod.apps.dev import _endpoint_input from runpod.apps.spec import ResourceKind, ResourceSpec + monkeypatch.setattr("runpod.apps.app._REGISTRY", []) spec = ResourceSpec( - kind=ResourceKind.QUEUE, name="chat", gpu="4090", gpu_count=2, - datacenter="EU-RO-1", workers=(1, 3), idle_timeout=60, + kind=ResourceKind.QUEUE, + name="chat", + gpu="4090", + gpu_count=2, + datacenter="EU-RO-1", + workers=(1, 3), + idle_timeout=60, ) payload = _endpoint_input(App("demo"), spec) payload["template"]["containerRegistryAuthId"] = "registry1" @@ -109,18 +113,25 @@ async def create(method, path, *, api_key, json): assert (await client.save_endpoint(payload))["id"] == "ep1" graphql.assert_not_awaited() - @pytest.mark.parametrize("capability", [ - {"flashEnvironmentId": "env1"}, - {"modelReferences": [{"name": "model"}]}, - {"schedule": {"cron": "* * * * *"}}, - {"template": {"name": "user-template", "imageName": "image"}}, - {"instanceIds": ["cpu3c-1-2"]}, - {"instanceIds": ["cpu3c-2-4", "cpu3c-1-2"]}, - ]) - async def test_unsupported_capability_remains_one_atomic_graphql_save(self, capability): + @pytest.mark.parametrize( + "capability", + [ + {"flashEnvironmentId": "env1"}, + {"modelReferences": [{"name": "model"}]}, + {"schedule": {"cron": "* * * * *"}}, + {"template": {"name": "user-template", "imageName": "image"}}, + {"instanceIds": ["cpu3c-1-2"]}, + {"instanceIds": ["cpu3c-2-4", "cpu3c-1-2"]}, + ], + ) + async def test_unsupported_capability_remains_one_atomic_graphql_save( + self, capability + ): payload = { - "name": "chat", "template": {"imageName": "image"}, - "scalerType": "QUEUE_DELAY", "scalerValue": 4, + "name": "chat", + "template": {"imageName": "image"}, + "scalerType": "QUEUE_DELAY", + "scalerValue": 4, **capability, } client = AppsApiClient(api_key="test-key") @@ -133,7 +144,8 @@ async def test_unsupported_capability_remains_one_atomic_graphql_save(self, capa rest.assert_not_awaited() expected_template = {"name": payload["name"], **payload["template"]} assert graphql.call_args.kwargs["variables"]["input"] == { - **payload, "template": expected_template, + **payload, + "template": expected_template, } assert payload["template"] == capability.get("template", {"imageName": "image"}) @@ -142,36 +154,55 @@ async def test_request_count_preserves_only_effective_idle_timeout(self, routing client = AppsApiClient(api_key="test-key") transport = AsyncMock(return_value={"id": "ep1"}) with patch("runpod.apps.api.run_rest_request_async", transport): - await client.save_endpoint({ - "name": "chat", "type": routing, "template": {"imageName": "image"}, - "scalerType": "REQUEST_COUNT", "scalerValue": 7, "idleTimeout": 60, - "workersMin": 0, "workersMax": 2, - }) + await client.save_endpoint( + { + "name": "chat", + "type": routing, + "template": {"imageName": "image"}, + "scalerType": "REQUEST_COUNT", + "scalerValue": 7, + "idleTimeout": 60, + "workersMin": 0, + "workersMax": 2, + } + ) body = transport.call_args.kwargs["json"] assert body["scaling"] == {"type": "REQUEST_COUNT", "requestCount": 7} assert body["workers"].get("idleTimeout") == (60 if routing == "LB" else None) - async def test_rest_update_preserves_cpu_alternatives_and_load_balancer_idle_timeout(self): + async def test_rest_update_preserves_cpu_alternatives_and_load_balancer_idle_timeout( + self, + ): client = AppsApiClient(api_key="test-key") - rest = AsyncMock(side_effect=[ - {"id": "ep1", "type": "LOAD_BALANCER", "cpu": [{"id": "cpu5c"}]}, - {"id": "ep1"}, - ]) + rest = AsyncMock( + side_effect=[ + {"id": "ep1", "type": "LOAD_BALANCER", "cpu": [{"id": "cpu5c"}]}, + {"id": "ep1"}, + ] + ) with ( patch("runpod.apps.api.run_rest_request_async", rest), patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, ): - await client.save_endpoint({ - "id": "ep1", "type": "LB", - "instanceIds": ["cpu5c-2-4", "cpu5g-4-16"], - "scalerType": "REQUEST_COUNT", "scalerValue": 7, "idleTimeout": 60, - "template": {"env": [{"key": "RUNPOD_DEV_GENERATION", "value": "2"}]}, - }) + await client.save_endpoint( + { + "id": "ep1", + "type": "LB", + "instanceIds": ["cpu5c-2-4", "cpu5g-4-16"], + "scalerType": "REQUEST_COUNT", + "scalerValue": 7, + "idleTimeout": 60, + "template": { + "env": [{"key": "RUNPOD_DEV_GENERATION", "value": "2"}] + }, + } + ) patch_request = rest.call_args assert patch_request.args == ("PATCH", "/v2/serverless/ep1") body = patch_request.kwargs["json"] assert body["cpu"] == [ - {"id": "cpu5c", "vcpuCount": 2}, {"id": "cpu5g", "vcpuCount": 4}, + {"id": "cpu5c", "vcpuCount": 2}, + {"id": "cpu5g", "vcpuCount": 4}, ] assert body["workers"]["idleTimeout"] == 60 assert body["env"] == {"RUNPOD_DEV_GENERATION": "2"} @@ -190,9 +221,13 @@ async def test_routing_change_remains_atomic_graphql_operation(self): assert rest.await_count == 1 assert graphql.call_args.kwargs["variables"]["input"]["type"] == "LB" - @pytest.mark.parametrize("failure", [ - aiohttp.ClientError("response lost"), QueryError("denied", status_code=403), - ]) + @pytest.mark.parametrize( + "failure", + [ + aiohttp.ClientError("response lost"), + QueryError("denied", status_code=403), + ], + ) async def test_rest_save_failure_never_retries_or_falls_back(self, failure): rest = AsyncMock(side_effect=failure) client = AppsApiClient(api_key="test-key") @@ -201,14 +236,34 @@ async def test_rest_save_failure_never_retries_or_falls_back(self, failure): patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, pytest.raises(type(failure)), ): - await client.save_endpoint({ - "name": "chat", "template": {"imageName": "image"}, - "scalerType": "QUEUE_DELAY", "scalerValue": 4, - }) + await client.save_endpoint( + { + "name": "chat", + "template": {"imageName": "image"}, + "scalerType": "QUEUE_DELAY", + "scalerValue": 4, + } + ) assert rest.await_count == 1 graphql.assert_not_awaited() - @pytest.mark.parametrize("method", ["delete_endpoint", "delete_registry_auth", "terminate_pod"]) + @pytest.mark.parametrize("response", [None, {}]) + async def test_empty_endpoint_read_prevents_mutation(self, response): + client = AppsApiClient(api_key="test-key") + rest = AsyncMock(return_value=response) + with ( + patch("runpod.apps.api.run_rest_request_async", rest), + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + pytest.raises(QueryError), + ): + await client.save_endpoint({"id": "ep1", "name": "chat"}) + + assert [call.args[0] for call in rest.await_args_list] == ["GET"] + graphql.assert_not_awaited() + + @pytest.mark.parametrize( + "method", ["delete_endpoint", "delete_registry_auth", "terminate_pod"] + ) async def test_delete_failure_is_not_reported_as_success(self, method): client = AppsApiClient(api_key="test-key") rest = AsyncMock(side_effect=QueryError("denied", status_code=403)) @@ -234,7 +289,8 @@ async def test_deploy_gpu_pod(self): "gpuTypeIdList": ["NVIDIA GeForce RTX 4090"], "terminateAfter": "2026-09-15T12:30:00Z", "supportPublicIp": True, - }, is_cpu=False + }, + is_cpu=False, ) assert result["id"] == "pod1" sent = transport.call_args.kwargs["variables"]["input"] @@ -267,7 +323,9 @@ async def test_cpu_alternative_can_deploy_in_volume_datacenter(self): @pytest.mark.parametrize( "failure", [QueryError("invalid image"), aiohttp.ClientError("response lost")] ) - async def test_cpu_alternatives_stop_after_ambiguous_or_invalid_failure(self, failure): + async def test_cpu_alternatives_stop_after_ambiguous_or_invalid_failure( + self, failure + ): client = AppsApiClient(api_key="test-key") transport = AsyncMock(side_effect=failure) with patch("runpod.apps.api.run_graphql_query_async", transport): @@ -277,6 +335,24 @@ async def test_cpu_alternatives_stop_after_ambiguous_or_invalid_failure(self, fa ) assert transport.await_count == 1 + @pytest.mark.parametrize( + "pod", + [ + {}, + {"instanceIds": []}, + {"instanceIds": ["cpu3c-2-4", None]}, + ], + ) + async def test_invalid_cpu_selection_cannot_provision(self, pod): + client = AppsApiClient(api_key="test-key") + with ( + patch("runpod.apps.api.run_graphql_query_async", AsyncMock()) as graphql, + pytest.raises(ValueError), + ): + await client.deploy_task_pod(pod, is_cpu=True) + + graphql.assert_not_awaited() + class TestAppLifecycle: async def test_get_app_by_name(self): @@ -303,9 +379,7 @@ async def test_create_environment(self): assert result["id"] == "env1" async def test_list_apps(self): - client, patcher = _client_with( - {"myself": {"flashApps": [{"id": "app1"}]}} - ) + client, patcher = _client_with({"myself": {"flashApps": [{"id": "app1"}]}}) with patcher: assert await client.list_apps() == [{"id": "app1"}] @@ -357,12 +431,16 @@ class TestStock: async def test_gpu_availability_respects_count_product_and_datacenter(self): async def catalog(method, path, *, api_key, params): available = ( - params["product"] == "SERVERLESS" and params["count"] == 1 - and params["cloud"] == "SECURE" and params["include"] == "AVAILABILITY" + params["product"] == "SERVERLESS" + and params["count"] == 1 + and params["cloud"] == "SECURE" + and params["include"] == "AVAILABILITY" ) - return {"dataCenters": [ - {"id": "EU-RO-1", "availability": "HIGH" if available else "LOW"} - ]} + return { + "dataCenters": [ + {"id": "EU-RO-1", "availability": "HIGH" if available else "LOW"} + ] + } client = AppsApiClient(api_key="test-key") with patch("runpod.apps.api.run_rest_request_async", side_effect=catalog): @@ -374,17 +452,27 @@ async def catalog(method, path, *, api_key, params): async def test_cpu_availability_respects_flavor_size_and_product(self): async def catalog(method, path, *, api_key, params): available = ( - path == "/v2/catalog/cpus/cpu3c" and params["vcpuCount"] == 2 - and params["product"] == "POD" and params["include"] == "AVAILABILITY" + path == "/v2/catalog/cpus/cpu3c" + and params["vcpuCount"] == 2 + and params["product"] == "POD" + and params["include"] == "AVAILABILITY" ) - return {"dataCenters": [ - {"id": "EU-RO-1", "availability": "MEDIUM" if available else "NONE"} - ]} + return { + "dataCenters": [ + {"id": "EU-RO-1", "availability": "MEDIUM" if available else "NONE"} + ] + } client = AppsApiClient(api_key="test-key") with patch("runpod.apps.api.run_rest_request_async", side_effect=catalog): - assert await client.cpu_stock_status("cpu3c-2-4", "EU-RO-1", pods=True) == "MEDIUM" - assert await client.cpu_stock_status("cpu3c-4-8", "EU-RO-1", pods=True) == "NONE" + assert ( + await client.cpu_stock_status("cpu3c-2-4", "EU-RO-1", pods=True) + == "MEDIUM" + ) + assert ( + await client.cpu_stock_status("cpu3c-4-8", "EU-RO-1", pods=True) + == "NONE" + ) assert await client.cpu_stock_status("cpu3c-2-4", "EU-RO-1") == "NONE" @pytest.mark.parametrize("pods", [False, True]) @@ -393,23 +481,28 @@ async def test_legacy_cpu_stock_keeps_exact_configuration(self, pods, instance_i async def graphql(query, *, api_key, variables, anonymous): flavor = variables["cpuFlavorInput"] specifics = variables["specificsInput"] - available = ( - flavor == {"id": "cpu3c", "isSls": not pods} - and specifics == { - "dataCenterId": "EU-RO-1", "instanceId": instance_id, - "isSls": not pods, + available = flavor == {"id": "cpu3c", "isSls": not pods} and specifics == { + "dataCenterId": "EU-RO-1", + "instanceId": instance_id, + "isSls": not pods, + } + return { + "data": { + "cpuFlavors": [ + {"specifics": {"stockStatus": "High" if available else "None"}} + ] } - ) - return {"data": {"cpuFlavors": [ - {"specifics": {"stockStatus": "High" if available else "None"}} - ]}} + } client = AppsApiClient(api_key="test-key") with ( patch("runpod.apps.api.run_graphql_query_async", side_effect=graphql), patch("runpod.apps.api.run_rest_request_async", AsyncMock()) as rest, ): - assert await client.cpu_stock_status(instance_id, "EU-RO-1", pods=pods) == "High" + assert ( + await client.cpu_stock_status(instance_id, "EU-RO-1", pods=pods) + == "High" + ) rest.assert_not_awaited() diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py index b202622f6..26d8d25e4 100644 --- a/tests/test_apps/test_app.py +++ b/tests/test_apps/test_app.py @@ -63,6 +63,34 @@ def t(): assert t.spec.cpu == ["cpu5c-2-4"] +@pytest.mark.parametrize("cpu", [[], "", ["cpu3c-2-4", ""], [None]]) +def test_cpu_selection_requires_nonempty_instance_ids(cpu): + app = App("a") + + with pytest.raises(InvalidResourceError): + app.task(cpu=cpu)(lambda: None) + + assert app.resources == {} + + +async def test_remote_rejects_cleared_cpu_selection(monkeypatch): + from unittest.mock import AsyncMock + + monkeypatch.setattr( + "runpod.apps.tasks.TaskExecution.start", + AsyncMock(side_effect=AssertionError("invalid specs must not provision")), + ) + app = App("a") + + @app.task(cpu="cpu3c-2-4") + def task(): + raise AssertionError("invalid specs must not execute") + + task.spec.cpu.clear() + with pytest.raises(InvalidResourceError): + await task.remote.aio() + + def test_gpu_cpu_mutually_exclusive(): app = App("a") with pytest.raises(InvalidResourceError): diff --git a/tests/test_apps/test_discovery.py b/tests/test_apps/test_discovery.py index f70783d94..e133e4dec 100644 --- a/tests/test_apps/test_discovery.py +++ b/tests/test_apps/test_discovery.py @@ -77,14 +77,22 @@ def test_package_relative_imports_preserve_local_handle_identity(tmp_path, singl "def double(value):\n" " return value * FACTOR\n" ) + existing = App("outside-discovery") + helper = package / "helpers.py" + original_mtime = helper.stat().st_mtime_ns + for factor in (2, 3): + helper.write_text(f"FACTOR = {factor}\n") + os.utime(helper, ns=(original_mtime, original_mtime)) - apps = discover_apps(target if single_file else tmp_path) - module = importlib.import_module("discovery_identity_package.main") + apps = discover_apps(target if single_file else tmp_path) + module = importlib.import_module("discovery_identity_package.main") + parent = importlib.import_module("discovery_identity_package") - assert apps == [module.app] - assert get_registered_apps() == apps - assert apps[0].resources["double"] is module.double - assert module.double.local(3) == 6 + assert apps == [module.app] + assert get_registered_apps() == [existing, *apps] + assert parent.app is module.app + assert apps[0].resources["double"] is module.double + assert module.double.local(3) == 3 * factor def test_discovery_restores_existing_invocation_guard(tmp_path, monkeypatch): @@ -98,17 +106,16 @@ def test_discovery_restores_existing_invocation_guard(tmp_path, monkeypatch): assert os.environ[DISCOVERY_ENV] == "outer-scan" -def test_interrupt_aborts_discovery_and_rolls_back_partial_import(tmp_path, monkeypatch): +def test_interrupt_aborts_discovery_and_rolls_back_partial_import( + tmp_path, monkeypatch +): monkeypatch.delenv(DISCOVERY_ENV, raising=False) existing = App("existing") (tmp_path / "a_interrupted.py").write_text( - "from runpod import App\n" - "app = App('partial')\n" - "raise KeyboardInterrupt\n" + "from runpod import App\n" "app = App('partial')\n" "raise KeyboardInterrupt\n" ) (tmp_path / "b_later.py").write_text( - "from pathlib import Path\n" - "Path(__file__).with_suffix('.started').touch()\n" + "from pathlib import Path\n" "Path(__file__).with_suffix('.started').touch()\n" ) with pytest.raises(KeyboardInterrupt): diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index df2370327..d7304467d 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -49,10 +49,7 @@ def test_cpu_pod_input(self): assert pod["instanceIds"] == ["cpu3c-1-2"] from runpod.apps.images import local_python_version - assert ( - pod["imageName"] - == f"runpod/task:py{local_python_version()}-latest" - ) + assert pod["imageName"] == f"runpod/task:py{local_python_version()}-latest" assert pod["ports"] == "8080/http" assert pod["terminateAfter"] env = {e["key"]: e["value"] for e in pod["env"]} @@ -69,10 +66,24 @@ def test_gpu_pod_input(self): # gpu runtime image matched to the local python version from runpod.apps.images import local_python_version - assert ( - pod["imageName"] - == f"runpod/task-gpu:py{local_python_version()}-latest" - ) + assert pod["imageName"] == f"runpod/task-gpu:py{local_python_version()}-latest" + + def test_any_gpu_selects_all_supported_devices(self): + from runpod.apps.gpu import GpuType + + pod = _pod_input(self._spec(gpu="any"), "tok", "t") + + assert set(pod["gpuTypeIdList"]) == {gpu.value for gpu in GpuType.all()} + + @pytest.mark.parametrize("gpu", ["H100", "ADA_80_PRO"]) + def test_h100_selection_includes_all_catalog_variants(self, gpu): + pod = _pod_input(self._spec(gpu=gpu), "tok", "t") + + assert set(pod["gpuTypeIdList"]) == { + "NVIDIA H100 80GB HBM3", + "NVIDIA H100 PCIe", + "NVIDIA H100 NVL", + } def test_custom_image_starts_runtime_package(self, monkeypatch): monkeypatch.setenv("RUNPOD_RUNTIME_PACKAGE_SPEC", "runpod-sdk-runtime==1.2.3") @@ -107,10 +118,9 @@ def test_success_cloudpickle(self): assert unwrap_task_response({"success": True, "result": _b64(42)}) == 42 def test_success_json(self): - assert ( - unwrap_task_response({"success": True, "json_result": {"a": 1}}) - == {"a": 1} - ) + assert unwrap_task_response({"success": True, "json_result": {"a": 1}}) == { + "a": 1 + } def test_failure_raises(self): with pytest.raises(RemoteExecutionError, match="boom"): @@ -165,9 +175,7 @@ def t(): with patch("runpod.apps.tasks.TaskExecution") as MockExec: instance = MockExec.return_value instance.start = AsyncMock() - instance.wait_ready = AsyncMock( - side_effect=TimeoutError("never ready") - ) + instance.wait_ready = AsyncMock(side_effect=TimeoutError("never ready")) instance.terminate = AsyncMock() with pytest.raises(TimeoutError): @@ -297,9 +305,7 @@ def test_none_expands_to_all_devices(self): names = _device_names(None) assert "any" not in names - all_devices = { - t.value for types in POOLS_TO_TYPES.values() for t in types - } + all_devices = {t.value for types in POOLS_TO_TYPES.values() for t in types} assert set(names) == all_devices def test_mixed_pool_and_device(self): @@ -373,9 +379,7 @@ async def test_shared_volume_placement_accounts_for_siblings(self): from runpod.apps.volume import Volume volume = Volume("shared") - spec = self._spec( - volume=volume, datacenter=["US-IL-1", "EU-RO-1"] - ) + spec = self._spec(volume=volume, datacenter=["US-IL-1", "EU-RO-1"]) sibling = ResourceSpec( kind=ResourceKind.QUEUE, name="sibling", @@ -392,8 +396,8 @@ async def test_shared_volume_placement_accounts_for_siblings(self): ) api = AsyncMock() api.list_network_volumes.return_value = [] - api.cpu_stock_status.side_effect = ( - lambda instance, dc, *, pods=False: "High" if dc == "US-IL-1" else "Low" + api.cpu_stock_status.side_effect = lambda instance, dc, *, pods=False: ( + "High" if dc == "US-IL-1" else "Low" ) api.create_network_volume.return_value = {"id": "volume-1"} api.deploy_task_pod.return_value = {"id": "pod-9"} @@ -402,7 +406,6 @@ async def test_shared_volume_placement_accounts_for_siblings(self): assert api.create_network_volume.call_args.kwargs["data_center_id"] == "EU-RO-1" assert api.deploy_task_pod.call_args.args[0]["dataCenterIds"] == ["EU-RO-1"] - async def test_start_resolves_registry_auth(self): from runpod.apps.tasks import TaskExecution @@ -495,9 +498,7 @@ async def test_wait_timeout_terminates_pod(self): execution.poll_result = AsyncMock(return_value=None) with ( patch("runpod.apps.tasks.asyncio.sleep", AsyncMock()), - patch( - "runpod.apps.tasks.time.monotonic", side_effect=[0, 100] - ), + patch("runpod.apps.tasks.time.monotonic", side_effect=[0, 100]), ): with pytest.raises(TimeoutError): await job.wait(timeout=10) @@ -548,7 +549,9 @@ async def test_setup_failure_terminates_pod(self, stage, cancelled): spec = ResourceSpec(kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"]) execution = TaskExecution(spec, api=MagicMock()) pods = set() - failure = asyncio.CancelledError() if cancelled else RuntimeError("setup failed") + failure = ( + asyncio.CancelledError() if cancelled else RuntimeError("setup failed") + ) async def start(): execution.pod_id = "pod-9" @@ -564,9 +567,7 @@ async def terminate(pod_id): execution.wait_ready = AsyncMock( side_effect=failure if stage == "wait_ready" else None ) - execution.submit = AsyncMock( - side_effect=failure if stage == "submit" else None - ) + execution.submit = AsyncMock(side_effect=failure if stage == "submit" else None) with patch("runpod.apps.tasks.TaskExecution", return_value=execution): with pytest.raises(type(failure)): await PodTarget(spec, lambda: None).submit({}) diff --git a/tests/test_cli/test_cli_utils/test_info.py b/tests/test_cli/test_cli_utils/test_info.py index b81cf0193..450337fe6 100644 --- a/tests/test_cli/test_cli_utils/test_info.py +++ b/tests/test_cli/test_cli_utils/test_info.py @@ -1,4 +1,4 @@ -""" Unit testing for runpod.cli.utils.rp_info.py """ +"""Unit testing for runpod.cli.utils.rp_info.py""" from unittest.mock import patch @@ -22,20 +22,33 @@ def test_get_pod_ssh_ip_port_normal(self): assert ip == "127.0.0.1" assert port == 2222 - def test_get_pod_ssh_ip_port_timeout(self): - """Test get_pod_ssh_ip_port timeout""" - with patch("runpod.cli.utils.rp_info.get_pod") as mock_get_pod: - mock_get_pod.return_value = {"status": "RUNNING", "runtime": None} - - with pytest.raises(TimeoutError): - get_pod_ssh_ip_port("pod_id", timeout=0.1) - - mock_get_pod.return_value = {"status": "PROVISIONING"} + def test_waits_through_temporary_pod_absence(self): + ready = { + "status": "RUNNING", + "ssh": {"direct": {"host": "127.0.0.1", "port": 2222}}, + } + with ( + patch("runpod.cli.utils.rp_info.get_pod", side_effect=[None, ready]), + patch("runpod.cli.utils.rp_info.time") as clock, + ): + clock.time.side_effect = [0, 0, 1] + + assert get_pod_ssh_ip_port("pod_id", timeout=2) == ("127.0.0.1", 2222) + + @pytest.mark.parametrize( + "pod", + [ + {"status": "RUNNING", "runtime": None}, + {"status": "PROVISIONING"}, + None, + ], + ) + def test_get_pod_ssh_ip_port_timeout(self, pod): + with ( + patch("runpod.cli.utils.rp_info.get_pod", return_value=pod), + patch("runpod.cli.utils.rp_info.time") as clock, + ): + clock.time.side_effect = [0, 0, 1] with pytest.raises(TimeoutError): - get_pod_ssh_ip_port("pod_id", timeout=0.1) - - mock_get_pod.return_value = {} - - with pytest.raises(ValueError, match="not found"): - get_pod_ssh_ip_port("pod_id", timeout=0.1) + get_pod_ssh_ip_port("pod_id", timeout=1) From d5abe4b11741f4cf3686d2b32d531e34766d2f41 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:04:29 -0700 Subject: [PATCH 111/120] fix: validate sandbox responses and clarify execution semantics --- README.md | 6 ++++-- runpod/api/sandboxes.py | 32 +++++++++++++++++++++++--------- runpod/sandbox/asyncio.py | 10 ++++++---- runpod/sandbox/models.py | 2 +- tests/test_sandbox.py | 28 ++++++++++++++++++++++++++-- 5 files changed, 60 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1fd18ab7d..fe46df00b 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,8 @@ runpod.api_key = "your_runpod_api_key_found_under_settings" ### Sandboxes +Sandboxes are an access-controlled preview. An authorized account and a sandbox-enabled API environment are required; installing the SDK does not grant access. Configure the API origin supplied for your environment with `RUNPOD_API_BASE_URL` or the handle's `base_url` option. + `Sandbox` and `AsyncioSandbox` manage isolated CPU sandboxes through REST API v2. Set `RUNPOD_API_KEY`, assign `runpod.api_key`, or pass `api_key` to a handle. Supply exactly one of `image_name` or `template_id`. ```python @@ -185,7 +187,7 @@ with Sandbox(image_name="python:3.12-slim") as sandbox: print(result.output) ``` -Use `AsyncioSandbox` in asynchronous applications: +Use `AsyncioSandbox` in asynchronous applications. Its name follows the SDK's `AsyncioEndpoint` and `AsyncioJob` convention: ```python import asyncio @@ -204,7 +206,7 @@ asyncio.run(main()) These contexts create a sandbox on entry and terminate it on exit, including when the body raises. `Sandbox.create(...)` and `await AsyncioSandbox.create(...)` return owned handles for explicit lifetime management. Call `terminate()` to release remote compute; `close()` releases local connections only. Handles returned by `get(sandbox_id)` or `list(state=..., labels=...)` are borrowed, so their contexts only close local resources. Async factories and lifecycle methods are awaited. -Handle properties such as `state`, `compute`, and `expires_at` read cached metadata; `refresh()` fetches a current snapshot. Command execution accepts an argument sequence. `check=True` raises `SandboxExecutionError` with the partial output available in `error.result`. Explicit startup rejections are retried within `startup_timeout`; ambiguous transport failures are not replayed. +Handle properties such as `state`, `compute`, and `expires_at` read cached metadata; `refresh()` fetches a current snapshot. Command execution accepts an argument sequence. Like `subprocess.run`, `exec` defaults to `check=False` so callers can inspect command failures through `ExecResult.error`. Use `check=True` when subsequent work depends on success: it raises `SandboxExecutionError` with the partial output available in `error.result`. HTTP errors and malformed responses raise regardless of `check`. Explicit startup rejections are retried within `startup_timeout`; ambiguous transport failures are not replayed. `sandbox.logs(source="container", tail=10)` streams typed log events from the container's main process; `source="system"` selects lifecycle logs. Command output is returned by `exec`, not this stream. Use `with` and regular iteration for synchronous log streams, or `async with` and `async for` for asynchronous streams, to close the connection when stopping early. Preserve an event's `id` and pass it as `last_event_id` to resume a stream, or filter by `since`. diff --git a/runpod/api/sandboxes.py b/runpod/api/sandboxes.py index 74dac1fbd..a529c3fcf 100644 --- a/runpod/api/sandboxes.py +++ b/runpod/api/sandboxes.py @@ -273,8 +273,9 @@ def _get_session(self) -> aiohttp.ClientSession: cookie_jar=aiohttp.DummyCookieJar(), ) # aiohttp otherwise transparently replays disconnected GET/DELETE - # requests. It currently exposes no public switch for this policy. - self._session._retry_connection = False + # requests. it currently exposes no public switch for this policy. + if hasattr(self._session, "_retry_connection"): + self._session._retry_connection = False return self._session async def _request( @@ -296,9 +297,20 @@ async def _request( await _raise_response_error(response, method, path) if response.status == 204: return None - payload = await response.json(content_type=None) + try: + payload = await response.json(content_type=None) + except ValueError as cause: + raise error.QueryError( + "Sandbox API response is not valid JSON", + f"{method} {path}", + status_code=response.status, + ) from cause if not isinstance(payload, dict): - raise ValueError("Sandbox API response must be a JSON object") + raise error.QueryError( + "Sandbox API response must be a JSON object", + f"{method} {path}", + status_code=response.status, + ) return payload async def create(self, body: Mapping[str, Any]) -> dict[str, Any]: @@ -342,11 +354,13 @@ async def terminate(self, sandbox_id: str) -> None: await self._request("DELETE", _sandbox_path(sandbox_id)) async def exec(self, sandbox_id: str, command: Sequence[str]) -> dict[str, Any]: - payload = await self._request( - "POST", f"{_sandbox_path(sandbox_id)}/exec", body={"command": list(command)} - ) - if payload is None: - raise ValueError("Sandbox execution returned no result") + path = f"{_sandbox_path(sandbox_id)}/exec" + payload = await self._request("POST", path, body={"command": list(command)}) + if payload is None or not isinstance(payload.get("output"), str): + raise error.QueryError( + "Sandbox execution response must contain a string 'output'", + f"POST {path}", + ) return payload def logs( diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index f45349a5e..853f495bb 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -354,12 +354,14 @@ async def exec( check: bool = False, startup_timeout: Optional[float] = None, ) -> ExecResult: - """Execute argv, retrying only explicit 409 startup rejections. + """execute argv, retrying only explicit 409 startup rejections. - The startup deadline limits retries, not execution of an accepted - command. Each request retains its request timeout. Transport errors, + the startup deadline limits retries, not execution of an accepted + command. each request retains its request timeout. transport errors, timeouts, and other HTTP errors are never retried because the command - might already have executed. ``check=True`` preserves output on failure. + might already have executed. ``check=False`` returns command failures + in ``ExecResult.error``, matching ``subprocess.run``. use ``check=True`` + to raise ``SandboxExecutionError`` with partial output on failure. """ if isinstance(command, (str, bytes)) or not isinstance(command, Sequence): raise TypeError( diff --git a/runpod/sandbox/models.py b/runpod/sandbox/models.py index 712989e13..db25f7bb2 100644 --- a/runpod/sandbox/models.py +++ b/runpod/sandbox/models.py @@ -16,7 +16,7 @@ def _timestamp(value: str) -> datetime: @dataclass(frozen=True) class SandboxCompute: - vcpu_count: float + vcpu_count: int memory_in_gb: float container_disk_in_gb: int cost_per_hr: float diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 8b347512f..589d68965 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -27,6 +27,7 @@ def __init__(self): self.requests = [] self.executed = [] self.exec_statuses = deque() + self.exec_responses = deque() self.log_statuses = deque() self.delete_status = 204 self.create_started = threading.Event() @@ -130,10 +131,12 @@ async def handle(request): service.executed.append(body["command"]) if status >= 400: return failure(status, "response lost after command execution") + if service.exec_responses: + return service.exec_responses.popleft() result = ( {"output": "partial output", "error": "command failed"} if body["command"] == ["fail"] - else {"output": "completed", "error": None} + else {"output": "completed"} ) return web.json_response(result) if operation == "logs": @@ -222,7 +225,7 @@ async def test_async_startup_conflict_and_borrowed_context_ownership(peer): peer.exec_statuses.extend([409, 200]) async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as owner: async with await AsyncioSandbox.get(owner.id, **peer.options) as borrowed: - assert (await borrowed.exec(["work"])).output == "completed" + assert (await borrowed.exec(["work"], check=True)).output == "completed" assert peer.records[owner.id]["state"] == "RUNNING" with pytest.raises(RuntimeError): async with owner: @@ -337,6 +340,27 @@ async def test_exec_never_replays_ambiguous_failure_and_preserves_partial_output assert failure.value.sandbox_id == sandbox.id +@pytest.mark.asyncio +@pytest.mark.parametrize( + "body, content_type", + [ + ("{}", "application/json"), + ('{"output": null}', "application/json"), + ("proxy error", "text/html"), + ("[]", "application/json"), + ], +) +async def test_malformed_exec_response_is_a_query_error_without_replay( + peer, body, content_type +): + async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as sandbox: + peer.exec_responses.append(web.Response(text=body, content_type=content_type)) + with pytest.raises(QueryError) as failure: + await sandbox.exec(["side-effect"]) + assert failure.value.query == f"POST /v2/sandboxes/{sandbox.id}/exec" + assert peer.executed == [["side-effect"]] + + @pytest.mark.asyncio async def test_startup_deadline_and_terminal_state_do_not_execute_commands(peer): async with AsyncioSandbox(image_name="python:3.12-slim", **peer.options) as sandbox: From a05e1945a6e68d8658e12eb4d0a44d123cfc6678 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:21:11 -0700 Subject: [PATCH 112/120] feat!: add explicit volume resources and execution mount bindings --- CHANGELOG.md | 6 + README.md | 26 +- examples/apps/train_and_eval.py | 18 +- runpod/__init__.py | 2 + runpod/apps/__init__.py | 4 +- runpod/apps/app.py | 15 +- runpod/apps/dev.py | 2 - runpod/apps/spec.py | 51 +--- runpod/apps/tasks.py | 38 +-- runpod/apps/volume.py | 399 ++++++++++++++++++---------- tests/e2e/examples/08_volumes.py | 6 +- tests/e2e/examples/11_train_eval.py | 6 +- tests/test_apps/test_app.py | 59 ++-- tests/test_apps/test_tasks.py | 19 +- tests/test_apps/test_volume.py | 263 ++++++++++++++---- 15 files changed, 588 insertions(+), 326 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a4fdcd1a..5a5493b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Features + +* Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts and execution-bound filesystem access. + ## [1.12.0](https://github.com/runpod/runpod-python/compare/v1.11.0...v1.12.0) (2026-08-10) diff --git a/README.md b/README.md index 82a5f46b4..3a150a9c4 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,11 @@ Requires Python 3.10+. Installing the package also installs the `rp` CLI. ```python import runpod -from runpod import App, Model, Secret, Volume +from runpod import App, Model, NetworkVolume, Secret app = App("inference") -models = Volume("models", size=100) +models = NetworkVolume("models", size=100) llama = Model("meta-llama/Llama-3.1-8B-Instruct") @@ -40,7 +40,7 @@ llama = Model("meta-llama/Llama-3.1-8B-Instruct") gpu="H100", workers=(0, 3), dependencies=["vllm"], - volume=models, + mounts={"/runpod-volume": models}, model=llama, env={"HF_TOKEN": Secret("hf-token")}, ) @@ -52,7 +52,7 @@ def chat(prompt: str): # one ephemeral pod per call: provisions, runs to completion, terminates -@app.task(gpu="H100", gpu_count=2, volume=models) +@app.task(gpu="H100", gpu_count=2, mounts={"/models": models}) def finetune(steps: int = 1000): ... return {"loss": final_loss} @@ -73,6 +73,24 @@ Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `awa Queue `.remote()` calls request a synchronous result and poll the same job if the server's wait window expires. Fast results need no client-side polling. Dev sessions use a short sync window to keep worker logs responsive. Use `.spawn()` for an asynchronous job handle. A transport failure before receiving a job ID raises an error without resubmitting the work. +## Storage + +`NetworkVolume(name_or_id, size=50, datacenter=None, create=True)` references +datacenter-local storage. Apps resolve names and choose a datacenter compatible +with every resource sharing the volume. `GlobalVolume(id)` references existing +global storage without imposing a network-volume datacenter constraint. Both +inherit from the abstract `Volume` base. + +Declare attachments with `mounts={"/path": volume}`. Tasks support one network +and one global volume at distinct, non-overlapping paths. Queue and API resources +support one volume at `/runpod-volume`; global storage requires a GPU endpoint. + +Inside worker code, `volume.path` returns the configured mount path. The runtime +binds declared references and resolved IDs before importing user code. Access +raises if the volume is unmounted or has multiple bindings. Evaluate `.path` +inside remote functions, not during local module discovery. Calling `.local()` +does not mount remote storage on the client machine. + ## Contributing Pull requests and issues are welcome — see the [contributing guide](CONTRIBUTING.md) to get started. diff --git a/examples/apps/train_and_eval.py b/examples/apps/train_and_eval.py index b06e75a1f..624be5865 100644 --- a/examples/apps/train_and_eval.py +++ b/examples/apps/train_and_eval.py @@ -1,11 +1,9 @@ -"""A training workflow: tasks, volumes, and functions working together. +"""a training workflow with shared network storage. -@app.task gives each call its own dedicated pod that terminates when -the function returns — right for work that runs minutes to hours. -A Volume persists files between calls and across resources: train() -writes a checkpoint, evaluate() reads it from a different pod. The -volume is created on first use, and both tasks are automatically -placed in its datacenter. +each task runs on a dedicated pod that terminates when the function returns. +train() writes a checkpoint and evaluate() reads it from another pod. +the network volume is created on first use, and both tasks are placed +in its datacenter. each worker resolves checkpoints.path from its mount. rp flash dev examples/apps/train_and_eval.py """ @@ -14,10 +12,10 @@ app = runpod.App("trainer") -checkpoints = runpod.Volume("checkpoints", size=10) +checkpoints = runpod.NetworkVolume("checkpoints", size=10) -@app.task(gpu="4090", volume=checkpoints) +@app.task(gpu="4090", mounts={"/checkpoints": checkpoints}) def train(steps: int = 200): import torch import torch.nn as nn @@ -47,7 +45,7 @@ def train(steps: int = 200): return {"checkpoint": "linear", "final_loss": round(loss.item(), 5)} -@app.task(gpu="4090", volume=checkpoints) +@app.task(gpu="4090", mounts={"/checkpoints": checkpoints}) def evaluate(checkpoint: str): import torch import torch.nn as nn diff --git a/runpod/__init__.py b/runpod/__init__.py index ddbfa21e3..3fc8b7571 100644 --- a/runpod/__init__.py +++ b/runpod/__init__.py @@ -41,8 +41,10 @@ "App": "runpod.apps", "DataCenter": "runpod.apps", "EndpointNotFound": "runpod.apps", + "GlobalVolume": "runpod.apps", "Job": "runpod.apps", "Model": "runpod.apps", + "NetworkVolume": "runpod.apps", "Queue": "runpod.apps", "Secret": "runpod.apps", "Volume": "runpod.apps", diff --git a/runpod/apps/__init__.py b/runpod/apps/__init__.py index 3651f574f..6a62fef93 100644 --- a/runpod/apps/__init__.py +++ b/runpod/apps/__init__.py @@ -18,7 +18,7 @@ from .stubs import Api, Queue from .model import Model from .secret import Secret -from .volume import Volume +from .volume import GlobalVolume, NetworkVolume, Volume __all__ = [ "Api", @@ -29,8 +29,10 @@ "DataCenter", "EndpointNotFound", "FunctionHandle", + "GlobalVolume", "Job", "Model", + "NetworkVolume", "Queue", "Secret", "Volume", diff --git a/runpod/apps/app.py b/runpod/apps/app.py index 855112325..c2a6a0e5d 100644 --- a/runpod/apps/app.py +++ b/runpod/apps/app.py @@ -15,7 +15,7 @@ async def transcribe(audio_url: str): ... """ import os -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union from .context import Context, current_context from .errors import EndpointNotFound, InvalidResourceError @@ -27,6 +27,7 @@ async def transcribe(audio_url: str): ... ResourceSpec, ) from .targets import InvocationTarget, PodTarget, SentinelTarget +from .volume import Volume DEFAULT_ENV = "default" @@ -83,7 +84,7 @@ def queue( idle_timeout: int = 60, dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, - volume: Optional[Any] = None, + mounts: Optional[Mapping[str, Volume]] = None, env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, @@ -111,7 +112,7 @@ def decorator(fn: Callable) -> FunctionHandle: idle_timeout=idle_timeout, dependencies=dependencies, system_dependencies=system_dependencies, - volume=volume, + mounts=mounts, env=env, datacenter=datacenter, image=image, @@ -141,7 +142,7 @@ def task( gpu_count: int = 1, dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, - volume: Optional[Any] = None, + mounts: Optional[Mapping[str, Volume]] = None, env: Optional[Dict[str, Any]] = None, image: Optional[str] = None, registry_auth: Optional[str] = None, @@ -161,7 +162,7 @@ def decorator(fn: Callable) -> FunctionHandle: gpu_count=gpu_count, dependencies=dependencies, system_dependencies=system_dependencies, - volume=volume, + mounts=mounts, env=env, image=image, registry_auth=registry_auth, @@ -187,7 +188,7 @@ def api( idle_timeout: int = 60, dependencies: Optional[List[str]] = None, system_dependencies: Optional[List[str]] = None, - volume: Optional[Any] = None, + mounts: Optional[Mapping[str, Volume]] = None, env: Optional[Dict[str, Any]] = None, datacenter: Optional[Union[str, List[str]]] = None, image: Optional[str] = None, @@ -234,7 +235,7 @@ def decorator(target: Any) -> ApiHandle: idle_timeout=idle_timeout, dependencies=dependencies, system_dependencies=system_dependencies, - volume=volume, + mounts=mounts, env=env, datacenter=datacenter, image=image, diff --git a/runpod/apps/dev.py b/runpod/apps/dev.py index 222b66a8e..c4e95ad1c 100644 --- a/runpod/apps/dev.py +++ b/runpod/apps/dev.py @@ -232,8 +232,6 @@ async def _attach_volumes(self, payload: Dict, spec, app) -> None: from .model import model_reference payload["modelReferences"] = [model_reference(spec.model)] - if not spec.volume: - return if self._volume_resolver is None: self._volume_resolver = VolumeResolver(self.api, events=self.events) await attach_endpoint_volumes(payload, spec, self._volume_resolver, app) diff --git a/runpod/apps/spec.py b/runpod/apps/spec.py index c7a186088..a6a41373d 100644 --- a/runpod/apps/spec.py +++ b/runpod/apps/spec.py @@ -7,11 +7,12 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Mapping, Optional, Tuple, Union from .errors import InvalidResourceError from .datacenter import DataCenter from .gpu import GpuGroup, GpuLike, GpuType +from .volume import Volume, normalize_mounts, validate_mounts DEFAULT_WORKERS: Tuple[int, int] = (0, 3) @@ -159,29 +160,6 @@ def normalize_datacenter( raise InvalidResourceError(f"invalid datacenter: {datacenter!r}") from exc -def normalize_volume(volume: Any) -> Any: - """preserve volume configuration while normalizing reference collections.""" - from .volume import Volume - - if volume is None or isinstance(volume, (str, Volume)): - return volume - if isinstance(volume, (list, tuple)): - refs = [] - for item in volume: - if item is None: - raise InvalidResourceError("volume collections must contain references") - normalized = normalize_volume(item) - if isinstance(normalized, list): - refs.extend(normalized) - else: - refs.append(normalized) - return refs - raise InvalidResourceError( - f"volume must be a runpod.Volume, a name/id string, or a list/tuple " - f"of references, got {type(volume).__name__}" - ) - - @dataclass class ResourceSpec: """declarative config for one app resource.""" @@ -195,7 +173,7 @@ class ResourceSpec: idle_timeout: int = 60 dependencies: Optional[List[str]] = None system_dependencies: Optional[List[str]] = None - volume: Optional[Any] = None + mounts: Optional[Mapping[str, Volume]] = None env: Optional[Dict[str, Any]] = None datacenter: Optional[Union[str, List[str]]] = None image: Optional[str] = None @@ -228,11 +206,12 @@ def validate(self) -> None: self.scaler_type = normalize_scaler_type(self.scaler_type) self.min_cuda_version = normalize_cuda_version(self.min_cuda_version) self.datacenter = normalize_datacenter(self.datacenter) - self.volume = normalize_volume(self.volume) + self.mounts = normalize_mounts(self.mounts) if self.gpu is not None and self.cpu is not None: raise InvalidResourceError( f"resource '{self.name}': gpu and cpu are mutually exclusive" ) + validate_mounts(self.mounts, self.kind.value, self.is_cpu) if not isinstance(self.name, str) or not self.name: raise InvalidResourceError("resource name must be a non-empty string") if self.gpu_count < 1: @@ -267,11 +246,6 @@ def validate(self) -> None: f"available on queue and api resources; tasks download " f"weights themselves" ) - if self.kind is ResourceKind.TASK and isinstance(self.volume, list): - if len(self.volume) > 1: - raise InvalidResourceError( - f"task '{self.name}': pods mount exactly one volume" - ) if self.schedule is not None and ( not isinstance(self.schedule, str) or not self.schedule ): @@ -323,16 +297,11 @@ def to_manifest(self) -> Dict[str, Any]: data["dependencies"] = self.dependencies if self.system_dependencies: data["systemDependencies"] = self.system_dependencies - if self.volume: - if isinstance(self.volume, list): - data["networkVolumes"] = [ - getattr(volume, "name", None) or str(volume) - for volume in self.volume - ] - else: - data["networkVolume"] = getattr(self.volume, "name", None) or str( - self.volume - ) + if self.mounts: + data["mounts"] = [ + {"kind": volume.kind, "reference": volume.reference, "path": path} + for path, volume in self.mounts.items() + ] if self.env: from .secret import render_env diff --git a/runpod/apps/tasks.py b/runpod/apps/tasks.py index 8d98164f9..d1707f300 100644 --- a/runpod/apps/tasks.py +++ b/runpod/apps/tasks.py @@ -173,8 +173,7 @@ async def start(self) -> None: pod["containerRegistryAuthId"] = await resolve_registry_auth( self.spec.registry_auth, api=self.api ) - if self.spec.volume: - pod = await self._attach_volume(pod) + pod = await self._attach_mounts(pod) result = await self._deploy_pod(pod) self.pod_id = result["id"] log.info("task pod %s deployed for %s", self.pod_id, self.spec.name) @@ -190,7 +189,7 @@ async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: capacity_error = exc datacenters = pod.get("dataCenterIds") or [] - if self.spec.volume or len(datacenters) < 2: + if len(datacenters) < 2: raise capacity_error for datacenter in datacenters: @@ -204,34 +203,13 @@ async def _deploy_pod(self, pod: Dict[str, Any]) -> Dict[str, Any]: raise raise capacity_error - async def _attach_volume(self, pod: Dict[str, Any]) -> Dict[str, Any]: - """resolve the task's volume and pin the pod to its datacenter.""" - from .volume import VolumeError, VolumeResolver, volume_list + async def _attach_mounts(self, pod: Dict[str, Any]) -> Dict[str, Any]: + """resolve storage mounts and apply their placement constraints.""" + from .volume import VolumeResolver, attach_pod_mounts - volumes = volume_list(self.spec.volume) - if len(volumes) > 1: - raise VolumeError( - f"task '{self.spec.name}': pods mount exactly one volume " - f"({len(volumes)} given)" - ) - resolver = VolumeResolver(self.api) - sharing = [ - spec - for spec in self.specs - if any(ref.name == volumes[0].name for ref in volume_list(spec.volume)) - ] - if not any(spec is self.spec for spec in sharing): - sharing.append(self.spec) - for spec in sharing: - spec.validate() - resolved = await resolver.resolve(volumes[0], sharing) - from .volume import POD_MOUNT_PATH - - pod["networkVolumeId"] = resolved["id"] - # the host bind-mounts to this target; empty means a broken - # mount config and the container never starts - pod["volumeMountPath"] = str(POD_MOUNT_PATH) - pod["dataCenterIds"] = [resolved["dataCenterId"]] + await attach_pod_mounts( + pod, self.spec, VolumeResolver(self.api), self.specs + ) return pod async def wait_ready(self, timeout: float = READY_TIMEOUT) -> None: diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index 90bb26d88..a6b859dc6 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -1,25 +1,12 @@ -"""network volumes: durable storage shared across resources. - -a Volume is a lazy reference by name, resolved (and created when -missing) at provision time. workers see the volume at /runpod-volume. - - models = runpod.Volume("models") # create if missing, 50GB - models = runpod.Volume("models", size=100) - - @app.task(gpu="4090", volume=models) - def train(): - torch.save(sd, models.path / "model.pt") - -placement: a volume lives in exactly one datacenter, so everything -attached to it must schedule there. resolution runs the placement -solve over every resource sharing the volume (see runpod.apps.placement). -tasks/pods take one volume; endpoints may take several (one per -datacenter, locations derived from the volumes). -""" +"""storage resources and their execution-specific mount bindings.""" import logging -from pathlib import Path -from typing import Any, Dict, List, Optional +import json +from abc import ABC, abstractmethod +from collections.abc import Mapping +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from typing import Any, Dict, List, Optional, Tuple from .errors import AppError from .utils.client import default_client @@ -29,9 +16,6 @@ def train(): log = logging.getLogger(__name__) DEFAULT_SIZE_GB = 50 -# platform mount conventions: pods bind network volumes at /workspace, -# serverless endpoint workers at /runpod-volume -POD_MOUNT_PATH = Path("/workspace") ENDPOINT_MOUNT_PATH = Path("/runpod-volume") @@ -39,8 +23,44 @@ class VolumeError(AppError): pass -class Volume: - """a named network volume reference, created on first use.""" +class Volume(ABC): + """a storage reference, independent of its attachment to a worker.""" + + def __init__(self, reference: str): + if not isinstance(reference, str) or not reference.strip(): + raise VolumeError("volume reference must be a non-empty string") + self._reference = reference + + @property + @abstractmethod + def kind(self) -> str: + """the storage backend for this reference.""" + + @property + def reference(self) -> str: + return self._reference + + @property + def path(self) -> Path: + """the unique mount path for this volume in the current worker.""" + paths = _mount_bindings.get((self.kind, self.reference), ()) + if not paths: + raise VolumeError( + f"{self.kind} volume {self.reference!r} is not mounted " + "in the current execution" + ) + if len(paths) != 1: + raise VolumeError( + f"{self.kind} volume {self.reference!r} has multiple mounts; " + "use an explicit mount path" + ) + return paths[0] + + +class NetworkVolume(Volume): + """a network volume resolved by name or id, with datacenter placement.""" + + kind = "network" def __init__( self, @@ -50,65 +70,101 @@ def __init__( datacenter: Optional[str] = None, create: bool = True, ): - if not name or not isinstance(name, str): - raise VolumeError("volume name must be a non-empty string") - self.name = name + super().__init__(name) self.size = size self.datacenter = datacenter self.create = create @property - def path(self) -> Path: - """where the volume appears in the current worker. + def name(self) -> str: + return self.reference - context-sensitive because the platform mounts differ: task - pods use /workspace, endpoint workers /runpod-volume. resolved - at access time inside the worker, so the same function body - works from either. - """ - import os + def __repr__(self) -> str: + return f"" - if os.environ.get("RUNPOD_ENDPOINT_ID"): - return ENDPOINT_MOUNT_PATH - return POD_MOUNT_PATH - def __repr__(self) -> str: - return f"" +class GlobalVolume(Volume): + """an existing global volume identified by its resource id.""" + kind = "global" -def _as_volume(ref: Any) -> Volume: - if isinstance(ref, Volume): - return ref - if isinstance(ref, str): - return Volume(ref) - raise VolumeError( - f"volume must be a runpod.Volume or name/id string, " - f"got {type(ref).__name__}" - ) + def __init__(self, id: str): + super().__init__(id) + @property + def id(self) -> str: + return self.reference -def volume_list(spec_volume: Any) -> List[Volume]: - """normalize a spec's volume field to a list of Volume refs.""" - if spec_volume is None: - return [] - if isinstance(spec_volume, (list, tuple)): - return [_as_volume(v) for v in spec_volume] - return [_as_volume(spec_volume)] + def __repr__(self) -> str: + return f"" -class VolumeResolver: - """resolves every volume in an app once per provision run. +def _mount_path(path: str) -> str: + if ( + not isinstance(path, str) + or not path.startswith("/") + or path.startswith("//") + or "\0" in path + or ".." in PurePosixPath(path).parts + ): + raise VolumeError("mount paths must be absolute paths without '..'") + return str(PurePosixPath(path)) + + +def normalize_mounts(mounts: Optional[Mapping[str, Volume]]) -> Dict[str, Volume]: + """copy and normalize an explicit path-to-volume mapping.""" + if mounts is None: + return {} + if not isinstance(mounts, Mapping): + raise VolumeError("mounts must map absolute paths to volume objects") + normalized = {} + for path, volume in mounts.items(): + path = _mount_path(path) + if not isinstance(volume, Volume): + raise VolumeError("mounts require NetworkVolume or GlobalVolume references") + if path in normalized: + raise VolumeError(f"duplicate mount path {path!r}") + normalized[path] = volume + return normalized + + +_mount_bindings: Mapping[Tuple[str, str], Tuple[Path, ...]] = MappingProxyType({}) + + +def _configure_mounts(bindings: List[Dict[str, str]]) -> None: + """install resolved mounts for this worker before importing user code.""" + if not isinstance(bindings, list): + raise VolumeError("worker mount bindings must be a list") + resolved = {} + for binding in bindings: + if not isinstance(binding, Mapping): + raise VolumeError("worker mount bindings must be objects") + kind = binding.get("kind") + reference = binding.get("reference") + volume_id = binding.get("id") + if kind not in ("network", "global") or any( + not isinstance(value, str) or not value.strip() + for value in (reference, volume_id) + ): + raise VolumeError("worker mount bindings require kind, reference, and id") + path = Path(_mount_path(binding.get("path"))) + for key in ((kind, reference), (kind, volume_id)): + paths = resolved.setdefault(key, []) + if path not in paths: + paths.append(path) + global _mount_bindings + _mount_bindings = MappingProxyType( + {key: tuple(paths) for key, paths in resolved.items()} + ) - resolution: find by id or name; when missing, run the placement - solve over all resources sharing the volume and create it in the - chosen datacenter. results cache by name so each volume resolves - exactly once regardless of how many resources reference it. - """ + +class VolumeResolver: + """resolve storage references once per provisioning run.""" def __init__(self, api=None, events: Optional[object] = None): self._api = api self.events = events - self._resolved: Dict[str, Dict[str, Any]] = {} + self._resolved: Dict[Tuple[str, str], Dict[str, Any]] = {} self._stock = None async def _client(self): @@ -116,111 +172,168 @@ async def _client(self): return self._api async def resolve( - self, volume: Volume, specs: List + self, volume: Volume, specs: Optional[List] = None ) -> Dict[str, Any]: - """resolve one volume to {'id', 'dataCenterId'}. - - specs are all resource specs that attach this volume; they - drive placement for creation and validate an existing DC. - """ + """resolve a reference, applying app placement to network storage.""" + specs = specs or [] for spec in specs: spec.validate() - cached = self._resolved.get(volume.name) + key = (volume.kind, volume.reference) + cached = self._resolved.get(key) if cached is not None: return cached - - from .placement import StockMap, solve_placement + if isinstance(volume, GlobalVolume): + resolved = {"id": volume.id} + self._resolved[key] = resolved + return resolved + if not isinstance(volume, NetworkVolume): + raise VolumeError(f"unsupported volume type {type(volume).__name__}") client = await self._client() existing = await client.list_network_volumes() - record = find_by_id_or_name( existing, volume.name, noun="volumes", error=VolumeError ) + if record is None and not volume.create: + raise VolumeError(f"volume '{volume.name}' not found and create=False") - if self._stock is None: - self._stock = StockMap(client) - from .placement import _hardware_keys - - keys = [k for spec in specs for k in _hardware_keys(spec)] - await self._stock.fetch(keys) + dc = record["dataCenter"] if record is not None else volume.datacenter + if specs: + from .placement import StockMap, _hardware_keys, solve_placement - if record is not None: - # existing volume: its DC is a hard constraint + if self._stock is None: + self._stock = StockMap(client) + await self._stock.fetch([k for spec in specs for k in _hardware_keys(spec)]) dc = solve_placement( - specs, - self._stock, - volume_name=volume.name, - existing_dc=record["dataCenter"], + specs, self._stock, volume_name=volume.name, existing_dc=dc ) - resolved = {"id": record["id"], "dataCenterId": dc} - self._resolved[volume.name] = resolved - return resolved - - if not volume.create: + elif not dc: raise VolumeError( - f"volume '{volume.name}' not found and create=False" + f"creating network volume {volume.name!r} requires a datacenter " + "when no app placement constraints are available" ) - if volume.datacenter: - dc = solve_placement( - specs, - self._stock, - volume_name=volume.name, - existing_dc=volume.datacenter, + if record is None: + record = await client.create_network_volume( + name=volume.name, size=volume.size, data_center_id=dc ) - else: - dc = solve_placement( - specs, self._stock, volume_name=volume.name + emit(self.events, "volume_created", volume.name, volume.size, dc) + log.info( + "created volume %s (%s, %dGB, %s)", + volume.name, + record["id"], + volume.size, + dc, ) - - created = await client.create_network_volume( - name=volume.name, size=volume.size, data_center_id=dc - ) - emit( - self.events, - "volume_created", - volume.name, - volume.size, - dc, - ) - log.info( - "created volume %s (%s, %dGB, %s)", - volume.name, - created["id"], - volume.size, - dc, - ) - resolved = {"id": created["id"], "dataCenterId": dc} - self._resolved[volume.name] = resolved + resolved = {"id": record["id"], "dataCenterId": dc} + self._resolved[key] = resolved return resolved + async def resolve_mounts( + self, mounts: Optional[Mapping[str, Volume]], specs: Optional[List] = None + ) -> List[Dict[str, str]]: + """resolve explicit attachments without changing their volume objects.""" + bindings = [] + for path, volume in normalize_mounts(mounts).items(): + sharing = [ + spec + for spec in specs or [] + if any( + (ref.kind, ref.reference) == (volume.kind, volume.reference) + for ref in spec.mounts.values() + ) + ] + resolved = await self.resolve(volume, sharing) + bindings.append( + { + "kind": volume.kind, + "reference": volume.reference, + "path": path, + **resolved, + } + ) + return bindings + + +def validate_mounts(mounts: Mapping[str, Volume], kind: str, is_cpu: bool) -> None: + """validate the mount capabilities of the target execution environment.""" + paths = [PurePosixPath(path) for path in mounts] + for index, path in enumerate(paths): + if any( + path.is_relative_to(other) or other.is_relative_to(path) + for other in paths[:index] + ): + raise VolumeError("mount paths must not overlap") + for volume_kind in ("network", "global"): + if sum(volume.kind == volume_kind for volume in mounts.values()) > 1: + raise VolumeError(f"resources support at most one {volume_kind} volume") + if kind == "sandbox": + reserved = PurePosixPath("/etc/resolv.conf") + if any( + path.is_relative_to(reserved) or reserved.is_relative_to(path) + for path in paths + ): + raise VolumeError("sandbox mounts must not overlap /etc/resolv.conf") + if kind in ("queue", "api") and mounts: + if len(mounts) != 1 or next(iter(mounts)) != str(ENDPOINT_MOUNT_PATH): + raise VolumeError("endpoints support one volume mounted at /runpod-volume") + if is_cpu and any( + isinstance(volume, GlobalVolume) for volume in mounts.values() + ): + raise VolumeError("global volumes require a gpu endpoint") + + +def _bind_worker_mounts( + payload: Dict[str, Any], bindings: List[Dict[str, str]] +) -> None: + env = payload.setdefault("env", []) + env[:] = [entry for entry in env if entry["key"] != "RUNPOD_MOUNTS"] + env.append({"key": "RUNPOD_MOUNTS", "value": json.dumps(bindings)}) + -def specs_sharing_volume(apps: List, name: str) -> List: - """every resource spec (across apps) attaching the named volume.""" - out = [] - for app in apps: - for handle in app.resources.values(): - for ref in volume_list(handle.spec.volume): - if ref.name == name: - out.append(handle.spec) - return out +async def attach_pod_mounts( + payload: Dict[str, Any], spec, resolver: VolumeResolver, specs: List +) -> None: + """attach task storage and supply the worker's resolved filesystem bindings.""" + validate_mounts(spec.mounts, spec.kind.value, spec.is_cpu) + bindings = await resolver.resolve_mounts(spec.mounts, specs) + payload["volumeMounts"] = [ + { + "volumeId": binding["id"], + "volumeType": ( + "NETWORK_VOLUME" + if binding["kind"] == "network" + else "OBJECT_STORE_VOLUME" + ), + "mountPath": binding["path"], + } + for binding in bindings + ] + for binding in bindings: + if binding["kind"] == "network": + payload["dataCenterIds"] = [binding["dataCenterId"]] + _bind_worker_mounts(payload, bindings) async def attach_endpoint_volumes( payload: Dict[str, Any], spec, resolver: VolumeResolver, app ) -> None: - """resolve a resource's volumes onto an endpoint payload.""" - volumes = volume_list(spec.volume) - if not volumes: - return - resolved = [] - for volume in volumes: - sharing = specs_sharing_volume([app], volume.name) or [spec] - resolved.append(await resolver.resolve(volume, sharing)) + """attach endpoint storage at the platform mount path.""" + validate_mounts(spec.mounts, spec.kind.value, spec.is_cpu) + bindings = await resolver.resolve_mounts( + spec.mounts, [handle.spec for handle in app.resources.values()] + ) payload["networkVolumeIds"] = [ - {"networkVolumeId": result["id"]} for result in resolved + {"networkVolumeId": binding["id"]} + for binding in bindings + if binding["kind"] == "network" ] - payload["locations"] = ",".join( - dict.fromkeys(result["dataCenterId"] for result in resolved) - ) + payload["volumes"] = [ + {"volumeId": binding["id"], "volumeType": "OBJECT_STORE_VOLUME"} + for binding in bindings + if binding["kind"] == "global" + ] + for binding in bindings: + if binding["kind"] == "network": + payload["locations"] = binding["dataCenterId"] + _bind_worker_mounts(payload.setdefault("template", {}), bindings) diff --git a/tests/e2e/examples/08_volumes.py b/tests/e2e/examples/08_volumes.py index cdbe2ab86..69e334085 100644 --- a/tests/e2e/examples/08_volumes.py +++ b/tests/e2e/examples/08_volumes.py @@ -10,10 +10,10 @@ app = runpod.App("ex-volumes") -scratch = runpod.Volume("ex-scratch", size=10) +scratch = runpod.NetworkVolume("ex-scratch", size=10) -@app.task(cpu="cpu3c-1-2", volume=scratch) +@app.task(cpu="cpu3c-1-2", mounts={"/data": scratch}) def write(content: str): target = scratch.path / "message.txt" target.write_text(content) @@ -21,7 +21,7 @@ def write(content: str): return str(target) -@app.task(cpu="cpu3c-1-2", volume=scratch) +@app.task(cpu="cpu3c-1-2", mounts={"/data": scratch}) def read(): target = scratch.path / "message.txt" content = target.read_text() diff --git a/tests/e2e/examples/11_train_eval.py b/tests/e2e/examples/11_train_eval.py index 16759c70d..ab01d1f99 100644 --- a/tests/e2e/examples/11_train_eval.py +++ b/tests/e2e/examples/11_train_eval.py @@ -7,10 +7,10 @@ app = runpod.App("ex-train-eval") -models = runpod.Volume("ex-models", size=10) +models = runpod.NetworkVolume("ex-models", size=10) -@app.task(gpu="4090", volume=models) +@app.task(gpu="4090", mounts={"/models": models}) def train(steps: int = 200): import json @@ -42,7 +42,7 @@ def train(steps: int = 200): return {"checkpoint": "linear-run", "loss": round(loss.item(), 5)} -@app.task(gpu="4090", volume=models) +@app.task(gpu="4090", mounts={"/models": models}) def evaluate(checkpoint: str): import torch import torch.nn as nn diff --git a/tests/test_apps/test_app.py b/tests/test_apps/test_app.py index 26d8d25e4..e23614be0 100644 --- a/tests/test_apps/test_app.py +++ b/tests/test_apps/test_app.py @@ -272,44 +272,29 @@ def q(): await q.remote.aio() -@pytest.mark.parametrize("collection", [list, tuple]) -async def test_endpoint_volume_collections_reach_attachment(collection): - from unittest.mock import AsyncMock - - from runpod.apps.volume import Volume, VolumeResolver, attach_endpoint_volumes +@pytest.mark.parametrize("decorator", ["queue", "task", "api"]) +@pytest.mark.parametrize( + "mounts", + ["models", ["models"], {"/runpod-volume": "models"}], +) +def test_decorators_reject_implicit_storage_references(decorator, mounts): + from runpod.apps.volume import VolumeError app = App("a") - - @app.api(name="web", cpu="cpu5c-2-4", volume=collection([Volume("models"), "data"])) - def web(): - return None - - api = AsyncMock() - api.list_network_volumes.return_value = [ - {"id": "nv-models", "name": "models", "dataCenter": "EU-RO-1"}, - {"id": "nv-data", "name": "data", "dataCenter": "EU-RO-1"}, - ] - api.cpu_stock_status.return_value = "HIGH" - payload = {} - await attach_endpoint_volumes(payload, web.spec, VolumeResolver(api), app) - assert payload["networkVolumeIds"] == [ - {"networkVolumeId": "nv-models"}, - {"networkVolumeId": "nv-data"}, - ] - assert payload["locations"] == "EU-RO-1" - assert web.spec.to_manifest()["networkVolumes"] == ["models", "data"] + with pytest.raises(VolumeError): + getattr(app, decorator)(mounts=mounts)(lambda: None) def test_manifest_serialization(): app = App("a") - volume = "my-volume" + volume = runpod.NetworkVolume("my-volume") @app.queue( name="q", gpu=[runpod.GpuGroup.ADA_24], workers=(1, 3), dependencies=["torch"], - volume=volume, + mounts={"/runpod-volume": volume}, env={"KEY": "val"}, ) def q(): @@ -325,11 +310,31 @@ def q(): "idleTimeout": 60, "gpus": ["ADA_24"], "dependencies": ["torch"], - "networkVolume": "my-volume", + "mounts": [ + {"kind": "network", "reference": "my-volume", "path": "/runpod-volume"} + ], "env": {"KEY": "val"}, } +def test_task_manifest_preserves_mount_backend_identity_and_path(): + app = App("a") + + @app.task( + mounts={ + "/network": runpod.NetworkVolume("shared"), + "/global": runpod.GlobalVolume("shared"), + } + ) + def work(): + pass + + assert work.spec.to_manifest()["mounts"] == [ + {"kind": "network", "reference": "shared", "path": "/network"}, + {"kind": "global", "reference": "shared", "path": "/global"}, + ] + + class TestGpuStringResolution: def test_pool_id_passthrough(self): from runpod.apps.spec import normalize_gpu diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index d7304467d..12ec125d4 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -14,6 +14,7 @@ from runpod.apps.spec import ResourceKind, ResourceSpec from runpod.apps.targets import PodTarget from runpod.apps.tasks import _pod_input, unwrap_task_response +from runpod.apps.volume import GlobalVolume, NetworkVolume # the sync bridge finalizes coroutines on a background loop thread; # asyncmock coroutines observed there trip the unraisable checker @@ -89,12 +90,11 @@ def test_custom_image_starts_runtime_package(self, monkeypatch): monkeypatch.setenv("RUNPOD_RUNTIME_PACKAGE_SPEC", "runpod-sdk-runtime==1.2.3") monkeypatch.setenv("RUNPOD_PACKAGE_SPEC", "runpod==2.0.0") pod = _pod_input( - self._spec(cpu=["cpu3c-1-2"], image="my/image:1", volume="vol-1"), + self._spec(cpu=["cpu3c-1-2"], image="my/image:1"), "tok", "t", ) assert pod["imageName"] == "my/image:1" - assert "networkVolumeId" not in pod env = {e["key"]: e["value"] for e in pod["env"]} assert env["RUNPOD_RUNTIME_PACKAGE_SPEC"] == "runpod-sdk-runtime==1.2.3" assert env["RUNPOD_PACKAGE_SPEC"] == "runpod==2.0.0" @@ -333,7 +333,8 @@ async def test_start_deploys_pod(self): assert execution.pod_id == "pod-9" assert api.deploy_task_pod.call_args[1]["is_cpu"] is True - async def test_start_retries_datacenters_after_capacity_error(self): + @pytest.mark.parametrize("mounts", [{}, {"/data": GlobalVolume("global-1")}]) + async def test_start_retries_datacenters_after_capacity_error(self, mounts): from runpod.apps.tasks import TaskExecution from runpod.error import QueryError @@ -351,6 +352,7 @@ async def test_start_retries_datacenters_after_capacity_error(self): gpu=["NVIDIA GeForce RTX 4090"], cpu=None, datacenter=["US-IL-1", "EU-RO-1"], + mounts=mounts, ), api=api, ) @@ -376,22 +378,23 @@ async def test_start_does_not_retry_non_capacity_error(self): async def test_shared_volume_placement_accounts_for_siblings(self): from runpod.apps.tasks import TaskExecution - from runpod.apps.volume import Volume - volume = Volume("shared") - spec = self._spec(volume=volume, datacenter=["US-IL-1", "EU-RO-1"]) + volume = NetworkVolume("shared") + spec = self._spec( + mounts={"/data": volume}, datacenter=["US-IL-1", "EU-RO-1"] + ) sibling = ResourceSpec( kind=ResourceKind.QUEUE, name="sibling", cpu=["cpu3c-1-2"], - volume=volume, + mounts={"/runpod-volume": volume}, datacenter=["EU-RO-1"], ) unrelated = ResourceSpec( kind=ResourceKind.QUEUE, name="unrelated", cpu=["cpu3c-1-2"], - volume=Volume("other"), + mounts={"/runpod-volume": NetworkVolume("other")}, datacenter=["US-IL-1"], ) api = AsyncMock() diff --git a/tests/test_apps/test_volume.py b/tests/test_apps/test_volume.py index ed346a175..f55b52636 100644 --- a/tests/test_apps/test_volume.py +++ b/tests/test_apps/test_volume.py @@ -1,24 +1,26 @@ """volume references and provision-time resolution.""" import asyncio +import json from unittest.mock import AsyncMock import pytest -from runpod.apps.errors import InvalidResourceError from runpod.apps.spec import ResourceKind, ResourceSpec from runpod.apps.volume import ( + GlobalVolume, + NetworkVolume, Volume, VolumeError, VolumeResolver, - volume_list, + _configure_mounts, + attach_endpoint_volumes, + normalize_mounts, ) def _spec(name="r", gpu=None, cpu=None): - return ResourceSpec( - kind=ResourceKind.TASK, name=name, gpu=gpu, cpu=cpu - ) + return ResourceSpec(kind=ResourceKind.TASK, name=name, gpu=gpu, cpu=cpu) def _api(volumes=None, created=None): @@ -36,26 +38,105 @@ def _api(volumes=None, created=None): return api +@pytest.fixture(autouse=True) +def clean_mounts(): + _configure_mounts([]) + yield + _configure_mounts([]) + + class TestVolumeRef: - def test_path_follows_context(self, monkeypatch): - from pathlib import Path + def test_base_is_abstract(self): + with pytest.raises(TypeError): + Volume("models") - # task pods mount at /workspace - monkeypatch.delenv("RUNPOD_ENDPOINT_ID", raising=False) - assert Volume("models").path == Path("/workspace") - # endpoint workers mount at /runpod-volume + def test_unmounted_volume_has_no_conventional_path(self, monkeypatch): monkeypatch.setenv("RUNPOD_ENDPOINT_ID", "ep-1") - assert Volume("models").path == Path("/runpod-volume") + with pytest.raises(VolumeError): + NetworkVolume("models").path - def test_empty_name_raises(self): + def test_kind_and_resolved_id_bind_to_actual_files(self, tmp_path): + network_path = tmp_path / "network" + global_path = tmp_path / "global" + network_path.mkdir() + global_path.mkdir() + (network_path / "model").write_text("network weights") + (global_path / "model").write_text("global weights") + _configure_mounts( + [ + { + "kind": "network", + "reference": "models", + "id": "nv-1", + "path": str(network_path), + }, + { + "kind": "global", + "reference": "models", + "id": "models", + "path": str(global_path), + }, + ] + ) + assert (NetworkVolume("models").path / "model").read_text() == "network weights" + assert (NetworkVolume("nv-1").path / "model").read_text() == "network weights" + assert (GlobalVolume("models").path / "model").read_text() == "global weights" + + def test_multiple_bindings_require_explicit_path(self): + _configure_mounts( + [ + { + "kind": "network", + "reference": "models", + "id": "nv-1", + "path": "/one", + }, + { + "kind": "network", + "reference": "models", + "id": "nv-1", + "path": "/two", + }, + ] + ) with pytest.raises(VolumeError): - Volume("") + NetworkVolume("models").path + with pytest.raises(VolumeError): + NetworkVolume("nv-1").path - def test_volume_list_normalizes(self): - vols = volume_list([Volume("a"), "b"]) - assert [v.name for v in vols] == ["a", "b"] - assert volume_list(None) == [] - assert [v.name for v in volume_list("single")] == ["single"] + def test_invalid_configuration_does_not_partially_publish(self, tmp_path): + (tmp_path / "model").write_text("retained") + binding = { + "kind": "network", + "reference": "models", + "id": "nv-1", + "path": str(tmp_path), + } + _configure_mounts([binding]) + with pytest.raises(VolumeError): + _configure_mounts([dict(binding, path="/other"), {"kind": "global"}]) + assert (NetworkVolume("models").path / "model").read_text() == "retained" + _configure_mounts([]) + with pytest.raises(VolumeError): + NetworkVolume("models").path + + @pytest.mark.parametrize("volume_type", [NetworkVolume, GlobalVolume]) + def test_empty_reference_raises(self, volume_type): + with pytest.raises(VolumeError): + volume_type("") + + @pytest.mark.parametrize( + "mounts", + [ + {"/models": "models"}, + {"models": NetworkVolume("models")}, + {"/models/../data": NetworkVolume("models")}, + {"/models": NetworkVolume("models"), "/models/": GlobalVolume("global")}, + ], + ) + def test_invalid_mount_mapping_raises(self, mounts): + with pytest.raises(VolumeError): + normalize_mounts(mounts) class TestVolumeResolver: @@ -67,7 +148,7 @@ def test_existing_by_name(self): ) resolver = VolumeResolver(api) resolved = asyncio.run( - resolver.resolve(Volume("models"), [_spec(gpu=None)]) + resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)]) ) assert resolved == {"id": "nv-1", "dataCenterId": "EU-RO-1"} api.create_network_volume.assert_not_awaited() @@ -80,7 +161,7 @@ def test_existing_by_id(self): ) resolver = VolumeResolver(api) resolved = asyncio.run( - resolver.resolve(Volume("nv-1"), [_spec(gpu=None)]) + resolver.resolve(NetworkVolume("nv-1"), [_spec(gpu=None)]) ) assert resolved["id"] == "nv-1" @@ -88,7 +169,7 @@ def test_missing_creates_with_placement(self): api = _api() resolver = VolumeResolver(api) resolved = asyncio.run( - resolver.resolve(Volume("models"), [_spec(gpu=None)]) + resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)]) ) assert resolved["id"] == "nv-new" api.create_network_volume.assert_awaited_once() @@ -99,7 +180,7 @@ def test_missing_no_create_raises(self): with pytest.raises(VolumeError, match="create=False"): asyncio.run( resolver.resolve( - Volume("models", create=False), [_spec(gpu=None)] + NetworkVolume("models", create=False), [_spec(gpu=None)] ) ) @@ -112,9 +193,7 @@ def test_duplicate_names_raise(self): ) resolver = VolumeResolver(api) with pytest.raises(VolumeError, match="reference by id"): - asyncio.run( - resolver.resolve(Volume("models"), [_spec(gpu=None)]) - ) + asyncio.run(resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)])) def test_resolution_cached_per_name(self): api = _api( @@ -125,36 +204,40 @@ def test_resolution_cached_per_name(self): resolver = VolumeResolver(api) async def run(): - await resolver.resolve(Volume("models"), [_spec(gpu=None)]) - await resolver.resolve(Volume("models"), [_spec(gpu=None)]) + await resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)]) + await resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)]) asyncio.run(run()) assert api.list_network_volumes.await_count == 1 - def test_created_event_emitted(self): - events = [] - - class Sink: - def volume_created(self, name, size, dc): - events.append((name, size, dc)) + def test_same_reference_in_different_backends_does_not_share_cache(self): + resolver = VolumeResolver( + _api(volumes=[{"id": "nv-1", "name": "models", "dataCenter": "EU-RO-1"}]) + ) + assert asyncio.run(resolver.resolve(GlobalVolume("models"))) == {"id": "models"} + assert asyncio.run(resolver.resolve(NetworkVolume("models"))) == { + "id": "nv-1", + "dataCenterId": "EU-RO-1", + } - api = _api() - resolver = VolumeResolver(api, events=Sink()) - asyncio.run(resolver.resolve(Volume("models"), [_spec(gpu=None)])) - assert len(events) == 1 - name, size, dc = events[0] - assert (name, size) == ("models", 50) - assert dc # placement picks a concrete datacenter + def test_creation_without_app_requires_explicit_placement(self): + resolver = VolumeResolver(_api()) + with pytest.raises(VolumeError): + asyncio.run(resolver.resolve(NetworkVolume("models"))) + resolved = asyncio.run( + resolver.resolve(NetworkVolume("models", datacenter="EU-RO-1")) + ) + assert resolved == {"id": "nv-new", "dataCenterId": "EU-RO-1"} class TestTaskVolume: - def test_single_volume_only(self): - with pytest.raises(InvalidResourceError): + def test_one_volume_per_backend(self): + with pytest.raises(VolumeError): ResourceSpec( kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"], - volume=[Volume("a"), Volume("b")], + mounts={"/a": NetworkVolume("a"), "/b": NetworkVolume("b")}, ) def test_pod_pins_to_volume_dc(self): @@ -169,9 +252,95 @@ def test_pod_pins_to_volume_dc(self): kind=ResourceKind.TASK, name="t", cpu=["cpu3c-1-2"], - volume=Volume("models"), + mounts={"/models": NetworkVolume("models"), "/data": GlobalVolume("gv-1")}, ) execution = TaskExecution(spec, api=api) - pod = asyncio.run(execution._attach_volume({})) - assert pod["networkVolumeId"] == "nv-1" + pod = asyncio.run(execution._attach_mounts({})) + assert pod["volumeMounts"] == [ + { + "volumeId": "nv-1", + "volumeType": "NETWORK_VOLUME", + "mountPath": "/models", + }, + { + "volumeId": "gv-1", + "volumeType": "OBJECT_STORE_VOLUME", + "mountPath": "/data", + }, + ] assert pod["dataCenterIds"] == ["EU-RO-1"] + _configure_mounts(json.loads(pod["env"][0]["value"])) + assert str(NetworkVolume("models").path) == "/models" + assert str(GlobalVolume("gv-1").path) == "/data" + + +class TestEndpointMounts: + @pytest.mark.parametrize( + "mounts,cpu", + [ + ({"/models": NetworkVolume("models")}, None), + ({"/runpod-volume": GlobalVolume("global")}, "cpu3c-1-2"), + ( + { + "/runpod-volume": NetworkVolume("models"), + "/data": GlobalVolume("global"), + }, + None, + ), + ], + ) + def test_unsupported_attachment_rejected_before_provisioning(self, mounts, cpu): + with pytest.raises(VolumeError): + ResourceSpec(kind=ResourceKind.QUEUE, name="queue", cpu=cpu, mounts=mounts) + + def test_global_attachment_does_not_pin_datacenter(self): + from runpod import App + + app = App("global-model") + + @app.queue(gpu="4090", mounts={"/runpod-volume": GlobalVolume("gv-1")}) + def generate(): + return None + + payload = {"locations": "EU-RO-1,US-KS-2"} + asyncio.run( + attach_endpoint_volumes(payload, generate.spec, VolumeResolver(), app) + ) + assert payload["locations"] == "EU-RO-1,US-KS-2" + assert payload["networkVolumeIds"] == [] + assert payload["volumes"] == [ + {"volumeId": "gv-1", "volumeType": "OBJECT_STORE_VOLUME"} + ] + + def test_removing_storage_clears_backend_and_worker_bindings(self): + from runpod import App + + app = App("no-storage") + + @app.queue() + def generate(): + return None + + old_binding = { + "kind": "network", + "reference": "models", + "id": "nv-1", + "path": "/runpod-volume", + } + _configure_mounts([old_binding]) + payload = { + "networkVolumeIds": [{"networkVolumeId": "nv-1"}], + "volumes": [{"volumeId": "gv-1", "volumeType": "OBJECT_STORE_VOLUME"}], + "template": { + "env": [{"key": "RUNPOD_MOUNTS", "value": json.dumps([old_binding])}] + }, + } + asyncio.run( + attach_endpoint_volumes(payload, generate.spec, VolumeResolver(), app) + ) + assert payload["networkVolumeIds"] == [] + assert payload["volumes"] == [] + env = {entry["key"]: entry["value"] for entry in payload["template"]["env"]} + _configure_mounts(json.loads(env["RUNPOD_MOUNTS"])) + with pytest.raises(VolumeError): + NetworkVolume("models").path From 135d652f65ce28852135d33ed62ce8a42631e1b7 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:33:29 -0700 Subject: [PATCH 113/120] test: allow additive public sdk exports --- tests/test_init.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/tests/test_init.py b/tests/test_init.py index 52339fce2..694ec2894 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -86,34 +86,6 @@ def test_private_imports_not_exported(self): for private_symbol in private_symbols: assert private_symbol not in all_symbols, f"Private symbol '{private_symbol}' should not be in __all__" - def test_all_covers_expected_public_api(self): - """Test that __all__ contains the expected public API symbols.""" - expected_symbols = { - # API functions - 'create_container_registry_auth', 'create_endpoint', 'create_pod', 'create_template', - 'delete_container_registry_auth', 'get_endpoints', 'get_gpu', 'get_gpus', - 'get_pod', 'get_pods', 'get_user', 'resume_pod', 'stop_pod', 'terminate_pod', - 'update_container_registry_auth', 'update_endpoint_template', 'update_user_settings', - # Config functions - 'check_credentials', 'get_credentials', 'set_credentials', - # Endpoint classes - 'AsyncioEndpoint', 'AsyncioJob', 'Endpoint', - # Apps surface - 'Api', 'App', 'CpuInstanceType', 'DataCenter', 'EndpointNotFound', 'GpuGroup', - 'GpuType', 'Job', 'Model', 'Queue', 'Secret', 'Volume', 'delete', 'get', - 'init', 'is_local', 'local_entrypoint', 'patch', 'post', 'put', 'schedule', - # Serverless module - 'serverless', - # Logger class - 'RunPodLogger', - # Version - '__version__', - # Module variables - 'SSH_KEY_PATH', 'profile', 'api_key', 'endpoint_url_base' - } - - actual_symbols = set(runpod.__all__) - assert expected_symbols == actual_symbols, f"Expected {expected_symbols}, got {actual_symbols}" def test_no_duplicate_symbols_in_all(self): """Test that __all__ contains no duplicate symbols.""" From b2d79791ea09a082bd2af12e01e1205ce23c8605 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:07:45 -0700 Subject: [PATCH 114/120] fix: provision global volumes lazily through graphql --- CHANGELOG.md | 2 +- README.md | 8 +++-- runpod/api/mutations/apps.py | 6 ++++ runpod/api/queries/apps.py | 8 +++++ runpod/apps/api.py | 14 ++++++-- runpod/apps/volume.py | 27 ++++++++++---- tests/test_apps/test_api_client.py | 5 +-- tests/test_apps/test_tasks.py | 7 ++-- tests/test_apps/test_volume.py | 58 +++++++++++++++++++++++------- 9 files changed, 106 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a5493b04..b33348340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -* Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts and execution-bound filesystem access. +* Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts, lazy create-if-missing provisioning, and execution-bound filesystem access. Global volumes use GraphQL for lookup and creation. ## [1.12.0](https://github.com/runpod/runpod-python/compare/v1.11.0...v1.12.0) (2026-08-10) diff --git a/README.md b/README.md index 3a150a9c4..48aa0fa71 100644 --- a/README.md +++ b/README.md @@ -77,9 +77,11 @@ Queue `.remote()` calls request a synchronous result and poll the same job if th `NetworkVolume(name_or_id, size=50, datacenter=None, create=True)` references datacenter-local storage. Apps resolve names and choose a datacenter compatible -with every resource sharing the volume. `GlobalVolume(id)` references existing -global storage without imposing a network-volume datacenter constraint. Both -inherit from the abstract `Volume` base. +with every resource sharing the volume. `GlobalVolume(name_or_id, create=True)` +references global storage without a datacenter constraint. Both inherit from +the abstract `Volume` base, resolve by name or ID, and create missing storage +when provisioning remote compute. Set `create=False` to require existing storage. +Global-volume lookup and creation use GraphQL; network volumes use REST. Declare attachments with `mounts={"/path": volume}`. Tasks support one network and one global volume at distinct, non-overlapping paths. Queue and API resources diff --git a/runpod/api/mutations/apps.py b/runpod/api/mutations/apps.py index 2e3c73a44..eb8f4db80 100644 --- a/runpod/api/mutations/apps.py +++ b/runpod/api/mutations/apps.py @@ -1,5 +1,11 @@ """GraphQL mutations for capabilities absent from the resource REST API.""" +MUTATION_CREATE_GLOBAL_VOLUME = """ +mutation createGlobalVolume($input: GlobalStoreBucketCreateInput!) { + globalStoreBucketCreate(input: $input) { id name } +} +""" + # endpoint saves retain flash bindings, cached models, and schedules atomically. MUTATION_SAVE_ENDPOINT = """ mutation saveEndpoint($input: EndpointInput!) { diff --git a/runpod/api/queries/apps.py b/runpod/api/queries/apps.py index 9da1b2cde..9362bb2fd 100644 --- a/runpod/api/queries/apps.py +++ b/runpod/api/queries/apps.py @@ -1,5 +1,13 @@ """GraphQL queries for capabilities absent from the resource REST API.""" +QUERY_GLOBAL_VOLUMES = """ +query myGlobalVolumes { + myself { + globalStoreBuckets { id name } + } +} +""" + # rest stock filters require a power-of-two count of at least two. QUERY_CPU_STOCK = """ query CpuStock($cpuFlavorInput: CpuFlavorInput, $specificsInput: SpecificsInput) { diff --git a/runpod/apps/api.py b/runpod/apps/api.py index 1988b9d1f..64ba26b3a 100644 --- a/runpod/apps/api.py +++ b/runpod/apps/api.py @@ -1,8 +1,8 @@ """control-plane calls for app provisioning. rest handles equivalent resource operations; graphql handles flash lifecycle, -auth, secrets, task provisioning, and endpoint capabilities absent from rest. -management verbs for the wider sdk stay in runpod.api.ctl_commands. +auth, secrets, global storage, task provisioning, and endpoint capabilities +absent from rest. management verbs for the wider sdk stay in runpod.api.ctl_commands. """ from typing import Any, Dict, List, Optional @@ -387,6 +387,16 @@ async def cpu_stock_status( ) return _stock_in_datacenter(data, data_center_id) + async def list_global_volumes(self) -> List[Dict[str, Any]]: + data = await self._execute(app_queries.QUERY_GLOBAL_VOLUMES, retry=True) + return data["myself"]["globalStoreBuckets"] + + async def create_global_volume(self, name: str) -> Dict[str, Any]: + data = await self._execute( + app_mutations.MUTATION_CREATE_GLOBAL_VOLUME, {"input": {"name": name}} + ) + return data["globalStoreBucketCreate"] + async def list_network_volumes(self) -> List[Dict[str, Any]]: data = await run_rest_request_async( "GET", "/v2/network-volumes", api_key=self._api_key diff --git a/runpod/apps/volume.py b/runpod/apps/volume.py index a6b859dc6..6673e09b3 100644 --- a/runpod/apps/volume.py +++ b/runpod/apps/volume.py @@ -84,19 +84,20 @@ def __repr__(self) -> str: class GlobalVolume(Volume): - """an existing global volume identified by its resource id.""" + """a global volume resolved by name or id without datacenter placement.""" kind = "global" - def __init__(self, id: str): - super().__init__(id) + def __init__(self, name: str, *, create: bool = True): + super().__init__(name) + self.create = create @property - def id(self) -> str: + def name(self) -> str: return self.reference def __repr__(self) -> str: - return f"" + return f"" def _mount_path(path: str) -> str: @@ -183,7 +184,21 @@ async def resolve( if cached is not None: return cached if isinstance(volume, GlobalVolume): - resolved = {"id": volume.id} + client = await self._client() + record = find_by_id_or_name( + await client.list_global_volumes(), + volume.name, + noun="global volumes", + error=VolumeError, + ) + if record is None: + if not volume.create: + raise VolumeError( + f"global volume '{volume.name}' not found and create=False" + ) + record = await client.create_global_volume(name=volume.name) + log.info("created global volume %s (%s)", volume.name, record["id"]) + resolved = {"id": record["id"]} self._resolved[key] = resolved return resolved if not isinstance(volume, NetworkVolume): diff --git a/tests/test_apps/test_api_client.py b/tests/test_apps/test_api_client.py index f66beadfe..0a4d42771 100644 --- a/tests/test_apps/test_api_client.py +++ b/tests/test_apps/test_api_client.py @@ -59,16 +59,17 @@ async def test_graphql_errors_propagate_immediately(self): await client._execute("query {}") assert transport.await_count == 1 + @pytest.mark.parametrize("operation", ["create_app", "create_global_volume"]) @pytest.mark.parametrize( "failure", [aiohttp.ClientError("response lost"), OSError("reset")], ) - async def test_mutation_transport_failure_is_not_retried(self, failure): + async def test_mutation_transport_failure_is_not_retried(self, failure, operation): client = AppsApiClient(api_key="test-key") transport = AsyncMock(side_effect=failure) with patch("runpod.apps.api.run_graphql_query_async", transport): with pytest.raises(type(failure)): - await client.create_app("demo") + await getattr(client, operation)("demo") assert transport.await_count == 1 diff --git a/tests/test_apps/test_tasks.py b/tests/test_apps/test_tasks.py index 12ec125d4..e04b1669f 100644 --- a/tests/test_apps/test_tasks.py +++ b/tests/test_apps/test_tasks.py @@ -339,6 +339,9 @@ async def test_start_retries_datacenters_after_capacity_error(self, mounts): from runpod.error import QueryError api = MagicMock() + api.list_global_volumes = AsyncMock( + return_value=[{"id": "global-1", "name": "models"}] + ) capacity = "This machine does not have the resources to deploy your pod" api.deploy_task_pod = AsyncMock( side_effect=[ @@ -380,9 +383,7 @@ async def test_shared_volume_placement_accounts_for_siblings(self): from runpod.apps.tasks import TaskExecution volume = NetworkVolume("shared") - spec = self._spec( - mounts={"/data": volume}, datacenter=["US-IL-1", "EU-RO-1"] - ) + spec = self._spec(mounts={"/data": volume}, datacenter=["US-IL-1", "EU-RO-1"]) sibling = ResourceSpec( kind=ResourceKind.QUEUE, name="sibling", diff --git a/tests/test_apps/test_volume.py b/tests/test_apps/test_volume.py index f55b52636..6c3cae5b7 100644 --- a/tests/test_apps/test_volume.py +++ b/tests/test_apps/test_volume.py @@ -23,9 +23,11 @@ def _spec(name="r", gpu=None, cpu=None): return ResourceSpec(kind=ResourceKind.TASK, name=name, gpu=gpu, cpu=cpu) -def _api(volumes=None, created=None): +def _api(volumes=None, created=None, global_volumes=None): api = AsyncMock() api.list_network_volumes.return_value = volumes or [] + api.list_global_volumes.return_value = global_volumes or [] + api.create_global_volume.return_value = {"id": "gv-new", "name": "models"} api.create_network_volume.return_value = created or { "id": "nv-new", "name": "models", @@ -174,26 +176,57 @@ def test_missing_creates_with_placement(self): assert resolved["id"] == "nv-new" api.create_network_volume.assert_awaited_once() - def test_missing_no_create_raises(self): + @pytest.mark.parametrize("volume_type", [NetworkVolume, GlobalVolume]) + def test_missing_no_create_raises(self, volume_type): api = _api() resolver = VolumeResolver(api) with pytest.raises(VolumeError, match="create=False"): asyncio.run( - resolver.resolve( - NetworkVolume("models", create=False), [_spec(gpu=None)] - ) + resolver.resolve(volume_type("models", create=False), [_spec(gpu=None)]) ) - def test_duplicate_names_raise(self): + @pytest.mark.parametrize("volume_type", [NetworkVolume, GlobalVolume]) + def test_duplicate_names_raise(self, volume_type): api = _api( volumes=[ {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"}, {"id": "nv-2", "name": "models", "size": 50, "dataCenter": "US-KS-2"}, ] ) + api.list_global_volumes.return_value = api.list_network_volumes.return_value resolver = VolumeResolver(api) with pytest.raises(VolumeError, match="reference by id"): - asyncio.run(resolver.resolve(NetworkVolume("models"), [_spec(gpu=None)])) + asyncio.run(resolver.resolve(volume_type("models"), [_spec(gpu=None)])) + + @pytest.mark.parametrize("reference", ["models", "gv-1"]) + def test_existing_global_resolves_without_creation_or_placement(self, reference): + api = _api(global_volumes=[{"id": "gv-1", "name": "models"}]) + resolved = asyncio.run( + VolumeResolver(api).resolve(GlobalVolume(reference, create=False)) + ) + assert resolved == {"id": "gv-1"} + api.create_global_volume.assert_not_awaited() + api.list_network_volumes.assert_not_awaited() + + def test_global_creation_is_cached_and_mounts_resolve_by_name_and_id( + self, tmp_path + ): + (tmp_path / "model").write_text("global weights") + api = _api() + resolver = VolumeResolver(api) + + async def run(): + bindings = await resolver.resolve_mounts( + {str(tmp_path): GlobalVolume("models")} + ) + assert await resolver.resolve(GlobalVolume("models")) == {"id": "gv-new"} + return bindings + + _configure_mounts(asyncio.run(run())) + assert (GlobalVolume("models").path / "model").read_text() == "global weights" + assert (GlobalVolume("gv-new").path / "model").read_text() == "global weights" + api.create_global_volume.assert_awaited_once() + api.list_network_volumes.assert_not_awaited() def test_resolution_cached_per_name(self): api = _api( @@ -214,7 +247,7 @@ def test_same_reference_in_different_backends_does_not_share_cache(self): resolver = VolumeResolver( _api(volumes=[{"id": "nv-1", "name": "models", "dataCenter": "EU-RO-1"}]) ) - assert asyncio.run(resolver.resolve(GlobalVolume("models"))) == {"id": "models"} + assert asyncio.run(resolver.resolve(GlobalVolume("models"))) == {"id": "gv-new"} assert asyncio.run(resolver.resolve(NetworkVolume("models"))) == { "id": "nv-1", "dataCenterId": "EU-RO-1", @@ -244,9 +277,10 @@ def test_pod_pins_to_volume_dc(self): from runpod.apps.tasks import TaskExecution api = _api( + global_volumes=[{"id": "gv-1", "name": "shared-global"}], volumes=[ {"id": "nv-1", "name": "models", "size": 50, "dataCenter": "EU-RO-1"} - ] + ], ) spec = ResourceSpec( kind=ResourceKind.TASK, @@ -298,18 +332,18 @@ def test_global_attachment_does_not_pin_datacenter(self): app = App("global-model") - @app.queue(gpu="4090", mounts={"/runpod-volume": GlobalVolume("gv-1")}) + @app.queue(gpu="4090", mounts={"/runpod-volume": GlobalVolume("models")}) def generate(): return None payload = {"locations": "EU-RO-1,US-KS-2"} asyncio.run( - attach_endpoint_volumes(payload, generate.spec, VolumeResolver(), app) + attach_endpoint_volumes(payload, generate.spec, VolumeResolver(_api()), app) ) assert payload["locations"] == "EU-RO-1,US-KS-2" assert payload["networkVolumeIds"] == [] assert payload["volumes"] == [ - {"volumeId": "gv-1", "volumeType": "OBJECT_STORE_VOLUME"} + {"volumeId": "gv-new", "volumeType": "OBJECT_STORE_VOLUME"} ] def test_removing_storage_clears_backend_and_worker_bindings(self): From 7363835496b9b2a2d71cc89af6d2447a0fdd379f Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:12:44 -0700 Subject: [PATCH 115/120] fix: preserve sandbox ownership across failures --- .github/workflows/ci.yml | 3 +- CHANGELOG.md | 6 +- README.md | 14 ++ runpod/api/sandboxes.py | 45 ++++- runpod/sandbox/__init__.py | 3 + runpod/sandbox/asyncio.py | 36 +++- runpod/sandbox/sync.py | 250 ++++++++++++++++++------ tests/e2e/conftest.py | 6 +- tests/e2e/test_sandbox.py | 98 ++++++++++ tests/test_sandbox.py | 384 ++++++++++++++++++++++++++++++++++++- 10 files changed, 772 insertions(+), 73 deletions(-) create mode 100644 tests/e2e/test_sandbox.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfc262c47..222632f07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,8 @@ jobs: - name: Run e2e tests run: | source .venv/bin/activate - pytest tests/e2e/ -v -p no:xdist --timeout=600 --reruns 1 --reruns-delay 5 --log-cli-level=INFO -o "addopts=" + # sandbox cloud checks run explicitly against an authorized dev environment. + pytest tests/e2e/ --ignore=tests/e2e/test_sandbox.py -v -p no:xdist --timeout=600 --reruns 1 --reruns-delay 5 --log-cli-level=INFO -o "addopts=" env: RUNPOD_API_KEY: ${{ secrets.RUNPOD_API_KEY }} RUNPOD_SDK_GIT_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fc2c99c4..50e3a68ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ ### Features * Apps expose `NetworkVolume` and `GlobalVolume` resources with explicit path-keyed mounts, lazy create-if-missing provisioning, and execution-bound filesystem access. Global volumes use GraphQL for lookup and creation. -* Sandboxes support shared storage references, structured ports, execution details, and lifetime updates through synchronous and asynchronous handles. +* Sandboxes support shared storage references, structured ports, execution details, command timeouts, detached execution, and lifetime updates through synchronous and asynchronous handles. + +### Fixes + +* Sandbox ownership cleanup survives interrupted handle delivery and overlapping context entry. Concurrent requests settle during loop shutdown, and failed creation preserves recoverable sandbox IDs and cleanup errors. ## [1.12.0](https://github.com/runpod/runpod-python/compare/v1.11.0...v1.12.0) (2026-08-10) diff --git a/README.md b/README.md index 7c10d2879..a985e3552 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,12 @@ The container stops when its main command exits. `Sandbox.create(...)` returns an owned handle. `get(...)` and `list(...)` return borrowed handles whose contexts only close local connections. `terminate()` releases remote compute; `close()` releases local resources and permits reuse. +Each handle permits one active context; overlapping entry is rejected. + +Failed creation triggers bounded cleanup. `SandboxCreationError`, exported from +`runpod.sandbox`, preserves the backend's `sandbox_id`, HTTP status, and error +details. If termination also fails, the cleanup error is chained as `__cause__`; +the ID remains available for explicit recovery with `get(...)` and `terminate()`. `info`, `state`, `compute`, and `expires_at` are cached snapshots. `refresh()` fetches a new snapshot, including mount, port, SSH, and placement metadata. @@ -147,6 +153,14 @@ backend extension contract accepts an absolute lifetime. Only explicit `sandbox_starting` conflicts are retried within `startup_timeout`. Other conflicts, malformed responses, and ambiguous transport failures propagate. +`exec(..., timeout_seconds=30)` requests a command timeout from 1 to 50 seconds. +The backend currently caps foreground execution at about four seconds. +`request_timeout` separately limits the client HTTP wait. +For longer work, use `background=True` and poll for completion. A successful +detached result confirms that the command started, not that it finished; redirect +output to a file to retrieve it afterward. These options require backend support +for `timeoutSeconds` and `background`. + `logs(source="container", tail=10)` streams main-process logs; `source="system"` selects lifecycle events. Exec output is returned by `exec`. Use `with` and regular iteration for synchronous streams, or `async with` and `async for` for diff --git a/runpod/api/sandboxes.py b/runpod/api/sandboxes.py index ebd939c0c..76552f68e 100644 --- a/runpod/api/sandboxes.py +++ b/runpod/api/sandboxes.py @@ -29,6 +29,19 @@ _LINE_END = re.compile(r"\r\n|\r|\n") +class SandboxCreationError(error.QueryError): + """a failed creation with a recoverable, potentially billable sandbox id.""" + + def __init__(self, sandbox_id: str, cause: error.QueryError) -> None: + super().__init__( + cause.message, + cause.query, + status_code=cause.status_code, + errors=cause.errors, + ) + self.sandbox_id = sandbox_id + + class SandboxConflictError(error.QueryError): """a structured rejection whose code determines whether startup can be retried.""" @@ -60,13 +73,25 @@ async def _raise_response_error( payload = {} if not isinstance(payload, dict): payload = {} - if response.status == 409: + sandbox_id = ( + payload.get("sandboxId") + if method == "POST" and path == _SANDBOX_PATH + else None + ) + if not isinstance(sandbox_id, str) or not sandbox_id: + sandbox_id = None + if response.status == 409 and sandbox_id is None: raise SandboxConflictError(payload, method, path) try: text = raw.decode(response.charset or "utf-8", errors="replace") except LookupError: text = raw.decode("utf-8", errors="replace") - _raise_for_status(response.status, payload, text, method, path) + try: + _raise_for_status(response.status, payload, text, method, path) + except error.QueryError as cause: + if sandbox_id is not None: + raise SandboxCreationError(sandbox_id, cause) from cause + raise # Redirects are deliberately not followed, including on read operations. raise error.QueryError( text or f"Unexpected HTTP status {response.status}", @@ -434,9 +459,21 @@ async def extend( async def terminate(self, sandbox_id: str) -> None: await self._request("DELETE", _sandbox_path(sandbox_id)) - async def exec(self, sandbox_id: str, command: Sequence[str]) -> dict[str, Any]: + async def exec( + self, + sandbox_id: str, + command: Sequence[str], + *, + timeout_seconds: int | None = None, + background: bool = False, + ) -> dict[str, Any]: path = f"{_sandbox_path(sandbox_id)}/exec" - payload = await self._request("POST", path, body={"command": list(command)}) + body: dict[str, Any] = {"command": list(command)} + if timeout_seconds is not None: + body["timeoutSeconds"] = timeout_seconds + if background: + body["background"] = True + payload = await self._request("POST", path, body=body) if payload is None or not isinstance(payload.get("output"), str): raise error.QueryError( "Sandbox execution response must contain a string 'output'", diff --git a/runpod/sandbox/__init__.py b/runpod/sandbox/__init__.py index 9bcb930d5..dd58cfcb8 100644 --- a/runpod/sandbox/__init__.py +++ b/runpod/sandbox/__init__.py @@ -1,5 +1,7 @@ """Sync and async sandbox lifecycles, execution results, and log streams.""" +from runpod.api.sandboxes import SandboxCreationError + from .asyncio import AsyncioSandbox, AsyncSandboxLogStream from .models import ( ExecResult, @@ -24,6 +26,7 @@ "LogSource", "SandboxCompute", "SandboxExecutionError", + "SandboxCreationError", "SandboxInfo", "SandboxStartupTimeout", "SandboxState", diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index 60d05ac8c..1f1b43066 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -8,7 +8,11 @@ from types import TracebackType from typing import Any, Literal, Optional, TypeVar -from runpod.api.sandboxes import AsyncSandboxAPI, SandboxConflictError +from runpod.api.sandboxes import ( + AsyncSandboxAPI, + SandboxConflictError, + SandboxCreationError, +) from runpod.apps.registry import resolve_registry_auth from runpod.apps.volume import Volume, VolumeResolver, normalize_mounts, validate_mounts from runpod.error import QueryError @@ -333,7 +337,12 @@ async def create_remote() -> None: body["registry"] = await resolve_registry_auth( self._registry_auth, api=self._api ) - data = await self._api.create(body) + try: + data = await self._api.create(body) + except SandboxCreationError as error: + self._sandbox_id = error.sandbox_id + self._owned = True + raise sandbox_id = data["id"] if not isinstance(sandbox_id, str) or not sandbox_id: raise ValueError("Create response has no valid sandbox id") @@ -504,6 +513,8 @@ async def exec( *, check: bool = False, startup_timeout: Optional[float] = None, + timeout_seconds: Optional[int] = None, + background: bool = False, ) -> ExecResult: """execute argv, retrying only explicit 409 startup rejections. @@ -513,6 +524,11 @@ async def exec( might already have executed. ``check=False`` returns command failures in ``ExecResult.error``, matching ``subprocess.run``. use ``check=True`` to raise ``SandboxExecutionError`` with partial output on failure. + + ``timeout_seconds`` (1–50) requests a command limit; the backend may + impose a shorter cap. ``request_timeout`` bounds the client http wait. + ``background=True`` returns after starting a detached command, not its + completion; redirect its output to a file and poll for results. """ if isinstance(command, (str, bytes)) or not isinstance(command, Sequence): raise TypeError( @@ -520,9 +536,23 @@ async def exec( ) if not command or any(not isinstance(argument, str) for argument in command): raise ValueError("command must be a nonempty sequence of strings") + if timeout_seconds is not None and ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, int) + or not 1 <= timeout_seconds <= 50 + ): + raise ValueError("timeout_seconds must be an integer from 1 to 50") + if not isinstance(background, bool): + raise TypeError("background must be a bool") argv = tuple(command) data = await self._ready_operation( - lambda: self._api.exec(self.id, argv), startup_timeout + lambda: self._api.exec( + self.id, + argv, + timeout_seconds=timeout_seconds, + background=background, + ), + startup_timeout, ) result = ExecResult( output=data["output"], diff --git a/runpod/sandbox/sync.py b/runpod/sandbox/sync.py index a5fac5981..3a23c2fad 100644 --- a/runpod/sandbox/sync.py +++ b/runpod/sandbox/sync.py @@ -16,7 +16,7 @@ ) from runpod.apps.volume import Volume -from runpod.sandbox.asyncio import AsyncioSandbox +from runpod.sandbox.asyncio import AsyncioSandbox, _cleanup from .models import ( ExecResult, LogEvent, @@ -35,7 +35,12 @@ class _LoopRunner: def __init__(self) -> None: self._loop: Optional[asyncio.AbstractEventLoop] = None self._thread: Optional[threading.Thread] = None - self._lock = threading.Lock() + self._lock = threading.RLock() + self._stopping = False + self._stopped = threading.Event() + self._stopped.set() + self._loop_finished = threading.Event() + self._tasks: dict[asyncio.Task, bool] = {} @property def started(self) -> bool: @@ -45,6 +50,7 @@ def _start(self) -> asyncio.AbstractEventLoop: with self._lock: if self._loop is None: ready = threading.Event() + self._loop_finished.clear() def serve() -> None: loop = asyncio.new_event_loop() @@ -53,9 +59,12 @@ def serve() -> None: try: loop.run_forever() finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.run_until_complete(loop.shutdown_default_executor()) - loop.close() + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + loop.close() + finally: + self._loop_finished.set() self._thread = threading.Thread( target=serve, name="runpod-sandbox", daemon=True @@ -65,11 +74,13 @@ def serve() -> None: assert self._loop is not None return self._loop - def run(self, coroutine: Coroutine[Any, Any, _T]) -> _T: + def run( + self, coroutine: Coroutine[Any, Any, _T], *, cleanup: bool = False + ) -> _T: if threading.current_thread() is self._thread: coroutine.close() raise RuntimeError("Cannot call the sync sandbox API from its own loop") - loop = self._start() + loop: asyncio.AbstractEventLoop future: Future[_T] = Future() finished = threading.Event() task: Optional[asyncio.Task[tuple[Optional[_T], Optional[BaseException]]]] = ( @@ -87,6 +98,7 @@ async def invoke() -> tuple[Optional[_T], Optional[BaseException]]: return None, error def complete(done: asyncio.Task) -> None: + self._tasks.pop(done, None) try: result, error = done.result() if error is not None: @@ -104,50 +116,102 @@ def complete(done: asyncio.Task) -> None: def submit() -> None: nonlocal task task = loop.create_task(invoke()) + self._tasks[task] = cleanup task.add_done_callback(complete) def cancel() -> None: if task is not None: task.cancel() - loop.call_soon_threadsafe(submit) try: - return future.result() - except BaseException: - if not future.done(): - # Cancelling a concurrent Future marks it done before the async - # task's finally blocks finish. Wait for the actual task before - # stopping its loop, or Ctrl-C can strand a newly created sandbox. - loop.call_soon_threadsafe(cancel) - while not finished.is_set(): + while True: + with self._lock: + if not self._stopping: + loop = self._start() + loop.call_soon_threadsafe(submit) + break + if not cleanup: + raise RuntimeError("Sandbox loop is shutting down") + stopped = self._stopped + while not stopped.is_set(): try: - finished.wait() + stopped.wait() except KeyboardInterrupt: - # Cleanup is bounded by the async domain's timeouts. + # owner cleanup must survive repeated interruptions. continue + except BaseException: + coroutine.close() + raise + try: + return future.result() + except BaseException as original: + with self._lock: + if not cleanup and not finished.is_set() and not self._stopping: + loop.call_soon_threadsafe(cancel) + while not finished.is_set(): + try: + finished.wait() + except KeyboardInterrupt: + # async cleanup owns its child tasks and bounds their lifetime. + continue + error = future.exception() + if error is not None and error is not original: + raise original from error raise def stop(self) -> None: + if threading.current_thread() is self._thread: + raise RuntimeError("Cannot stop the sync sandbox API from its own loop") with self._lock: - loop, thread = self._loop, self._thread - if loop is None or thread is None: - return - - async def drain() -> None: - current = asyncio.current_task() - tasks = [task for task in asyncio.all_tasks() if task is not current] - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - + if self._stopping: + stopped = self._stopped + shutdown = None + else: + loop, thread = self._loop, self._thread + if loop is None or thread is None: + return + self._stopping = True + stopped = self._stopped = threading.Event() + + async def drain() -> None: + # cancel invocations, not the protected cleanup tasks they own. + tasks = tuple(self._tasks) + for task in tasks: + if not self._tasks[task]: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + shutdown = asyncio.run_coroutine_threadsafe(drain(), loop) + interrupted = None + if shutdown is None: + while not stopped.is_set(): + try: + stopped.wait() + except KeyboardInterrupt as error: + interrupted = error + else: try: - asyncio.run_coroutine_threadsafe(drain(), loop).result() + while True: + try: + shutdown.result() + break + except KeyboardInterrupt as error: + interrupted = error finally: loop.call_soon_threadsafe(loop.stop) - thread.join() - self._loop = None - self._thread = None + while not self._loop_finished.is_set(): + try: + self._loop_finished.wait() + except KeyboardInterrupt as error: + interrupted = error + with self._lock: + self._loop = None + self._thread = None + self._stopping = False + stopped.set() + if interrupted is not None: + raise interrupted class SandboxLogs(Iterator[LogEvent]): @@ -239,6 +303,7 @@ def __init__( ) -> None: self._runner = _LoopRunner() self._entered = False + self._context_lock = threading.Lock() self._sandbox = AsyncioSandbox( image=image, template_id=template_id, @@ -272,18 +337,27 @@ def _from_async( instance._sandbox = sandbox instance._runner = runner if runner is not None else _LoopRunner() instance._entered = False + instance._context_lock = threading.Lock() return instance @classmethod def create(cls, **options: Any) -> "Sandbox": """Create immediately; the returned snapshot may still be CREATING.""" sandbox = cls(**options) + completed = False + + async def create() -> AsyncioSandbox: + nonlocal completed + await sandbox._sandbox._create() + completed = True + return sandbox._sandbox + try: - sandbox._runner.run(sandbox._sandbox._create()) - except BaseException: - sandbox.close() + sandbox._runner.run(create()) + return sandbox + except BaseException as original: + sandbox._abort_entry(original, completed) raise - return sandbox @classmethod def get( @@ -297,20 +371,31 @@ def get( ) -> "Sandbox": """Fetch a borrowed handle; closing its context does not terminate it.""" runner = _LoopRunner() - try: - sandbox = runner.run( - AsyncioSandbox.get( - sandbox_id, - api_key=api_key, - base_url=base_url, - request_timeout=request_timeout, - startup_timeout=startup_timeout, - ) + sandbox = None + + async def get() -> AsyncioSandbox: + nonlocal sandbox + sandbox = await AsyncioSandbox.get( + sandbox_id, + api_key=api_key, + base_url=base_url, + request_timeout=request_timeout, + startup_timeout=startup_timeout, ) - except BaseException: - runner.stop() + return sandbox + + try: + result = runner.run(get()) + return cls._from_async(result, runner) + except BaseException as original: + if sandbox is not None: + cls._from_async(sandbox, runner)._abort_entry(original, True) + else: + try: + runner.stop() + except BaseException as error: + raise original from error raise - return cls._from_async(sandbox, runner) @classmethod def list( @@ -388,10 +473,18 @@ def exec( *, check: bool = False, startup_timeout: Optional[float] = None, + timeout_seconds: Optional[int] = None, + background: bool = False, ) -> ExecResult: """Execute argv, waiting only for explicit startup rejections.""" return self._runner.run( - self._sandbox.exec(command, check=check, startup_timeout=startup_timeout) + self._sandbox.exec( + command, + check=check, + startup_timeout=startup_timeout, + timeout_seconds=timeout_seconds, + background=background, + ) ) def logs( @@ -430,21 +523,58 @@ def close(self) -> None: finally: self._runner.stop() + def _abort_entry(self, original: BaseException, completed: bool) -> None: + async def release() -> None: + try: + # shutdown may have closed the session's loop before handoff. + await _cleanup(self._sandbox.close(), self._sandbox._request_timeout) + finally: + if completed: + await self._sandbox.__aexit__( + type(original), original, original.__traceback__ + ) + + try: + try: + self._runner.run(release(), cleanup=True) + finally: + self._runner.stop() + except BaseException as error: + if error is not original: + raise original from error + raise + def __enter__(self) -> "Sandbox": - if self._entered: - raise RuntimeError("Sandbox context is already entered") + with self._context_lock: + if self._entered: + raise RuntimeError("Sandbox context is already entered") + self._entered = True + completed = False + + async def enter() -> AsyncioSandbox: + nonlocal completed + result = await self._sandbox.__aenter__() + completed = True + return result + try: - self._runner.run(self._sandbox.__aenter__()) - except BaseException: - self._runner.stop() + self._runner.run(enter()) + return self + except BaseException as original: + try: + self._abort_entry(original, completed) + finally: + with self._context_lock: + self._entered = False raise - self._entered = True - return self def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: try: self._runner.run(self._sandbox.__aexit__(exc_type, exc, traceback)) return False finally: - self._entered = False - self._runner.stop() + try: + self._runner.stop() + finally: + with self._context_lock: + self._entered = False diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8bb8b577c..b0d861fca 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -8,8 +8,6 @@ import pytest import runpod -from tests.e2e.e2e_provisioner import load_test_cases, provision_endpoints - log = logging.getLogger(__name__) REQUEST_TIMEOUT = 300 # seconds per job request @@ -41,6 +39,8 @@ def require_api_key(): @pytest.fixture(scope="session") def test_cases(): """Load test cases from tests.json.""" + from tests.e2e.e2e_provisioner import load_test_cases + cases = load_test_cases() log.info("Loaded %d test cases: %s", len(cases), [c.get("id") for c in cases]) return cases @@ -52,6 +52,8 @@ def endpoints(require_api_key, test_cases): Endpoints deploy lazily on first .run()/.runsync() call. """ + from tests.e2e.e2e_provisioner import provision_endpoints + eps = provision_endpoints(test_cases) for key, ep in eps.items(): log.info("Endpoint ready: name=%s image=%s template.dockerArgs=%s", ep.name, ep.image, ep.template.dockerArgs if ep.template else "N/A") diff --git a/tests/e2e/test_sandbox.py b/tests/e2e/test_sandbox.py new file mode 100644 index 000000000..dac6da597 --- /dev/null +++ b/tests/e2e/test_sandbox.py @@ -0,0 +1,98 @@ +import asyncio +import os +import time + +import pytest + +from runpod import AsyncioSandbox, Sandbox +from runpod.sandbox import SandboxExecutionError + + +async def _read_with_borrowed_async(sandbox_id, connection_options): + async with await AsyncioSandbox.get(sandbox_id, **connection_options) as borrowed: + result = await borrowed.exec( + ["python", "-c", "print('borrowed async')"], + check=True, + timeout_seconds=4, + ) + assert result.output.strip() == "borrowed async" + + +def test_sandbox_execution_and_borrowed_handles(require_api_key): + connection_options = { + "api_key": os.environ["RUNPOD_API_KEY"], + "request_timeout": 30, + "startup_timeout": 90, + } + with Sandbox( + image="python:3.12-slim", + cmd=["sleep", "infinity"], + vcpu_count=2, + memory_in_gb=4, + idle_timeout_seconds=180, + max_lifetime_seconds=300, + **connection_options, + ) as sandbox: + result = sandbox.exec( + ["python", "-c", "print('foreground ready')"], + check=True, + timeout_seconds=4, + ) + assert result.output.strip() == "foreground ready" + + with pytest.raises(SandboxExecutionError) as failure: + sandbox.exec( + ["python", "-c", "print('expected failure', flush=True); raise SystemExit(17)"], + check=True, + timeout_seconds=4, + ) + assert failure.value.sandbox_id == sandbox.id + assert "expected failure" in failure.value.result.output + + sandbox.exec( + [ + "python", + "-c", + "import pathlib, time; time.sleep(6); " + "pathlib.Path('/tmp/runpod-sandbox-e2e').write_text('detached complete')", + ], + check=True, + timeout_seconds=15, + background=True, + ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + result = sandbox.exec( + [ + "python", + "-c", + "import pathlib; path = pathlib.Path('/tmp/runpod-sandbox-e2e'); " + "print(path.read_text() if path.exists() else 'pending')", + ], + check=True, + timeout_seconds=4, + startup_timeout=0, + ) + if result.output.strip() == "detached complete": + break + assert result.output.strip() == "pending", result + time.sleep(1) + else: + pytest.fail("detached command did not complete within 30 seconds") + + with Sandbox.get(sandbox.id, **connection_options) as borrowed: + result = borrowed.exec( + ["python", "-c", "print('borrowed sync')"], + check=True, + timeout_seconds=4, + ) + assert result.output.strip() == "borrowed sync" + + asyncio.run(_read_with_borrowed_async(sandbox.id, connection_options)) + result = sandbox.exec( + ["python", "-c", "print('owner still running')"], check=True + ) + assert result.output.strip() == "owner still running" + + with Sandbox.get(sandbox.id, **connection_options) as terminated: + assert terminated.info.state == "TERMINATED" diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index c8a446318..08fd15374 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -5,6 +5,7 @@ import sys import threading from collections import deque +from concurrent.futures import Future from contextlib import contextmanager from datetime import datetime, timedelta @@ -15,6 +16,7 @@ from runpod.apps.volume import GlobalVolume, NetworkVolume, VolumeError from runpod.error import AuthenticationError, QueryError from runpod.sandbox import ( + SandboxCreationError, SandboxExecutionError, SandboxStartupTimeout, SandboxStateError, @@ -37,6 +39,12 @@ def __init__(self): self.global_error = None self.registries = [] self.delete_status = 204 + self.create_status = 201 + self.create_error = "container provisioning failed" + self.job_started = threading.Event() + self.job_completed = threading.Event() + self.allow_job = threading.Event() + self.jobs = set() self.create_started = threading.Event() self.allow_create = threading.Event() self.allow_create.set() @@ -147,6 +155,12 @@ async def handle(request): record = service.create_record(body) service.create_started.set() await asyncio.to_thread(service.allow_create.wait, 5) + if service.create_status != 201: + return web.json_response( + {"detail": service.create_error, "sandboxId": record["id"]}, + status=service.create_status, + content_type="application/problem+json", + ) return web.json_response(record, status=201) state = request.query.get("state") labels = [term.split("=", 1) for term in request.query.getall("labels", [])] @@ -190,6 +204,14 @@ async def handle(request): record.update(state="TERMINATED", compute=None) return web.Response(status=204) if operation == "exec": + timeout = body.get("timeoutSeconds", 50) + background = body.get("background", False) + if ( + type(timeout) is not int + or not 1 <= timeout <= 50 + or type(background) is not bool + ): + return failure(422, "invalid execution controls") status = service.exec_statuses.popleft() if service.exec_statuses else 200 if status == 409 or record["state"] in ("FAILED", "TERMINATED"): return failure(409, "container not started", service.conflict_code) @@ -198,6 +220,36 @@ async def handle(request): return failure(status, "response lost after command execution") if service.exec_responses: return service.exec_responses.popleft() + if body["command"] == ["wait-for-release"]: + async def run_job(): + service.job_started.set() + while not service.allow_job.is_set(): + if service.stopping.is_set(): + return + await asyncio.sleep(0.01) + service.job_completed.set() + + if background: + task = asyncio.create_task(run_job()) + service.jobs.add(task) + task.add_done_callback(service.jobs.discard) + return web.json_response({"output": "started", "exitCode": None}) + try: + await asyncio.wait_for(run_job(), timeout) + except asyncio.TimeoutError: + return web.json_response( + {"output": "started", "error": "execution timed out"} + ) + return web.json_response({"output": "finished", "exitCode": 0}) + if body["command"] == ["job-status"]: + return web.json_response( + { + "output": ( + "completed" if service.job_completed.is_set() else "pending" + ), + "exitCode": 0, + } + ) result = ( {"output": "partial output", "error": "command failed"} if body["command"] == ["fail"] @@ -253,8 +305,20 @@ async def serve(): await stopped.wait() finally: await runner.cleanup() + if service.jobs: + await asyncio.gather(*service.jobs) - thread = threading.Thread(target=lambda: asyncio.run(serve()), daemon=True) + outcome = Future() + + def run_peer(): + try: + asyncio.run(serve()) + except BaseException as error: + outcome.set_exception(error) + else: + outcome.set_result(None) + + thread = threading.Thread(target=run_peer, daemon=True) thread.start() assert ready.wait(5), "local sandbox peer did not start" try: @@ -262,9 +326,12 @@ async def serve(): finally: service.allow_create.set() service.allow_logs.set() + service.allow_job.set() service.stopping.set() service.loop.call_soon_threadsafe(stopped.set) - thread.join() + thread.join(5) + assert not thread.is_alive(), "local sandbox peer did not stop" + outcome.result(timeout=0) @pytest.fixture @@ -273,6 +340,319 @@ def peer(): yield service +@pytest.mark.asyncio +@pytest.mark.parametrize("async_api", [False, True], ids=["sync", "async"]) +async def test_execution_controls_bound_foreground_and_detach_background(peer, async_api): + api = AsyncioSandbox if async_api else Sandbox + + async def invoke(method, *args, **kwargs): + if async_api: + return await method(*args, **kwargs) + return await asyncio.to_thread(method, *args, **kwargs) + + sandbox = await invoke(api.create, image="image", **peer.options) + try: + for controls in ( + {"timeout_seconds": 0}, + {"timeout_seconds": 51}, + {"timeout_seconds": True}, + {"timeout_seconds": 1.5}, + {"background": 1}, + ): + with pytest.raises((TypeError, ValueError)): + await invoke(sandbox.exec, ["wait-for-release"], **controls) + assert peer.executed == [] + assert not any(path.endswith("/exec") for _, path, _, _ in peer.requests) + + with pytest.raises(SandboxExecutionError) as expired: + await invoke( + sandbox.exec, ["wait-for-release"], timeout_seconds=1, check=True + ) + assert expired.value.result.output == "started" + assert expired.value.result.error == "execution timed out" + assert not peer.job_completed.is_set() + + peer.allow_job.set() + completed = await invoke( + sandbox.exec, ["wait-for-release"], timeout_seconds=50, check=True + ) + assert completed.output == "finished" + assert completed.exit_code == 0 + assert peer.job_completed.is_set() + + peer.allow_job.clear() + peer.job_started.clear() + peer.job_completed.clear() + launched = await invoke( + sandbox.exec, + ["wait-for-release"], + timeout_seconds=1, + background=True, + check=True, + ) + assert launched.output == "started" + assert await asyncio.to_thread(peer.job_started.wait, 2) + assert not peer.job_completed.is_set() + assert (await invoke(sandbox.exec, ["job-status"])).output == "pending" + peer.allow_job.set() + assert await asyncio.to_thread(peer.job_completed.wait, 2) + assert (await invoke(sandbox.exec, ["job-status"])).output == "completed" + finally: + peer.allow_job.set() + await invoke(sandbox.terminate) + + +def test_sync_submission_racing_close_settles_and_allows_reuse(): + script = """ +import asyncio +import aiohttp +import threading +from concurrent.futures import Future +from runpod import Sandbox +from runpod.error import QueryError +from tests.test_sandbox import sandbox_peer + +with sandbox_peer() as peer: + sandbox = Sandbox.create(image="image", **peer.options) + loop = sandbox._runner._loop + schedule = loop.call_soon_threadsafe + submitting = threading.Event() + release_submission = threading.Event() + submitted = threading.Event() + stopping = threading.Event() + release_loop = threading.Event() + refreshed = Future() + closed = Future() + + def controlled_schedule(callback, *args, **kwargs): + if threading.current_thread().name == "racing-refresh": + submitting.set() + assert release_submission.wait(4), "submission was not released" + try: + return schedule(callback, *args, **kwargs) + finally: + submitted.set() + if callback == loop.stop: + schedule(release_loop.wait, 4) + result = schedule(callback, *args, **kwargs) + stopping.set() + return result + return schedule(callback, *args, **kwargs) + + def refresh(): + try: + refreshed.set_result(sandbox.refresh()) + except BaseException as error: + refreshed.set_exception(error) + + def close(): + try: + sandbox.close() + closed.set_result(None) + except BaseException as error: + closed.set_exception(error) + + loop.call_soon_threadsafe = controlled_schedule + reader = threading.Thread(target=refresh, name="racing-refresh", daemon=True) + closer = threading.Thread(target=close, daemon=True) + try: + reader.start() + assert submitting.wait(2), "refresh never reached submission" + closer.start() + # admission may exclude shutdown until the submission gate is released. + stopping.wait(1) + release_submission.set() + assert submitted.wait(2), "refresh submission did not resume" + release_loop.set() + try: + assert refreshed.result(timeout=3).state == "RUNNING" + except (RuntimeError, QueryError, asyncio.CancelledError, aiohttp.ClientConnectionError): + pass + closed.result(timeout=3) + reader.join(2) + closer.join(2) + assert not reader.is_alive(), "refresh was abandoned by shutdown" + assert not closer.is_alive(), "close did not finish" + assert sandbox.refresh().state == "RUNNING" + finally: + release_submission.set() + release_loop.set() + loop.call_soon_threadsafe = schedule + sandbox.terminate() + assert peer.records[sandbox.id]["state"] == "TERMINATED" +""" + process = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=15 + ) + assert process.returncode == 0, process.stdout + process.stderr + + +@pytest.mark.parametrize("delete_status", [204, 503]) +def test_sync_failed_creation_preserves_resource_id_and_cleans_up(peer, delete_status): + peer.create_status = 502 + peer.delete_status = delete_status + with pytest.raises(SandboxCreationError) as failed: + Sandbox.create(image="image", **peer.options) + error = failed.value + assert isinstance(error, QueryError) + assert error.sandbox_id == "sandbox-1" + assert error.status_code == 502 + assert peer.create_error in str(error) + assert [method for method, path, _, _ in peer.requests if path == "/v2/sandboxes"] == [ + "POST" + ] + assert [ + method + for method, path, _, _ in peer.requests + if path == "/v2/sandboxes/sandbox-1" + ] == ["DELETE"] + if delete_status == 204: + assert peer.records[error.sandbox_id]["state"] == "TERMINATED" + else: + assert isinstance(error.__cause__, QueryError) + assert error.__cause__.status_code == 503 + assert peer.records[error.sandbox_id]["state"] == "RUNNING" + peer.delete_status = 204 + Sandbox.get(error.sandbox_id, **peer.options).terminate() + assert peer.records[error.sandbox_id]["state"] == "TERMINATED" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delete_status", [204, 503]) +async def test_async_failed_creation_preserves_resource_id_and_cleans_up( + peer, delete_status +): + peer.create_status = 502 + peer.delete_status = delete_status + with pytest.raises(SandboxCreationError) as failed: + await AsyncioSandbox.create(image="image", **peer.options) + error = failed.value + assert error.sandbox_id == "sandbox-1" + assert error.status_code == 502 + assert peer.create_error in str(error) + assert [method for method, path, _, _ in peer.requests if path == "/v2/sandboxes"] == [ + "POST" + ] + assert [ + method + for method, path, _, _ in peer.requests + if path == "/v2/sandboxes/sandbox-1" + ] == ["DELETE"] + if delete_status == 204: + assert peer.records[error.sandbox_id]["state"] == "TERMINATED" + else: + assert isinstance(error.__cause__, QueryError) + assert error.__cause__.status_code == 503 + peer.delete_status = 204 + recovery = await AsyncioSandbox.get(error.sandbox_id, **peer.options) + await recovery.terminate() + assert peer.records[error.sandbox_id]["state"] == "TERMINATED" + + +@pytest.mark.parametrize( + "operation, delete_status", + [ + ("create", 204), + ("enter", 204), + ("get", 204), + ("borrowed_enter", 204), + ("create", 503), + ("enter", 503), + ], +) +def test_sync_interrupt_at_successful_handle_handoff( + peer, monkeypatch, operation, delete_status +): + peer.delete_status = delete_status + borrowed = operation in ("get", "borrowed_enter") + handle = None + if borrowed: + peer.create_record({"imageName": "image"}) + if operation == "borrowed_enter": + handle = Sandbox.get("sandbox-1", **peer.options) + elif operation == "enter": + handle = Sandbox(image="image", **peer.options) + original_result = Future.result + caller = threading.get_ident() + interrupt = KeyboardInterrupt("interrupted after remote success") + armed = True + + def interrupt_result(future, *args, **kwargs): + nonlocal armed + result = original_result(future, *args, **kwargs) + if armed and threading.get_ident() == caller: + armed = False + raise interrupt + return result + + monkeypatch.setattr(Future, "result", interrupt_result) + try: + with pytest.raises(KeyboardInterrupt) as failed: + if operation == "create": + Sandbox.create(image="image", **peer.options) + elif operation == "get": + Sandbox.get("sandbox-1", **peer.options) + else: + handle.__enter__() + assert failed.value is interrupt + assert peer.records["sandbox-1"]["state"] == ( + "RUNNING" if borrowed or delete_status != 204 else "TERMINATED" + ) + deletes = [path for method, path, _, _ in peer.requests if method == "DELETE"] + assert deletes == ([] if borrowed else ["/v2/sandboxes/sandbox-1"]) + if delete_status != 204: + assert isinstance(failed.value.__cause__, QueryError) + assert failed.value.__cause__.status_code == 503 + peer.delete_status = 204 + Sandbox.get("sandbox-1", **peer.options).terminate() + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + if operation == "borrowed_enter": + with handle: + assert handle.refresh().state == "RUNNING" + finally: + armed = False + if handle is not None: + handle.close() + + +def test_overlapping_sync_entries_preserve_the_first_owner(peer): + sandbox = Sandbox(image="image", **peer.options) + peer.allow_create.clear() + leave = threading.Event() + entered = threading.Event() + outcome = Future() + + def own_context(): + try: + with sandbox: + entered.set() + assert leave.wait(4), "test did not release the first context" + assert sandbox.refresh().state == "RUNNING" + outcome.set_result(None) + except BaseException as error: + outcome.set_exception(error) + + worker = threading.Thread(target=own_context, daemon=True) + worker.start() + try: + assert peer.create_started.wait(2) + with pytest.raises(RuntimeError): + sandbox.__enter__() + peer.allow_create.set() + assert entered.wait(3), "first context was disrupted by a rejected entry" + finally: + peer.allow_create.set() + leave.set() + worker.join(5) + assert not worker.is_alive(), "first context did not finish" + try: + outcome.result(timeout=0) + finally: + sandbox.close() + assert peer.records["sandbox-1"]["state"] == "TERMINATED" + assert [method for method, _, _, _ in peer.requests].count("POST") == 1 + + def test_sync_startup_conflict_and_borrowed_context_ownership(peer): peer.exec_statuses.extend([409, 200]) with Sandbox(image="python:3.12-slim", **peer.options) as owner: From c3f531ff9da36dcc74465e0d6db5a0dcade5f272 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:27:17 -0700 Subject: [PATCH 116/120] test: clarify sandbox command and thread boundaries --- tests/e2e/test_sandbox.py | 12 ++++++++---- tests/test_sandbox.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/e2e/test_sandbox.py b/tests/e2e/test_sandbox.py index dac6da597..1520bdb5b 100644 --- a/tests/e2e/test_sandbox.py +++ b/tests/e2e/test_sandbox.py @@ -53,8 +53,10 @@ def test_sandbox_execution_and_borrowed_handles(require_api_key): [ "python", "-c", - "import pathlib, time; time.sleep(6); " - "pathlib.Path('/tmp/runpod-sandbox-e2e').write_text('detached complete')", + ( + "import pathlib, time; time.sleep(6); " + + "pathlib.Path('/tmp/runpod-sandbox-e2e').write_text('detached complete')" + ), ], check=True, timeout_seconds=15, @@ -66,8 +68,10 @@ def test_sandbox_execution_and_borrowed_handles(require_api_key): [ "python", "-c", - "import pathlib; path = pathlib.Path('/tmp/runpod-sandbox-e2e'); " - "print(path.read_text() if path.exists() else 'pending')", + ( + "import pathlib; path = pathlib.Path('/tmp/runpod-sandbox-e2e'); " + + "print(path.read_text() if path.exists() else 'pending')" + ), ], check=True, timeout_seconds=4, diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 08fd15374..8a76d3119 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -313,7 +313,13 @@ async def serve(): def run_peer(): try: asyncio.run(serve()) - except BaseException as error: + except ( + Exception, + asyncio.CancelledError, + KeyboardInterrupt, + SystemExit, + ) as error: + # cancellation and interrupts must reach the test thread. outcome.set_exception(error) else: outcome.set_result(None) @@ -629,7 +635,13 @@ def own_context(): assert leave.wait(4), "test did not release the first context" assert sandbox.refresh().state == "RUNNING" outcome.set_result(None) - except BaseException as error: + except ( + Exception, + asyncio.CancelledError, + KeyboardInterrupt, + SystemExit, + ) as error: + # cancellation and interrupts must reach the test thread. outcome.set_exception(error) worker = threading.Thread(target=own_context, daemon=True) From 8007a549f1f1c5974cccd2bb5eaf7361e8b179d4 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:01:19 -0700 Subject: [PATCH 117/120] docs: isolate sandbox documentation --- README.md | 251 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) diff --git a/README.md b/README.md index a985e3552..f99eb3995 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,30 @@ rp flash dev main.py # live dev session: edit, re-run, logs stream back rp flash deploy # deploy production endpoints ``` +Functions keep their Python identity: `chat.remote(...)` runs in the cloud, `await chat.remote.aio(...)` is the async form, `chat.local(...)` runs in-process. See [`examples/apps`](examples/apps) for runnable examples and [docs.runpod.io](https://docs.runpod.io) for the full guide. + +Queue `.remote()` calls request a synchronous result and poll the same job if the server's wait window expires. Fast results need no client-side polling. Dev sessions use a short sync window to keep worker logs responsive. Use `.spawn()` for an asynchronous job handle. A transport failure before receiving a job ID raises an error without resubmitting the work. + +## Storage + +`NetworkVolume(name_or_id, size=50, datacenter=None, create=True)` references +datacenter-local storage. Apps resolve names and choose a datacenter compatible +with every resource sharing the volume. `GlobalVolume(name_or_id, create=True)` +references global storage without a datacenter constraint. Both inherit from +the abstract `Volume` base, resolve by name or ID, and create missing storage +when provisioning remote compute. Set `create=False` to require existing storage. +Global-volume lookup and creation use GraphQL; network volumes use REST. + +Declare attachments with `mounts={"/path": volume}`. Tasks support one network +and one global volume at distinct, non-overlapping paths. Queue and API resources +support one volume at `/runpod-volume`; global storage requires a GPU endpoint. + +Inside worker code, `volume.path` returns the configured mount path. The runtime +binds declared references and resolved IDs before importing user code. Access +raises if the volume is unmounted or has multiple bindings. Evaluate `.path` +inside remote functions, not during local module discovery. Calling `.local()` +does not mount remote storage on the client machine. + ## Sandboxes Sandboxes are an access-controlled preview. An authorized account and a @@ -222,6 +246,233 @@ You can also test your worker locally before deploying it to Runpod. This is use python my_worker.py --rp_serve_api ``` +### Worker Fitness Checks + +Fitness checks allow you to validate your worker environment at startup before processing jobs. If any check fails, the worker exits immediately, allowing your orchestrator to restart it. + +```python +# my_worker.py + +import runpod +import torch + +# Register fitness checks using the decorator +@runpod.serverless.register_fitness_check +def check_gpu_available(): + """Verify GPU is available.""" + if not torch.cuda.is_available(): + raise RuntimeError("GPU not available") + +@runpod.serverless.register_fitness_check +def check_disk_space(): + """Verify sufficient disk space.""" + import shutil + stat = shutil.disk_usage("/") + free_gb = stat.free / (1024**3) + if free_gb < 10: + raise RuntimeError(f"Insufficient disk space: {free_gb:.2f}GB free") + +def handler(job): + job_input = job["input"] + # Your handler code here + return {"output": "success"} + +# Fitness checks run before handler initialization (production only) +runpod.serverless.start({"handler": handler}) +``` + +**Key Features:** +- Supports both synchronous and asynchronous check functions +- Hardware checks run at the first Serverless `import runpod`; processes it launches inherit the result via `RUNPOD_EARLY_FITNESS_CHECKS_DONE` and skip the pass +- Local tests and non-worker imports remain exempt; network readiness and custom checks run at worker start +- Successful early checks are reused at worker start unless their configuration changes +- Runs before handler initialization and job processing begins +- Any check failure exits with code 1 (worker marked unhealthy) + +See [Worker Fitness Checks](https://github.com/runpod/runpod-python/blob/main/docs/serverless/worker_fitness_checks.md) documentation for more examples and best practices. + +### Network-Volume Warm Cache + +When a network volume is attached, `VolumeCache` warms local directories (such as a model cache) across cold starts — hydrating them on startup and syncing new files back on exit — so a repeated multi-GB model download becomes a one-time cost per endpoint. It is stdlib-only and best-effort. + +```python +from runpod.serverless import VolumeCache + +with VolumeCache(dirs=["/root/.cache/huggingface"]): + model = load_model() +``` + +See [Network-Volume Warm Cache](https://github.com/runpod/runpod-python/blob/main/docs/serverless/volume_cache.md) documentation for configuration and details. + +## 📚 | REST API v2 Wrapper + +Use the API wrapper to manage Runpod resources through REST API v2. + +```python +import runpod + +runpod.api_key = "your_runpod_api_key_found_under_settings" +``` + +### Endpoints + +You can interact with Runpod endpoints via a `run` or `run_sync` method. + +#### Basic Usage + +```python +endpoint = runpod.Endpoint("ENDPOINT_ID") + +run_request = endpoint.run( + {"your_model_input_key": "your_model_input_value"} +) + +# Check the status of the endpoint run request +print(run_request.status()) + +# Get the output of the endpoint run request, blocking until the endpoint run is complete. +print(run_request.output()) +``` + +```python +endpoint = runpod.Endpoint("ENDPOINT_ID") + +run_request = endpoint.run_sync( + {"your_model_input_key": "your_model_input_value"} +) + +# Returns the job results if completed within 90 seconds, otherwise, returns the job status. +print(run_request ) +``` + +#### API Key Management + +The SDK supports multiple ways to set API keys: + +**1. Global API Key** (Default) +```python +import runpod + +# Set global API key +runpod.api_key = "your_runpod_api_key" + +# All endpoints will use this key by default +endpoint = runpod.Endpoint("ENDPOINT_ID") +result = endpoint.run_sync({"input": "data"}) +``` + +**2. Endpoint-Specific API Key** +```python +# Create endpoint with its own API key +endpoint = runpod.Endpoint("ENDPOINT_ID", api_key="specific_api_key") + +# This endpoint will always use the provided API key +result = endpoint.run_sync({"input": "data"}) +``` + +#### API Key Precedence + +The SDK uses this precedence order (highest to lowest): +1. Endpoint instance API key (if provided to `Endpoint()`) +2. Global API key (set via `runpod.api_key`) + +```python +import runpod + +# Example showing precedence +runpod.api_key = "GLOBAL_KEY" + +# This endpoint uses GLOBAL_KEY +endpoint1 = runpod.Endpoint("ENDPOINT_ID") + +# This endpoint uses ENDPOINT_KEY (overrides global) +endpoint2 = runpod.Endpoint("ENDPOINT_ID", api_key="ENDPOINT_KEY") + +# All requests from endpoint2 will use ENDPOINT_KEY +result = endpoint2.run_sync({"input": "data"}) +``` + +#### Thread-Safe Operations + +Each `Endpoint` instance maintains its own API key, making concurrent operations safe: + +```python +import threading +import runpod + +def process_request(api_key, endpoint_id, input_data): + # Each thread gets its own Endpoint instance + endpoint = runpod.Endpoint(endpoint_id, api_key=api_key) + return endpoint.run_sync(input_data) + +# Safe concurrent usage with different API keys +threads = [] +for customer in customers: + t = threading.Thread( + target=process_request, + args=(customer["api_key"], customer["endpoint_id"], customer["input"]) + ) + threads.append(t) + t.start() +``` + +### GPU Cloud (Pods) + +```python +import runpod + +runpod.api_key = "your_runpod_api_key_found_under_settings" + +# get all my pods +pods = runpod.get_pods() + +# get a specific pod +pod = runpod.get_pod(pods[0]["id"]) + +# create a pod with a gpu +pod = runpod.create_pod("test", "runpod/stack", "NVIDIA GeForce RTX 4090") + +# create a pod with a cpu +pod = runpod.create_pod("test", "runpod/stack", instance_id="cpu3c-2-4") + +# stop the pod +runpod.stop_pod(pod["id"]) + +# resume the pod +runpod.resume_pod(pod["id"]) + +# terminate the pod +runpod.terminate_pod(pod["id"]) +``` + +### Template and placement options + +- With `create_pod(template_id=...)`, omitting `docker_args` inherits the template's + command. Pass `docker_args=""` to clear that inherited command. +- Omitting `volume_mount_path` preserves an inherited GPU template volume's path. + Setting it explicitly changes the path while retaining the inherited volume size. + New persistent volumes and network volume mounts default to `/runpod-volume`. + CPU pods do not inherit template persistent volumes; a CPU `volume_mount_path` + requires `network_volume_id`. +- For GPU pods, `min_memory_in_gb` and `min_vcpu_count` specify minimum host RAM + and vCPUs **per GPU**, not GPU VRAM or totals for the pod. +- CPU `instance_id` must use `--`, with positive + integer vCPU and memory values, for example `cpu3c-4-8`. +- REST v2 cannot require a public-IP-capable host. `support_public_ip=False` is + the default and does not disable public networking. `True` raises `ValueError` + rather than silently ignoring that requirement. +- `create_template(volume_in_gb=0)` creates a template without a persistent volume. +- `create_endpoint(locations="US-KS-2,EU-RO-1")` accepts comma-separated + datacenter IDs. Country codes such as `US` and `RO` are not supported. + Endpoint creation sends a single REST request without catalog lookups. +- Endpoint `gpu_ids` accepts pool IDs and excluded GPU types, for example + `gpu_ids="ADA_48_PRO,-NVIDIA L40"` selects that pool without NVIDIA L40 GPUs. +- `create_endpoint` accepts only `QUEUE_DELAY` and `REQUEST_COUNT` scaling. + `idle_timeout` applies to `QUEUE_DELAY` (default: 5 seconds); explicitly setting + it with `REQUEST_COUNT` raises `ValueError`. +- `resume_pod(pod_id)` keeps the existing GPU allocation. REST v2 does not support + resizing on resume, so providing `gpu_count` raises `ValueError`. + ## 📁 | Directory ```BASH From 2f4243be0cc42983c01858f5223344060621a099 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:54:32 -0700 Subject: [PATCH 118/120] fix: align sandbox stream naming and compute types --- runpod/sandbox/__init__.py | 4 +-- runpod/sandbox/asyncio.py | 14 +++++------ runpod/sandbox/models.py | 2 +- tests/test_sandbox.py | 50 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/runpod/sandbox/__init__.py b/runpod/sandbox/__init__.py index dd58cfcb8..f6d2fd128 100644 --- a/runpod/sandbox/__init__.py +++ b/runpod/sandbox/__init__.py @@ -2,7 +2,7 @@ from runpod.api.sandboxes import SandboxCreationError -from .asyncio import AsyncioSandbox, AsyncSandboxLogStream +from .asyncio import AsyncioSandbox, AsyncioSandboxLogStream from .models import ( ExecResult, LogEvent, @@ -18,7 +18,7 @@ __all__ = [ "AsyncioSandbox", - "AsyncSandboxLogStream", + "AsyncioSandboxLogStream", "Sandbox", "SandboxLogs", "ExecResult", diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index 1f1b43066..c6ee087f8 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -211,7 +211,7 @@ def _initialize( self._owned = False self._entering = False self._creating = False - self._streams: set[AsyncSandboxLogStream] = set() + self._streams: set[AsyncioSandboxLogStream] = set() @classmethod async def create(cls, **options: Any) -> "AsyncioSandbox": @@ -575,7 +575,7 @@ def logs( since: Optional[str] = None, last_event_id: Optional[str] = None, startup_timeout: Optional[float] = None, - ) -> "AsyncSandboxLogStream": + ) -> "AsyncioSandboxLogStream": """Return a lazy, closeable stream of main-process or lifecycle logs. This does not include exec output. Only the initial HTTP handshake is @@ -583,7 +583,7 @@ def logs( Use an async context or ``aclose`` when stopping iteration early. """ sandbox_id = self.id - stream = AsyncSandboxLogStream( + stream = AsyncioSandboxLogStream( self, sandbox_id, source, tail, since, last_event_id, startup_timeout ) self._streams.add(stream) @@ -628,7 +628,7 @@ async def close(self) -> None: raise error -class AsyncSandboxLogStream: +class AsyncioSandboxLogStream: """A handle-owned typed log stream with a separately awaitable handshake.""" def __init__( @@ -662,7 +662,7 @@ async def _open_once(self) -> None: await _cleanup(stream.aclose(), self._sandbox._request_timeout, error) raise - async def open(self) -> "AsyncSandboxLogStream": + async def open(self) -> "AsyncioSandboxLogStream": """Complete the HTTP handshake without waiting for the first log event.""" if self._closed: raise RuntimeError("Log stream is closed") @@ -683,7 +683,7 @@ async def open(self) -> "AsyncSandboxLogStream": raise return self - def __aiter__(self) -> "AsyncSandboxLogStream": + def __aiter__(self) -> "AsyncioSandboxLogStream": return self async def __anext__(self) -> LogEvent: @@ -714,7 +714,7 @@ async def aclose(self) -> None: if stream is not None: await stream.aclose() - async def __aenter__(self) -> "AsyncSandboxLogStream": + async def __aenter__(self) -> "AsyncioSandboxLogStream": return await self.open() async def __aexit__( diff --git a/runpod/sandbox/models.py b/runpod/sandbox/models.py index bfdf6b3cd..5bd308526 100644 --- a/runpod/sandbox/models.py +++ b/runpod/sandbox/models.py @@ -16,7 +16,7 @@ def _timestamp(value: str) -> datetime: @dataclass(frozen=True) class SandboxCompute: - vcpu_count: int + vcpu_count: float memory_in_gb: float container_disk_in_gb: int cost_per_hr: float diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 8a76d3119..6f997c676 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -8,11 +8,14 @@ from concurrent.futures import Future from contextlib import contextmanager from datetime import datetime, timedelta +from typing import get_type_hints import pytest from aiohttp import web +import runpod.sandbox as sandbox_domain from runpod import AsyncioSandbox, Sandbox +from runpod.api.sandboxes import AsyncSandboxAPI from runpod.apps.volume import GlobalVolume, NetworkVolume, VolumeError from runpod.error import AuthenticationError, QueryError from runpod.sandbox import ( @@ -346,6 +349,53 @@ def peer(): yield service +@pytest.mark.asyncio +async def test_transport_sessions_disable_connection_retries(): + api = AsyncSandboxAPI() + try: + session = api._get_session() + assert session._retry_connection is False + assert api._get_session() is session + await api.close() + assert session.closed + reopened = api._get_session() + assert reopened is not session + assert reopened._retry_connection is False + finally: + await api.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vcpu_count", [0.5, 2.0, 2]) +async def test_compute_snapshot_preserves_numeric_vcpu_count(peer, vcpu_count): + assert get_type_hints(sandbox_domain.SandboxCompute)["vcpu_count"] is float + record = peer.create_record({}) + record["compute"]["vcpuCount"] = vcpu_count + sandbox = await AsyncioSandbox.get(record["id"], **peer.options) + try: + assert sandbox.compute.vcpu_count == vcpu_count + assert sandbox.info.compute is sandbox.compute + finally: + await sandbox.close() + + +@pytest.mark.asyncio +async def test_async_log_stream_matches_public_export(peer): + assert "AsyncioSandboxLogStream" in sandbox_domain.__all__ + assert ( + get_type_hints(AsyncioSandbox.logs)["return"] + is sandbox_domain.AsyncioSandboxLogStream + ) + async with AsyncioSandbox(image="image", **peer.options) as sandbox: + stream = sandbox.logs() + assert type(stream) is sandbox_domain.AsyncioSandboxLogStream + assert aiter(stream) is stream + assert await stream.open() is stream + async with stream as opened: + assert opened is stream + assert (await anext(opened)).line == "café" + + @pytest.mark.asyncio @pytest.mark.parametrize("async_api", [False, True], ids=["sync", "async"]) async def test_execution_controls_bound_foreground_and_detach_background(peer, async_api): From abb210e6727166038a07a16e5af6703c437fa562 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:31:14 -0700 Subject: [PATCH 119/120] test: separate sandbox stream operations from assertions --- tests/test_sandbox.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index a5c10d1f3..af24e9523 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -400,10 +400,12 @@ async def test_async_log_stream_matches_public_export(peer): stream = sandbox.logs() assert type(stream) is sandbox_domain.AsyncioSandboxLogStream assert aiter(stream) is stream - assert await stream.open() is stream + opened_stream = await stream.open() + assert opened_stream is stream async with stream as opened: assert opened is stream - assert (await anext(opened)).line == "café" + event = await anext(opened) + assert event.line == "café" @pytest.mark.asyncio From 2ed0f3c548284f6e16575004e55d150ddc2fc107 Mon Sep 17 00:00:00 2001 From: zeke <40004347+KAJdev@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:05:16 -0700 Subject: [PATCH 120/120] fix: align sandbox SDK with current REST contract --- runpod/sandbox/asyncio.py | 21 +++--- runpod/sandbox/models.py | 4 +- runpod/sandbox/sync.py | 7 +- tests/e2e/test_sandbox.py | 2 - tests/test_sandbox_contract.py | 114 +++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 15 deletions(-) create mode 100644 tests/test_sandbox_contract.py diff --git a/runpod/sandbox/asyncio.py b/runpod/sandbox/asyncio.py index c6ee087f8..e1c602791 100644 --- a/runpod/sandbox/asyncio.py +++ b/runpod/sandbox/asyncio.py @@ -126,8 +126,11 @@ async def _cleanup( class AsyncioSandbox: """a cached sandbox handle; construction never performs network I/O. - supply exactly one of ``image`` or ``template_id``. unspecified - resource and lifetime options are left to the server's account policy. + supply at most one of ``image`` or ``template_id``; omitting both uses the + server's default image. unspecified resource and lifetime options are left + to the server's account policy. supply ``cpu_flavor_id`` and ``vcpu_count`` + together, or omit both for server defaults. memory is derived by the server + from the CPU flavor and vCPU count. created handles own their remote sandbox in a context; ``get`` and ``list`` return borrowed handles whose context only closes local resources. """ @@ -140,7 +143,6 @@ def __init__( name: Optional[str] = None, cpu_flavor_id: Optional[str] = None, vcpu_count: Optional[int] = None, - memory_in_gb: Optional[int] = None, disk_gb: Optional[int] = None, data_center_ids: Optional[Sequence[str]] = None, mounts: Optional[Mapping[str, Volume]] = None, @@ -164,18 +166,17 @@ def __init__( request_timeout: float = 30, startup_timeout: float = 60, ) -> None: - if (image is None) == (template_id is None): - raise ValueError("Supply exactly one of image or template_id") + if image is not None and template_id is not None: + raise ValueError("Supply at most one of image or template_id") + if (cpu_flavor_id is None) != (vcpu_count is None): + raise ValueError("Supply cpu_flavor_id and vcpu_count together") self._initialize(api_key, base_url, request_timeout, startup_timeout) self._mounts = _sandbox_mounts(mounts) self._registry_auth = registry_auth self._create_body = { - "imageName": image, + "image": image, "templateId": template_id, "name": name, - "cpuFlavorId": cpu_flavor_id, - "vcpuCount": vcpu_count, - "memoryInGb": memory_in_gb, "disk": disk_gb, "dataCenterIds": _string_list( data_center_ids, "data_center_ids", nonempty=True @@ -189,6 +190,8 @@ def __init__( "maxLifetimeSeconds": max_lifetime_seconds, "labels": dict(labels) if labels is not None else None, } + if cpu_flavor_id is not None and vcpu_count is not None: + self._create_body["cpu"] = {"id": cpu_flavor_id, "vcpuCount": vcpu_count} def _initialize( self, diff --git a/runpod/sandbox/models.py b/runpod/sandbox/models.py index 5bd308526..f65da1fed 100644 --- a/runpod/sandbox/models.py +++ b/runpod/sandbox/models.py @@ -70,8 +70,8 @@ def from_dict(cls, data: Mapping[str, Any]) -> "SandboxInfo": created_at=_timestamp(data["createdAt"]), updated_at=_timestamp(data["updatedAt"]), template_id=data.get("templateId"), - image=data.get("imageName"), - cpu_flavor_id=data.get("cpuFlavorId"), + image=data.get("image"), + cpu_flavor_id=(data.get("cpu") or {}).get("id"), termination_reason=data.get("terminationReason"), terminated_at=_timestamp(terminated_at) if terminated_at else None, labels=dict(data["labels"]) if data.get("labels") is not None else None, diff --git a/runpod/sandbox/sync.py b/runpod/sandbox/sync.py index 3a23c2fad..f01acf625 100644 --- a/runpod/sandbox/sync.py +++ b/runpod/sandbox/sync.py @@ -264,6 +264,11 @@ class Sandbox: finished, or ``close`` to release only local connections. Handles retrieved with ``get`` or ``list`` never terminate compute on context exit. + supply at most one of ``image`` or ``template_id``; omitting both uses the + server's default image. supply ``cpu_flavor_id`` and ``vcpu_count`` together, + or omit both for server defaults. memory is derived by the server from the + CPU flavor and vCPU count. + A stable background event loop runs the async implementation. No thread or connection is started by the constructor. Prefer AsyncioSandbox in async applications so blocking calls do not pause the application's event loop. @@ -277,7 +282,6 @@ def __init__( name: Optional[str] = None, cpu_flavor_id: Optional[str] = None, vcpu_count: Optional[int] = None, - memory_in_gb: Optional[int] = None, disk_gb: Optional[int] = None, data_center_ids: Optional[Sequence[str]] = None, mounts: Optional[Mapping[str, Volume]] = None, @@ -310,7 +314,6 @@ def __init__( name=name, cpu_flavor_id=cpu_flavor_id, vcpu_count=vcpu_count, - memory_in_gb=memory_in_gb, disk_gb=disk_gb, data_center_ids=data_center_ids, mounts=mounts, diff --git a/tests/e2e/test_sandbox.py b/tests/e2e/test_sandbox.py index 1520bdb5b..844b33598 100644 --- a/tests/e2e/test_sandbox.py +++ b/tests/e2e/test_sandbox.py @@ -27,8 +27,6 @@ def test_sandbox_execution_and_borrowed_handles(require_api_key): with Sandbox( image="python:3.12-slim", cmd=["sleep", "infinity"], - vcpu_count=2, - memory_in_gb=4, idle_timeout_seconds=180, max_lifetime_seconds=300, **connection_options, diff --git a/tests/test_sandbox_contract.py b/tests/test_sandbox_contract.py new file mode 100644 index 000000000..34b26bd68 --- /dev/null +++ b/tests/test_sandbox_contract.py @@ -0,0 +1,114 @@ +"""Sandbox requests and snapshots use the current REST contract.""" + +import asyncio + +import pytest + +from runpod import AsyncioSandbox, Sandbox +from runpod.sandbox import SandboxInfo +from tests.test_sandbox import SandboxService, sandbox_peer + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_api", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "options, expected_body", + [ + ({}, {}), + ({"image": "python:3.12-slim"}, {"image": "python:3.12-slim"}), + ({"template_id": "template"}, {"templateId": "template"}), + ( + {"cpu_flavor_id": "cpu3c", "vcpu_count": 2}, + {"cpu": {"id": "cpu3c", "vcpuCount": 2}}, + ), + ( + {"image": "image", "cpu_flavor_id": "cpu3c", "vcpu_count": 2}, + {"image": "image", "cpu": {"id": "cpu3c", "vcpuCount": 2}}, + ), + ], +) +async def test_create_wire_contract(async_api, options, expected_body, monkeypatch): + with sandbox_peer() as peer: + create_record = peer.create_record + + def modern_record(body): + record = create_record(body) + record.pop("imageName") + record.pop("cpuFlavorId") + record["image"] = body.get("image", "default-image") + record["cpu"] = { + "id": "cpu3c", + "vcpuCount": 2, + "memory": 4, + } + return record + + monkeypatch.setattr(peer, "create_record", modern_record) + if async_api: + async with AsyncioSandbox(**options, **peer.options) as sandbox: + info = sandbox.info + else: + + def create_sync(): + with Sandbox(**options, **peer.options) as sandbox: + return sandbox.info + + info = await asyncio.to_thread(create_sync) + + assert peer.requests[0][:3] == ("POST", "/v2/sandboxes", expected_body) + assert info.image == options.get("image", "default-image") + assert info.cpu_flavor_id == "cpu3c" + assert peer.records[info.id]["state"] == "TERMINATED" + + +@pytest.mark.parametrize("api", [Sandbox, AsyncioSandbox]) +def test_image_and_template_remain_mutually_exclusive(api): + with pytest.raises(ValueError, match="at most one"): + api(image="image", template_id="template") + + +@pytest.mark.parametrize("api", [Sandbox, AsyncioSandbox]) +@pytest.mark.parametrize("options", [{"cpu_flavor_id": "cpu3c"}, {"vcpu_count": 2}]) +def test_partial_cpu_selection_is_rejected_before_any_request(api, options): + with sandbox_peer() as peer: + with pytest.raises(ValueError, match="cpu_flavor_id and vcpu_count together"): + api(**options, **peer.options) + assert peer.requests == [] + + +@pytest.mark.parametrize("api", [Sandbox, AsyncioSandbox]) +def test_explicit_memory_constraint_is_not_a_create_option(api): + with sandbox_peer() as peer: + with pytest.raises(TypeError, match="memory_in_gb"): + api(memory_in_gb=4, **peer.options) + assert peer.requests == [] + + +@pytest.mark.parametrize( + "fields, expected_image, expected_cpu", + [ + ({"image": "image", "cpu": {"id": "cpu3c"}}, "image", "cpu3c"), + ({"imageName": "legacy", "cpuFlavorId": "cpu5c"}, None, None), + ( + { + "image": "image", + "imageName": "legacy", + "cpu": {"id": "cpu3c"}, + "cpuFlavorId": "cpu5c", + }, + "image", + "cpu3c", + ), + ({"image": None, "imageName": "legacy"}, None, None), + ({"cpu": None, "cpuFlavorId": "cpu5c"}, None, None), + ({}, None, None), + ], +) +def test_snapshot_reads_only_current_fields(fields, expected_image, expected_cpu): + record = SandboxService().create_record({}) + record.pop("imageName") + record.pop("cpuFlavorId") + record.update(fields) + info = SandboxInfo.from_dict(record) + assert info.image == expected_image + assert info.cpu_flavor_id == expected_cpu