diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2888a2c..8ddd1d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,9 +148,9 @@ jobs: # 3.15, its `requires-python` and classifiers), and the default (3.13, the # newest in its security phase), which reports coverage with the entry # marked `coverage`; all gate, as in FOCA. FOCA: the lowest supported - # version (0.18.0, the first with the plugin API), the other releases - # up to the newest (0.19.0, 0.20.0, 0.21.0, 0.22.0) and FOCA's `dev` branch, so that changes of the plugin API show - # before FOCA releases them (raise the lowest version with the plugins' + # version (0.19.0, the first with the plugin store), the other releases + # up to the newest (0.20.0, 0.21.0, 0.22.0) and FOCA's `dev` branch, so + # that changes of the plugin API show before FOCA releases them (raise the lowest version with the plugins' # lower bounds, and the newest with their upper bounds) test: name: Test (${{ matrix.package.name }}, Python ${{ matrix.python-version }}, FOCA ${{ matrix.foca }}) @@ -163,14 +163,12 @@ jobs: matrix: package: ${{ fromJSON(needs.changes.outputs.packages) }} python-version: ["3.11", "3.13", "3.15"] - foca: ["0.18.0", "0.19.0", "0.20.0", "0.21.0", "0.22.0", dev] + foca: ["0.19.0", "0.20.0", "0.21.0", "0.22.0", dev] include: - - foca: "0.18.0" - foca-spec: "foca==0.18.0" - # The entry that reports coverage (with the default Python) - coverage: true - foca: "0.19.0" foca-spec: "foca==0.19.0" + # The entry that reports coverage (with the default Python) + coverage: true - foca: "0.20.0" foca-spec: "foca==0.20.0" - foca: "0.21.0" diff --git a/README.md b/README.md index 9e460f2..97474e0 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ repositories work the same way. **Status:** the plugin API is new in FOCA 0.18.0; the plugins support -FOCA 0.18 to 0.22. Plugins so far, both experimental and released as -0.1.x: `generic.routing.random` and -`tes.routing.distance`, plus the example `generic.example.noop`, the +FOCA 0.19 (the first release with the plugin store) to 0.22. Plugins so +far, both experimental and released as 0.1.x: `generic.routing.random` +and `tes.routing.distance`, plus the example `generic.example.noop`, the template of new plugins (never published). The latest release train is `foca-plugins` 0.3.0. diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 0a8ba1a..5097d3e 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -138,11 +138,11 @@ page](https://foca.readthedocs.io/en/latest/plugins/#testing-plugins). CI tests each plugin with the lowest and highest Python versions that FOCA supports and the default one (3.11, 3.13, 3.15), and with the lowest -supported FOCA version, 0.18.0, the first release with the plugin API (the -lower bound of every plugin, `foca>=0.18.0`), the later FOCA releases up -to the newest, 0.19.0, 0.20.0, 0.21.0 and 0.22.0 (the upper bounds are `<0.23`), and -FOCA's `dev` branch, from Git, -so that changes of the plugin API show before FOCA releases them. The +supported FOCA version, 0.19.0, the first release with the plugin store +(`ctx.store`; the lower bound of every plugin, `foca>=0.19.0`; 0.18.0, the +first release with the plugin API, is no longer supported), the later FOCA +releases up to the newest, 0.20.0, 0.21.0 and 0.22.0 (the upper bounds are +`<0.23`), and FOCA's `dev` branch, from Git, so that changes of the plugin API show before FOCA releases them. The development environment has the newest FOCA release (`uv.lock`). ## Changes and releases diff --git a/meta/pyproject.toml b/meta/pyproject.toml index 5b192ec..1d6840e 100644 --- a/meta/pyproject.toml +++ b/meta/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] # The FOCA versions that the train was tested with; set in the release pull # request -dependencies = ["foca>=0.18.0,<0.23"] +dependencies = ["foca>=0.19.0,<0.23"] # Extras per group (``) and subgroup (`-`), plus # `all` (stable plugins only) and `experimental` (experimental ones). Each diff --git a/plugins/generic/example/noop/README.md b/plugins/generic/example/noop/README.md index e8d7c2b..80d359a 100644 --- a/plugins/generic/example/noop/README.md +++ b/plugins/generic/example/noop/README.md @@ -17,7 +17,7 @@ and is **not published**. | Maturity | experimental | | Data transfers | none | | Data stored | none (no plugin store) | -| FOCA | 0.18, 0.19, 0.20, 0.21, 0.22 (`>=0.18.0,<0.23`) | +| FOCA | 0.19, 0.20, 0.21, 0.22 (`>=0.19.0,<0.23`) | ## What it does diff --git a/plugins/generic/example/noop/pyproject.toml b/plugins/generic/example/noop/pyproject.toml index e3cdead..02cdc0f 100644 --- a/plugins/generic/example/noop/pyproject.toml +++ b/plugins/generic/example/noop/pyproject.toml @@ -46,9 +46,9 @@ classifiers = [ # <<< template only ] dependencies = [ - # The plugin API of FOCA 0.18 to 0.22 (`foca.plugins`, API version 1); + # The plugin API of FOCA 0.19 to 0.22 (`foca.plugins`, API version 1); # raise the upper bound once CI has tested the next minor version - "foca>=0.18.0,<0.23", + "foca>=0.19.0,<0.23", "pydantic>=2.7,<3", ] diff --git a/plugins/generic/routing/random/CHANGELOG.md b/plugins/generic/routing/random/CHANGELOG.md index 7c17045..9a08cc2 100644 --- a/plugins/generic/routing/random/CHANGELOG.md +++ b/plugins/generic/routing/random/CHANGELOG.md @@ -7,6 +7,13 @@ Features, Security, Fixes, Documentation, Maintenance; empty ones left out), each entry with its issue and pull request numbers (see "Releasing" in the docs). +## Unreleased + +### Breaking changes + +- Drop FOCA 0.18 (`foca>=0.19.0,<0.23`), as all plugins of the repository + do: `tes.routing.distance` needs FOCA 0.19's plugin store (#21). + ## 0.1.3 Support for FOCA 0.21 and 0.22: the plugin now installs alongside FOCA diff --git a/plugins/generic/routing/random/README.md b/plugins/generic/routing/random/README.md index bf3350c..96d3676 100644 --- a/plugins/generic/routing/random/README.md +++ b/plugins/generic/routing/random/README.md @@ -16,7 +16,7 @@ proTES's random task distribution. | Maturity | experimental | | Data transfers | none | | Data stored | none (no plugin store) | -| FOCA | 0.18, 0.19, 0.20, 0.21, 0.22 (`>=0.18.0,<0.23`) | +| FOCA | 0.19, 0.20, 0.21, 0.22 (`>=0.19.0,<0.23`) | ## What it does diff --git a/plugins/generic/routing/random/pyproject.toml b/plugins/generic/routing/random/pyproject.toml index f73dc10..5583f1e 100644 --- a/plugins/generic/routing/random/pyproject.toml +++ b/plugins/generic/routing/random/pyproject.toml @@ -37,9 +37,9 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - # The plugin API of FOCA 0.18 to 0.22 (`foca.plugins`, API version 1); + # The plugin API of FOCA 0.19 to 0.22 (`foca.plugins`, API version 1); # raise the upper bound once CI has tested the next minor version - "foca>=0.18.0,<0.23", + "foca>=0.19.0,<0.23", "pydantic>=2.7,<3", ] diff --git a/plugins/generic/routing/random/src/foca_plugins/generic/routing/random/__init__.py b/plugins/generic/routing/random/src/foca_plugins/generic/routing/random/__init__.py index 9b0e98f..d2557ec 100644 --- a/plugins/generic/routing/random/src/foca_plugins/generic/routing/random/__init__.py +++ b/plugins/generic/routing/random/src/foca_plugins/generic/routing/random/__init__.py @@ -67,8 +67,7 @@ class Random(Plugin[RandomConfig, RandomParams]): " the order depends only on the seed and the order of the candidates" " (on the same Python version)." ) - # Nothing in the plugin store (listed by FOCA 0.19; declared here, as - # FOCA 0.18 has no such attribute) + # Nothing in the plugin store (listed in FOCA's plugin listing) data_stored = () # Set by `setup()`, from the admin settings diff --git a/plugins/tes/data/crypt4gh/README.md b/plugins/tes/data/crypt4gh/README.md index 4fbff84..d2549b4 100644 --- a/plugins/tes/data/crypt4gh/README.md +++ b/plugins/tes/data/crypt4gh/README.md @@ -21,7 +21,7 @@ middleware of proTES. | Maturity | experimental | | Data transfers | none | | Data stored | none (no plugin store) | -| FOCA | 0.18, 0.19, 0.20, 0.21, 0.22 (`>=0.18.0,<0.23`) | +| FOCA | 0.19, 0.20, 0.21, 0.22 (`>=0.19.0,<0.23`) | ## What it does diff --git a/plugins/tes/data/crypt4gh/pyproject.toml b/plugins/tes/data/crypt4gh/pyproject.toml index 0ff9650..647dc01 100644 --- a/plugins/tes/data/crypt4gh/pyproject.toml +++ b/plugins/tes/data/crypt4gh/pyproject.toml @@ -37,9 +37,9 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - # The plugin API of FOCA 0.18 to 0.22 (`foca.plugins`, API version 1); + # The plugin API of FOCA 0.19 to 0.22 (`foca.plugins`, API version 1); # raise the upper bound once CI has tested the next minor version - "foca>=0.18.0,<0.23", + "foca>=0.19.0,<0.23", "pydantic>=2.7,<3", ] diff --git a/plugins/tes/data/crypt4gh/src/foca_plugins/tes/data/crypt4gh/__init__.py b/plugins/tes/data/crypt4gh/src/foca_plugins/tes/data/crypt4gh/__init__.py index 7131317..5784963 100644 --- a/plugins/tes/data/crypt4gh/src/foca_plugins/tes/data/crypt4gh/__init__.py +++ b/plugins/tes/data/crypt4gh/src/foca_plugins/tes/data/crypt4gh/__init__.py @@ -155,8 +155,7 @@ class Crypt4gh(Plugin[Crypt4ghConfig, EmptyModel]): " Crypt4GH inputs as they are." ) data_transfers = () - # Nothing in the plugin store (listed by FOCA 0.19; declared here, as - # FOCA 0.18 has no such attribute) + # Nothing in the plugin store (listed in FOCA's plugin listing) data_stored = () settings: Crypt4ghConfig diff --git a/plugins/tes/routing/distance/CHANGELOG.md b/plugins/tes/routing/distance/CHANGELOG.md index b92211e..ed9bcc8 100644 --- a/plugins/tes/routing/distance/CHANGELOG.md +++ b/plugins/tes/routing/distance/CHANGELOG.md @@ -7,6 +7,30 @@ Features, Security, Fixes, Documentation, Maintenance; empty ones left out), each entry with its issue and pull request numbers (see "Releasing" in the docs). +## Unreleased + +### Breaking changes + +- Require FOCA 0.19 (`foca>=0.19.0,<0.23`), for its plugin store (#21). +- With `provider: dbip`, each caller now gets at most 100 uncached + lookups per day by default (`dbip_user_daily_lookups`; set `null` for + the earlier behaviour), and the plugin keeps locations and lookup counts + in FOCA's plugin store: with a database, in its collection + `plugin_store`, as personal data (hashed keys) for up to `cache_ttl` or a + day; set FOCA's `plugins.defaults.store.key_secret` and review + `max_ttl` before upgrading (#21). + +### Features + +- With `provider: dbip`, keep the locations of addresses (also misses and + failures) in FOCA's plugin store (`ctx.store`), shared by all processes + and replicas, instead of per process (#21). +- With `provider: dbip`, limit the lookups of each caller to + `dbip_user_daily_lookups` (default 100) per day, counted in the plugin + store by a hash of the caller's identity; `null` turns the limit off + (#21). +- Declare the plugin's entries in the plugin store (`data_stored`) (#21). + ## 0.1.3 Support for FOCA 0.21 and 0.22: the plugin now installs alongside FOCA diff --git a/plugins/tes/routing/distance/README.md b/plugins/tes/routing/distance/README.md index db33cc2..b217279 100644 --- a/plugins/tes/routing/distance/README.md +++ b/plugins/tes/routing/distance/README.md @@ -18,8 +18,8 @@ successor of proTES's distance-based task distribution. | Capabilities | `routing-by-data-locality` | | Maturity | experimental | | Data transfers | none by default; with `provider: dbip`, db-ip.com and Nominatim (see [Data transfers](#data-transfers)) | -| Data stored | none (no plugin store) | -| FOCA | 0.18, 0.19, 0.20, 0.21, 0.22 (`>=0.18.0,<0.23`) | +| Data stored | none by default; with `provider: dbip`, located addresses and daily lookup counts per caller, by hashes (see [Data stored](#data-stored)) | +| FOCA | 0.19, 0.20, 0.21, 0.22 (`>=0.19.0,<0.23`) | ## What it does @@ -57,19 +57,28 @@ Annotations, counts only (hosts, addresses and places of inputs could be personal data): `inputs` (input URLs), `input_hosts`, `located_inputs` (input URLs with a location), `located_backends`, `lookups` (by the provider), `cache_hits`, `failed_lookups` and `skipped_lookups` -(addresses not looked up for lack of time or beyond `dbip_max_lookups`). +(addresses not looked up for lack of time, beyond `dbip_max_lookups` or +beyond the caller's `dbip_user_daily_lookups`). The order is in the provenance of each request (`route.candidates_out`). ### Cache and limits -Each process caches the lookups of addresses, also misses (addresses -without location), so that backends are looked up once per process until -the entry expires, not per task: `cache_ttl` (1 day) for locations, +The plugin caches the lookups of addresses, also misses (addresses +without location), so that backends are looked up once until the entry +expires, not per task: `cache_ttl` (1 day) for locations, `cache_miss_ttl` (1 hour) for misses and `cache_failure_ttl` (1 minute) for failed lookups (e.g., a service is down; not lookups that only ran -out of time), at most `cache_size` addresses (the least recently used go -first). The cache doesn't use FOCA 0.19's plugin store yet (#21), so it -is not shared between processes. +out of time). + +- With the `offline` provider, each process caches at most `cache_size` + addresses (the least recently used go first). +- With `dbip`, the cache is in FOCA's plugin store (`ctx.store`), shared + by all processes and replicas if the gateway has a database (otherwise + per process, see FOCA's [plugin + store](https://foca.readthedocs.io/en/latest/configuration/plugins/#plugin-store)). + Its entries are limited by the TTLs above, capped by the operator's + `max_ttl` of the store, not by `cache_size` (0 still turns it off). If + the store fails or is full, the call goes on without the cache. Name resolution can't be interrupted, and hosts of inputs come from clients, so slow or hostile name servers must not tie up the gateway's @@ -118,12 +127,13 @@ know them: they are exact, and need no lookup. | `config.nominatim_interval` | seconds, 0 to 60 | `1.0` | time between requests to Nominatim, per process (its policy: at most 1 per second) | | `config.dbip_backoff` | seconds, 0 to 86400 | `600` | time to ask DB-IP nothing after it refused a lookup, e.g., over its daily quota (`dbip`) | | `config.dbip_max_lookups` | 1 to 1000 | `5` | uncached lookups per call, for inputs and for backends each (`dbip`), so that no single task uses up DB-IP's quota | +| `config.dbip_user_daily_lookups` | 1 to 1000000, or `null` | `100` | uncached lookups per caller and day, counted in FOCA's plugin store (`dbip`), so that no single user uses up DB-IP's quota; `null` for no limit | | `config.locate_backends` | boolean | `true` | whether backends without configured location are located by their address | | `config.max_hosts` | 1 to 1000 | `20` | input hosts to locate per task | | `config.resolve_timeout` | seconds, up to 60 | `2.0` | time to resolve the host names of a call, in total: half for inputs, the rest for backends (and never beyond the step's deadline) | | `config.resolve_threads` | 1 to 256 | `16` | host names resolved at a time, per process, for inputs and for backends each (a call: at most a quarter for its inputs) | | `config.resolve_cache_ttl` | seconds, ≥ 0 | `300` | time to keep the addresses of host names, also of names that couldn't be resolved | -| `config.cache_size` | ≥ 0 | `10000` | entries of each cache (addresses; host names of inputs and of backends; with `dbip`, places and coordinates); 0 turns the caches off | +| `config.cache_size` | ≥ 0 | `10000` | entries of each cache of a process (addresses with `offline`; host names of inputs and of backends; with `dbip`, places and coordinates); 0 turns the caches off, also that of addresses in the plugin store (`dbip`) | | `config.cache_ttl` | seconds, ≥ 0 | `86400` | time to keep located addresses | | `config.cache_miss_ttl` | seconds, ≥ 0 | `3600` | time to keep addresses without location | | `config.cache_failure_ttl` | seconds, ≥ 0 | `60` | time to keep addresses whose lookup failed as unlocated | @@ -188,11 +198,21 @@ public instance one per second. The plugin caches places and coordinates, looks up at most `dbip_max_lookups` uncached addresses per call (for inputs and for backends each), asks DB-IP only if Nominatim has a turn before the deadline, and backs off for `dbip_backoff` when DB-IP refuses -lookups. These limits can't keep users who submit many tasks with fresh -public addresses from using up DB-IP's daily quota; distance routing then -doesn't apply (the next alternative routes) until the quota resets. Use -`offline` where that matters, and your own Nominatim instance -(`nominatim_url`) if you can. +lookups. Each caller may make at most `dbip_user_daily_lookups` (100) +uncached lookups per day (a fixed window from the caller's first lookup), +counted in FOCA's plugin store by a hash of the caller's identity +(`subject_type` and `subject`; all anonymous callers share one count, and +a service client has one, whichever users it acts for; lookups that sent +no request, e.g., while DB-IP is backed off, don't count); if the store +fails, the call makes no lookups (the budget fails closed). Without a +database, the counts are per process, so the limit is only approximate. +The window is capped by the operator's `max_ttl` of the store: keep it at +a day or more, or the limit holds per `max_ttl` instead of per day. +These limits make it harder, but can't keep users from using up DB-IP's +daily quota (e.g., several users together); distance routing then doesn't +apply (the next alternative routes) until the quota resets. Use `offline` +where that matters, and your own Nominatim instance (`nominatim_url`) if +you can. DB-IP's page of the free API asks for no attribution; check DB-IP's terms of service for your use. Nominatim's data is © OpenStreetMap contributors @@ -213,6 +233,25 @@ of service for your use. Nominatim's data is © OpenStreetMap contributors e.g., in your gateway's privacy notice. Clients see these transfers in the plugin listing (`data_transfers`), whatever the provider. +## Data stored + +- **Default (`offline`):** nothing in FOCA's plugin store. +- **`provider: dbip`**, in FOCA's plugin store (`ctx.store`): + - the locations (latitude and longitude) of the public IP addresses of + the hosts of task inputs and of backends, also misses, keyed by a hash + of the address, for up to `cache_ttl`; + - daily lookup counts per caller, keyed by a hash of the caller's + identity, for a day (only with `dbip_user_daily_lookups`). + + Keys are hashes from `ctx.store.key_for()` (HMACs, if the operator set + FOCA's key secret, `plugins.defaults.store.key_secret`), never the + addresses or identifiers as they are. They pseudonymize, but don't + anonymize: the entries remain personal data, kept at most for the + operator's `max_ttl` of the store. Clients see them in the plugin + listing (`data_stored`). Set the key secret with `provider: dbip`: + without it, anyone who can read the database (or a backup) can recover + the addresses by hashing all IPv4 addresses. + ## Installation ```bash diff --git a/plugins/tes/routing/distance/pyproject.toml b/plugins/tes/routing/distance/pyproject.toml index 9f4932e..898724e 100644 --- a/plugins/tes/routing/distance/pyproject.toml +++ b/plugins/tes/routing/distance/pyproject.toml @@ -37,9 +37,9 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - # The plugin API of FOCA 0.18 to 0.22 (`foca.plugins`, API version 1); + # The plugin API of FOCA 0.19 to 0.22 (`foca.plugins`, API version 1); # raise the upper bound once CI has tested the next minor version - "foca>=0.18.0,<0.23", + "foca>=0.19.0,<0.23", "pydantic>=2.7,<3", # Reads the offline GeoIP database (MaxMind DB format, e.g., DB-IP Lite) "maxminddb>=2.6,<4", diff --git a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/__init__.py b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/__init__.py index 3c28c21..c07ce4e 100644 --- a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/__init__.py +++ b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/__init__.py @@ -17,9 +17,12 @@ Host names are resolved in parallel, within a time budget per call and with a bounded number of resolver threads per process. Resolutions and -lookups (also misses and, briefly, failures) are cached per process, with -TTLs. Annotations hold counts only: hosts, addresses and places of inputs -could be personal data. +lookups (also misses and, briefly, failures) are cached, with TTLs: per +process, except the lookups of the `dbip` provider, which are kept in +FOCA's plugin store (`ctx.store`), shared by all processes and replicas, +together with a daily budget of lookups per caller (cf. :py:mod:`.store`). +Annotations hold counts only: hosts, addresses and places of inputs could +be personal data. The plugin API is documented in FOCA's `docs/plugins.md` (https://foca.readthedocs.io/en/latest/plugins/). @@ -31,7 +34,14 @@ import time from typing import Literal -from foca.plugins import Backend, Context, Location, NotApplicable, Plugin +from foca.plugins import ( + Backend, + Context, + Location, + NotApplicable, + Plugin, + PluginError, +) from foca.plugins.base import EmptyModel from pydantic import ( BaseModel, @@ -42,7 +52,7 @@ model_validator, ) -from foca_plugins.tes.routing.distance.cache import TtlCache +from foca_plugins.tes.routing.distance.cache import LocationCache, TtlCache from foca_plugins.tes.routing.distance.geo import ( Resolver, distance_km, @@ -57,6 +67,7 @@ OfflineProvider, Provider, ) +from foca_plugins.tes.routing.distance.store import StoreCache, UserBudget class DistanceConfig(BaseModel): @@ -83,6 +94,10 @@ class DistanceConfig(BaseModel): per call (not counting cached ones), for the inputs and for the backends each; further addresses are not located in that call, so that no single task uses up DB-IP's quota. + dbip_user_daily_lookups: Maximum number of lookups by DB-IP and + Nominatim per caller and day (not counting cached ones), in all + processes that share the plugin store, so that no single user + uses up DB-IP's quota; `None` for no limit. locate_backends: Whether backends without configured location are located by their address; if not, they rank after the located ones. @@ -97,9 +112,11 @@ class DistanceConfig(BaseModel): resolved. resolve_cache_ttl: Seconds to keep the addresses of host names, also of names that couldn't be resolved. - cache_size: Maximum number of entries of each cache (addresses, - host names of inputs and of backends, and, with `dbip`, places - and coordinates); 0 turns the caches off. + cache_size: Maximum number of entries of each cache of the + process (addresses, host names of inputs and of backends, and, + with `dbip`, places and coordinates); 0 turns the caches off, + also that of addresses in the plugin store (`dbip`), which this + setting doesn't limit. cache_ttl: Seconds to keep located addresses. cache_miss_ttl: Seconds to keep addresses without location. cache_failure_ttl: Seconds to keep addresses whose lookup failed as @@ -121,6 +138,7 @@ class DistanceConfig(BaseModel): nominatim_interval: float = Field(default=1.0, ge=0, le=60) dbip_backoff: float = Field(default=600, ge=0, le=86400) dbip_max_lookups: int = Field(default=5, ge=1, le=1000) + dbip_user_daily_lookups: int | None = Field(default=100, ge=1, le=1_000_000) locate_backends: bool = True max_hosts: int = Field(default=20, ge=1, le=1000) resolve_timeout: float = Field(default=2.0, gt=0, le=60) @@ -175,6 +193,7 @@ class _Locator: ctx: Context of the call. max_lookups: Maximum number of lookups by the provider per batch of hosts (inputs, backends); `None` for no limit. + budget: Budget of lookups of the caller; `None` for no limit. hosts: Locations of the hosts located in this call. lookups: Number of lookups by the provider. cache_hits: Number of addresses found in the cache. @@ -186,10 +205,11 @@ class _Locator: """ provider: Provider - cache: TtlCache[str, Location | None] + cache: LocationCache settings: DistanceConfig ctx: Context max_lookups: int | None + budget: UserBudget | None = None hosts: dict[str, Location | None] = field(default_factory=dict) lookups: int = 0 cache_hits: int = 0 @@ -232,8 +252,10 @@ def _lookup(self, address: str) -> Location | None: if cached: self.cache_hits += 1 return location - if self.ctx.remaining() <= 0 or ( - self.max_lookups is not None and self.batch_lookups >= self.max_lookups + if ( + self.ctx.remaining() <= 0 + or (self.max_lookups is not None and self.batch_lookups >= self.max_lookups) + or (self.budget is not None and not self.budget.take()) ): self.skipped_lookups += 1 return None @@ -244,7 +266,12 @@ def _lookup(self, address: str) -> Location | None: except LookupFailed as exc: self.failed_lookups += 1 self.ctx.log.warning("Lookup failed: %s", exc) - if not exc.transient: + if not exc.sent and self.budget is not None: + # no request was sent (e.g., backed off): it doesn't count + self.budget.give_back() + if exc.sent and not exc.transient: + # only failures of services: not, e.g., this process's + # back-off, as the cache may be shared with other processes; # keeps a location that a concurrent call found meanwhile self.cache.add(address, None, self.settings.cache_failure_ttl) return None @@ -261,7 +288,8 @@ class Distance(Plugin[DistanceConfig, EmptyModel]): provider: Provider of locations (cf. `DistanceConfig.provider`); kept after `close()`, when its lookups fail. cache: Cache of looked-up locations, by address, shared by all - calls. + calls of the process; used by the `offline` provider only + (`dbip` keeps them in the plugin store). input_resolver: Resolver of the host names of inputs, shared by all calls; a call starts at most a quarter of its threads. backend_resolver: Resolver of the host names of backends, with @@ -288,9 +316,13 @@ class Distance(Plugin[DistanceConfig, EmptyModel]): "OpenStreetMap Nominatim (or the instance the admin configures): city," " region and country of those addresses; only with provider 'dbip'", ) - # Nothing in the plugin store: lookups are cached per process (listed by - # FOCA 0.19; declared here, as FOCA 0.18 has no such attribute) - data_stored = () + data_stored = ( + "Locations of public IP addresses: of the hosts of task inputs and" + " backends, by a hash of the address (an HMAC if the operator set a key" + " secret), for up to 'cache_ttl'; only with provider 'dbip'", + "Daily lookup counts per caller: by a hash of the caller's identity," + " for a day; only with provider 'dbip' and 'dbip_user_daily_lookups'", + ) provider: Provider cache: TtlCache[str, Location | None] @@ -360,19 +392,33 @@ def on_route(self, candidates: Sequence[Backend], ctx: Context) -> list[Backend] Raises: NotApplicable: The task has no input URLs, no input could be located, or no candidate could be located. + PluginError: The provider is `dbip`, and the context has no + plugin store. """ urls = input_urls(ctx.original) if not urls: raise NotApplicable("The task has no input URLs") settings = self.settings + cache: LocationCache = self.cache + user_budget: UserBudget | None = None + max_lookups: int | None = None + if settings.provider == "dbip": + store = ctx.store + if store is None: + raise PluginError("Provider 'dbip' needs ctx.store") + cache = StoreCache(store, ctx.log, enabled=settings.cache_size > 0) + if settings.dbip_user_daily_lookups is not None: + user_budget = UserBudget( + store, ctx.principal, settings.dbip_user_daily_lookups, ctx.log + ) + max_lookups = settings.dbip_max_lookups locator = _Locator( provider=self.provider, - cache=self.cache, + cache=cache, settings=settings, ctx=ctx, - max_lookups=( - settings.dbip_max_lookups if settings.provider == "dbip" else None - ), + max_lookups=max_lookups, + budget=user_budget, ) # inputs per host: distances are computed once per host, weighted weights = Counter(host for url in urls if (host := url_host(url))) diff --git a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/cache.py b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/cache.py index d57f508..ed3d262 100644 --- a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/cache.py +++ b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/cache.py @@ -1,15 +1,62 @@ -"""Cache with TTLs and a size limit, for lookups of the plugin.""" +"""Caches of the plugin's lookups. + +- :py:class:`TtlCache`: in the memory of the process, with TTLs and a size + limit (for host names, places and, with the offline provider, the + locations of addresses). +- :py:class:`LocationCache`: what the plugin needs of a cache of + locations; :py:class:`TtlCache` and + :py:class:`~foca_plugins.tes.routing.distance.store.StoreCache` (in the + plugin store, for the online provider) have it. +""" from collections import OrderedDict from collections.abc import Callable, Hashable import threading import time -from typing import Generic, TypeVar +from typing import Generic, Protocol, TypeVar + +from foca.plugins import Location K = TypeVar("K", bound=Hashable) V = TypeVar("V") +class LocationCache(Protocol): + """Cache of the locations of addresses; `None` for misses.""" + + def get(self, key: str) -> tuple[bool, Location | None]: + """Get the cached location of an address. + + Args: + key: Address. + + Returns: + Whether the address is cached, and its location (`None` if not + cached or a miss). + """ + ... # pragma: no cover + + def put(self, key: str, value: Location | None, ttl: float) -> None: + """Cache the location of an address. + + Args: + key: Address. + value: Location; `None` for a miss. + ttl: Seconds to keep it; 0 keeps it not at all. + """ + ... # pragma: no cover + + def add(self, key: str, value: Location | None, ttl: float) -> None: + """Cache the location of an address unless it is cached already. + + Args: + key: Address. + value: Location; `None` for a miss. + ttl: Seconds to keep it; 0 keeps it not at all. + """ + ... # pragma: no cover + + class TtlCache(Generic[K, V]): """Cache whose entries expire, each after a TTL of its own. diff --git a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/providers.py b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/providers.py index 19ec839..a7007bc 100644 --- a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/providers.py +++ b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/providers.py @@ -41,10 +41,17 @@ class LookupFailed(Exception): transient: Whether the lookup failed only for lack of time in this call (e.g., a rate limit), so that it may succeed in the next call; such failures aren't cached. + sent: Whether the lookup may have sent a request to a service + (e.g., DB-IP), so that it counts towards the caller's budget. """ def __init__( - self, message: str, status: int | None = None, *, transient: bool = False + self, + message: str, + status: int | None = None, + *, + transient: bool = False, + sent: bool = True, ) -> None: """Create the error. @@ -52,10 +59,13 @@ def __init__( message: Message, without addresses. status: HTTP status of the response, if any. transient: Whether the lookup failed only for lack of time. + sent: Whether the lookup may have sent a request; `False` only + if it certainly didn't. """ super().__init__(message) self.status = status self.transient = transient + self.sent = sent class Provider(Protocol): @@ -258,17 +268,19 @@ def locate(self, address: str, ctx: Context) -> Location | None: PluginError: The context has no HTTP client. """ if self._closed: - raise LookupFailed("The provider is closed") + raise LookupFailed("The provider is closed", sent=False) http = ctx.http if http is None: raise PluginError("Provider 'dbip' needs ctx.http") turn: float | None = None + requested = False # whether a request may have been sent try: cached, place = self._places.get(address) if not cached: if self._clock() < self._blocked_until: raise LookupFailed("DB-IP refused lookups recently; backing off") turn = self._book(ctx) + requested = True place = self._place(http, address) self._places.put(address, place, self._ttl(place)) if place is None: @@ -282,11 +294,16 @@ def locate(self, address: str, ctx: Context) -> Location | None: if not cached: self._wait(self._book(ctx) if turn is None else turn) turn = None + requested = True location = self._search(http, query) self._searches.put(query, location, self._ttl(location)) if location is not None: return location return None + except LookupFailed as exc: + if not requested: + exc.sent = False + raise finally: if turn is not None: self._give_back(turn) diff --git a/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/store.py b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/store.py new file mode 100644 index 0000000..5a97794 --- /dev/null +++ b/plugins/tes/routing/distance/src/foca_plugins/tes/routing/distance/store.py @@ -0,0 +1,238 @@ +"""Entries of the plugin in FOCA's plugin store (`ctx.store`). + +With the online provider (`dbip`), the plugin keeps in the store: + +- the locations of addresses, also misses (:py:class:`StoreCache`), so + that all processes and replicas that share the store look up each + address once until its entry expires; +- a daily count of the lookups of each caller (:py:class:`UserBudget`), + so that no single user can use up DB-IP's daily quota. + +Keys are derived with `ctx.store.key_for()`: hashes (HMACs, if the +operator set FOCA's key secret) of the address or of the caller's +identity, never the address or identifier as they are. Store failures +never fail a call: the cache then counts addresses as not cached, the +budget allows no lookup (it fails closed), and neither asks the store +again in that call. +""" + +import logging +from typing import Any + +from foca.plugins import Location, Principal, Store, StoreError + +LOCATION_KEY = "location" +"""First part of the keys of locations (cf. `Store.key_for()`).""" + +BUDGET_KEY = "dbip-lookups" +"""First part of the keys of the callers' lookup counts.""" + +BUDGET_WINDOW = 86400 +"""Seconds of the window of the callers' lookup counts (a day).""" + + +def _location(value: Any) -> tuple[bool, Location | None]: + """Read a stored location. + + Args: + value: Stored value: `[latitude, longitude]`, or `[]` for a miss. + + Returns: + Whether the value is a valid location or miss, and the location + (`None` for a miss or an invalid value). + """ + if value == []: + return True, None + if not (isinstance(value, list) and len(value) == 2): + return False, None + latitude, longitude = value + for number in value: + if isinstance(number, bool) or not isinstance(number, int | float): + return False, None + # also rejects NaN, infinities and integers too large for floats (which + # `math.isfinite()` can't convert) + if not (-90 <= latitude <= 90 and -180 <= longitude <= 180): + return False, None + return True, Location(latitude=latitude, longitude=longitude) + + +class StoreCache: + """Cache of the locations of addresses in the plugin store, for one call. + + Has the methods of :py:class:`.cache.LocationCache`. Entries are kept + for their TTL, capped by the operator's `max_ttl` of the store; the + cache has no size limit of its own, but the store may be full (FOCA + 0.23's `max_namespace_size`), which counts as a failure. Entries that + are neither locations nor misses count as not cached. After the first + failure of the store, the cache counts all addresses as not cached and + keeps none, for the rest of the call. + + Attributes: + store: Plugin store of the call. + log: Logger of the call. + enabled: Whether the cache keeps entries. + failed: Whether the store failed in this call. + """ + + def __init__( + self, + store: Store, + log: "logging.LoggerAdapter[logging.Logger]", + *, + enabled: bool = True, + ) -> None: + """Create the cache of a call. + + Args: + store: Plugin store of the call (`ctx.store`). + log: Logger of the call (`ctx.log`). + enabled: Whether the cache keeps entries; if not, it counts all + addresses as not cached. + """ + self.store = store + self.log = log + self.enabled = enabled + self.failed = False + + def _failure(self, exc: StoreError) -> None: + """Note a failure of the store; logs it, without the address. + + Args: + exc: Error of the store. + """ + self.failed = True + self.log.warning("Plugin store unavailable for locations: %s", exc) + + def get(self, key: str) -> tuple[bool, Location | None]: + """Get the cached location of an address. + + Args: + key: Address. + + Returns: + Whether the address is cached, and its location (`None` if not + cached or a miss). + """ + if not self.enabled or self.failed: + return False, None + try: + value = self.store.get(self.store.key_for(LOCATION_KEY, key)) + except StoreError as exc: + self._failure(exc) + return False, None + return _location(value) + + def put(self, key: str, value: Location | None, ttl: float) -> None: + """Cache the location of an address. + + Args: + key: Address. + value: Location; `None` for a miss. + ttl: Seconds to keep it; 0 keeps it not at all. + """ + if not self.enabled or self.failed or ttl <= 0: + return + stored = [] if value is None else [value.latitude, value.longitude] + try: + self.store.set(self.store.key_for(LOCATION_KEY, key), stored, ttl=ttl) + except StoreError as exc: + self._failure(exc) + + def add(self, key: str, value: Location | None, ttl: float) -> None: + """Cache the location of an address unless it is cached already. + + Not atomic: a location that another process stores between the + check and the write is replaced. + + Args: + key: Address. + value: Location; `None` for a miss. + ttl: Seconds to keep it; 0 keeps it not at all. + """ + if not self.enabled or ttl <= 0: + return + cached, _ = self.get(key) + if not cached: + self.put(key, value, ttl) + + +class UserBudget: + """Daily budget of lookups of the caller of one call, in the plugin store. + + A counter per caller (`subject_type` and `subject` of the principal; + all anonymous callers share one), which starts with the caller's first + lookup and expires a day later (a fixed window; capped by the + operator's `max_ttl` of the store). A service client has one budget, + whichever users it acts for. If the store fails, the budget allows no + lookup, and asks the store no more, for the rest of the call. + + Attributes: + store: Plugin store of the call. + key: Key of the caller's counter. + limit: Maximum number of lookups per window. + log: Logger of the call. + exhausted: Whether the caller's budget is used up. + failed: Whether the store failed in this call. + """ + + def __init__( + self, + store: Store, + principal: Principal, + limit: int, + log: "logging.LoggerAdapter[logging.Logger]", + ) -> None: + """Create the budget of a call. + + Args: + store: Plugin store of the call (`ctx.store`). + principal: Caller (`ctx.principal`). + limit: Maximum number of lookups per caller and window. + log: Logger of the call (`ctx.log`). + """ + self.store = store + self.key = store.key_for( + BUDGET_KEY, principal.subject_type or "", principal.subject or "" + ) + self.limit = limit + self.log = log + self.exhausted = False + self.failed = False + + def take(self) -> bool: + """Count a lookup, if the budget allows it. + + Returns: + Whether the lookup may run. + """ + if self.exhausted or self.failed: + return False + try: + count = self.store.increment(self.key, ttl=BUDGET_WINDOW, limit=self.limit) + except StoreError as exc: + self.failed = True + self.log.warning("Plugin store unavailable for the budget: %s", exc) + return False + if count is None: + self.exhausted = True + self.log.info("The caller's daily budget of lookups is used up") + return False + return True + + def give_back(self) -> None: + """Uncount a lookup that sent no request; failures leave it counted. + + Counters that expired (or were never positive) are left alone, so + that no new window starts below 0 (not atomic: a counter that + expires between the check and the decrement still does, rarely). + """ + if self.failed: + return + try: + count = self.store.get(self.key) + if not isinstance(count, int) or count <= 0: + return + self.store.increment(self.key, -1, ttl=BUDGET_WINDOW) + except StoreError as exc: + self.failed = True + self.log.warning("Plugin store unavailable for the budget: %s", exc) diff --git a/plugins/tes/routing/distance/tests/test_distance.py b/plugins/tes/routing/distance/tests/test_distance.py index 6863a6b..b15bf75 100644 --- a/plugins/tes/routing/distance/tests/test_distance.py +++ b/plugins/tes/routing/distance/tests/test_distance.py @@ -17,16 +17,26 @@ from typing import Any from unittest.mock import MagicMock -from foca.plugins import Backend, Context, Location, NotApplicable, Principal +from foca.plugins import ( + Backend, + Context, + Location, + NotApplicable, + PluginError, + Principal, + StoreError, +) from foca.plugins.base import EmptyModel, check_plugin_class from foca.plugins.config import PluginsConfig from foca.plugins.engine import PluginHost +from foca.plugins.store import MemoryStoreBackend from foca.plugins.testing import ( FakeHttp, PluginHarness, check_plugin_contract, make_backend, make_inbound, + make_principal, ) from pydantic import ValidationError import pytest @@ -159,8 +169,12 @@ def test_plugin_class_is_valid() -> None: "db-ip.com (api.db-ip.com)", "OpenStreetMap Nominatim (or the instance the admin configures)", ] - # The lookups are cached per process, not in the plugin store - assert Distance.data_stored == () + # With provider 'dbip' only: locations and counts in the plugin store + stored = [entry.split(":")[0] for entry in Distance.data_stored] + assert stored == [ + "Locations of public IP addresses", + "Daily lookup counts per caller", + ] def test_plugin_meets_contract( @@ -198,7 +212,10 @@ def test_config_defaults(database: Path) -> None: assert config.resolve_timeout == 2 assert config.resolve_threads == 16 assert config.dbip_max_lookups == 5 + assert config.dbip_user_daily_lookups == 100 assert config.cache_failure_ttl == 60 + unlimited = DistanceConfig(provider="dbip", dbip_user_daily_lookups=None) + assert unlimited.dbip_user_daily_lookups is None @pytest.mark.parametrize( @@ -215,6 +232,7 @@ def test_config_defaults(database: Path) -> None: {"provider": "dbip", "user_agent": "gw/1\u00e9"}, {"provider": "dbip", "resolve_threads": 0}, {"provider": "dbip", "dbip_max_lookups": 0}, + {"provider": "dbip", "dbip_user_daily_lookups": 0}, {"provider": "dbip", "max_hosts": 0}, {"provider": "dbip", "cache_ttl": -1}, {"provider": "dbip", "other": "value"}, @@ -890,6 +908,285 @@ def test_online_lookups_are_capped_per_call( assert len(http.calls) == 6 +def test_online_lookups_are_shared_by_processes( + world: list[Backend], addresses: dict[str, str], places: dict[str, Location] +) -> None: + """With `dbip`, locations are kept in the plugin store, for all processes. + + Args: + world: Backends. + addresses: Addresses in the networks of the test database. + places: Places of the networks. + """ + backend = MemoryStoreBackend() + config = {"provider": "dbip", "nominatim_interval": 0} + http = dbip_http(addresses, places) + first, second = ( + PluginHarness( + Distance, + config=config, + plugin_id=PLUGIN_ID, + http=http, + store_backend=backend, + ) + for _ in range(2) + ) + task = make_task(addresses["zurich"]) + assert ids(first.run_route(world, make_inbound(task))) == [ + "helsinki", + "mountain_view", + "sydney", + ] + assert len(http.calls) == 8 + # another process (its own plugin instance), the same store: no lookups + assert ids(second.run_route(world, make_inbound(task)))[0] == "helsinki" + assert len(http.calls) == 8 + (execution,) = second.executions + assert (execution.annotations["lookups"], execution.annotations["cache_hits"]) == ( + 0, + 4, + ) + # keys are hashes of the addresses + store = second.store + zurich = places["zurich"] + assert store.get(addresses["zurich"]) is None + assert store.get(store.key_for("location", addresses["zurich"])) == [ + zurich.latitude, + zurich.longitude, + ] + + +def test_offline_lookups_stay_in_the_process( + database: Path, world: list[Backend], addresses: dict[str, str] +) -> None: + """With `offline`, nothing is kept in the plugin store. + + Args: + database: Test database. + world: Backends. + addresses: Addresses in the networks of the test database. + """ + harness = PluginHarness( + Distance, config={"database": str(database)}, plugin_id=PLUGIN_ID + ) + harness.run_route(world, make_inbound(make_task(addresses["zurich"]))) + store = harness.store + for address in addresses.values(): + assert store.get(store.key_for("location", address)) is None + assert store.get(store.key_for("dbip-lookups", "user", "user")) is None + + +def test_online_lookups_have_a_daily_budget_per_caller( + world: list[Backend], addresses: dict[str, str], places: dict[str, Location] +) -> None: + """With `dbip`, each caller gets `dbip_user_daily_lookups` lookups a day. + + Args: + world: Backends. + addresses: Addresses in the networks of the test database. + places: Places of the networks. + """ + http = dbip_http(addresses, places) + harness = PluginHarness( + Distance, + config={ + "provider": "dbip", + "nominatim_interval": 0, + "dbip_user_daily_lookups": 2, + }, + plugin_id=PLUGIN_ID, + http=http, + ) + inbound = make_inbound(make_task(addresses["zurich"])) + alice, bob = make_principal("alice"), make_principal("bob") + # the input and the first backend; the other two backends are skipped + assert ids(harness.run_route(world, inbound, alice)) == [ + "sydney", + "mountain_view", + "helsinki", + ] + (execution,) = harness.executions + assert execution.annotations["lookups"] == 2 + assert execution.annotations["skipped_lookups"] == 2 + assert len(http.calls) == 4 + # alice's budget is used up: no lookups, the cached ones still count + harness.run_route(world, inbound, alice) + (execution,) = harness.executions + assert (execution.annotations["lookups"], execution.annotations["cache_hits"]) == ( + 0, + 2, + ) + assert execution.annotations["skipped_lookups"] == 2 + # bob has a budget of his own + assert ids(harness.run_route(world, inbound, bob)) == [ + "helsinki", + "mountain_view", + "sydney", + ] + (execution,) = harness.executions + assert execution.annotations["lookups"] == 2 + assert len(http.calls) == 8 + # cached locations need no budget + assert ids(harness.run_route(world, inbound, alice))[0] == "helsinki" + (execution,) = harness.executions + assert (execution.annotations["lookups"], execution.annotations["cache_hits"]) == ( + 0, + 4, + ) + store = harness.store + assert store.get(store.key_for("dbip-lookups", "user", "alice")) == 2 + + +def test_online_budget_can_be_turned_off( + world: list[Backend], addresses: dict[str, str], places: dict[str, Location] +) -> None: + """With `dbip_user_daily_lookups: null`, callers have no budget. + + Args: + world: Backends. + addresses: Addresses in the networks of the test database. + places: Places of the networks. + """ + harness = PluginHarness( + Distance, + config={ + "provider": "dbip", + "nominatim_interval": 0, + "dbip_user_daily_lookups": None, + }, + plugin_id=PLUGIN_ID, + http=dbip_http(addresses, places), + ) + harness.run_route(world, make_inbound(make_task(addresses["zurich"]))) + (execution,) = harness.executions + assert execution.annotations["lookups"] == 4 + store = harness.store + assert store.get(store.key_for("dbip-lookups", "user", "user")) is None + + +@pytest.mark.parametrize( + ("transient", "sent", "count"), + [(True, False, 0), (False, False, 0), (True, True, 1), (False, True, 1)], +) +def test_online_budget_is_not_spent_on_lookups_without_requests( + world: list[Backend], + addresses: dict[str, str], + places: dict[str, Location], + monkeypatch: pytest.MonkeyPatch, + transient: bool, + sent: bool, + count: int, +) -> None: + """Failed lookups that sent no request are given back to the budget. + + E.g., for lack of time, or while DB-IP is backed off; failed lookups + that may have sent requests count. + + Args: + world: Backends. + addresses: Addresses in the networks of the test database. + places: Places of the networks. + monkeypatch: Pytest fixture. + transient: Whether the failure is transient. + sent: Whether the failed lookup may have sent a request. + count: Expected count of the caller's lookups. + """ + harness = PluginHarness( + Distance, + config={"provider": "dbip", "dbip_user_daily_lookups": 2}, + plugin_id=PLUGIN_ID, + http=dbip_http(addresses, places), + on_failure="skip", + ) + + def busy(address: str, ctx: Context) -> Location | None: + """Have no time. + + Args: + address: Address. + ctx: Context. + + Raises: + LookupFailed: Always. + """ + raise LookupFailed("no time", transient=transient, sent=sent) + + plugin = harness.plugin.instance + assert isinstance(plugin, Distance) + monkeypatch.setattr(plugin.provider, "locate", busy) + harness.run_route(world, make_inbound(make_task(addresses["zurich"]))) + execution = harness.executions[0] + assert execution.outcome == "not_applicable" + assert execution.annotations["failed_lookups"] == 1 + store = harness.store + assert store.get(store.key_for("dbip-lookups", "user", "user")) == count + # failures of services are cached (as misses) unless transient; others + # (e.g., back-off of the process) not, as other processes share the cache + cached = store.get(store.key_for("location", addresses["zurich"])) + assert cached == ([] if sent and not transient else None) + + +@pytest.mark.parametrize(("budget", "lookups", "calls"), [(5, 0, 0), (None, 4, 8)]) +def test_online_lookups_without_store( + world: list[Backend], + addresses: dict[str, str], + places: dict[str, Location], + budget: int | None, + lookups: int, + calls: int, +) -> None: + """If the store fails, nothing is cached, and the budget allows nothing. + + Args: + world: Backends. + addresses: Addresses in the networks of the test database. + places: Places of the networks. + budget: `dbip_user_daily_lookups`. + lookups: Expected number of lookups. + calls: Expected number of requests. + """ + error = StoreError("Plugin store failed: ServerSelectionTimeoutError") + backend = MagicMock( + shared=True, + **{ + f"{name}.side_effect": error + for name in ("get", "set", "delete", "increment") + }, + ) + http = dbip_http(addresses, places) + harness = PluginHarness( + Distance, + config={ + "provider": "dbip", + "nominatim_interval": 0, + "dbip_user_daily_lookups": budget, + }, + plugin_id=PLUGIN_ID, + http=http, + on_failure="skip", + store_backend=backend, + ) + harness.run_route(world, make_inbound(make_task(addresses["zurich"]))) + execution = harness.executions[0] + assert execution.annotations["lookups"] == lookups + assert len(http.calls) == calls + assert backend.get.call_count == 1 + + +def test_online_provider_needs_store(world: list[Backend]) -> None: + """With `dbip`, contexts without `ctx.store` are refused. + + Args: + world: Backends. + """ + plugin = Distance(PLUGIN_ID, DistanceConfig(provider="dbip")) + try: + with pytest.raises(PluginError, match=r"ctx\.store"): + plugin.on_route(world, make_context(make_task("9.9.9.9"))) + finally: + plugin.close() + + def dbip_http(addresses: dict[str, str], places: dict[str, Location]) -> FakeHttp: """Build answers of DB-IP and Nominatim for the test networks. diff --git a/plugins/tes/routing/distance/tests/test_providers.py b/plugins/tes/routing/distance/tests/test_providers.py index 7b0eb37..0388a01 100644 --- a/plugins/tes/routing/distance/tests/test_providers.py +++ b/plugins/tes/routing/distance/tests/test_providers.py @@ -1,5 +1,6 @@ -"""Tests of the location providers and the cache of `tes.routing.distance`.""" +"""Tests of the providers and caches of `tes.routing.distance`.""" +import logging from pathlib import Path import threading import time @@ -26,6 +27,7 @@ AGENT = "test-agent/1.0" ADDRESS = "9.9.9.9" PLACE = {"city": "Zurich", "stateProv": "Zurich", "countryCode": "CH"} +LOG = logging.LoggerAdapter(logging.getLogger("foca.plugins.tes.routing.distance")) class Clock: @@ -408,6 +410,7 @@ def sleep(seconds: float) -> None: with pytest.raises(LookupFailed, match="rate limit") as failed: provider.locate("9.9.9.2", make_context(http, seconds=100)) assert failed.value.transient + assert not failed.value.sent finally: go_on.set() second.join(5) @@ -482,11 +485,13 @@ def test_dbip_provider_backs_off(status: int, body: Any) -> None: provider = make_dbip(clock) http = FakeHttp() http.add("GET", f"{DBIP}/{ADDRESS}", status=status, json=body) - with pytest.raises(LookupFailed, match=r"refused|429"): + with pytest.raises(LookupFailed, match=r"refused|429") as refused: provider.locate(ADDRESS, make_context(http)) + assert refused.value.sent clock.now += 599 - with pytest.raises(LookupFailed, match="backing off"): + with pytest.raises(LookupFailed, match="backing off") as backing_off: provider.locate(ADDRESS, make_context(http)) + assert not backing_off.value.sent assert len(http.calls) == 1 clock.now += 2 with pytest.raises(LookupFailed): @@ -594,11 +599,27 @@ def test_dbip_provider_fails_after_close() -> None: provider = make_dbip() provider.close() http = FakeHttp() - with pytest.raises(LookupFailed, match="closed"): + with pytest.raises(LookupFailed, match="closed") as closed: provider.locate(ADDRESS, make_context(http)) + assert not closed.value.sent assert http.calls == [] +def test_dbip_provider_failures_after_requests_were_sent() -> None: + """Failures after requests were sent say so, also if transient.""" + clock = Clock() + provider = DbipProvider(DBIP, NOMINATIM, AGENT, 60, cache_size=0, clock=clock) + http = FakeHttp() + http.add("GET", f"{DBIP}/{ADDRESS}", json=PLACE) + http.add("GET", search_url("Zurich, Zurich CH"), json=[]) + # the second query's turn (in 60 s) comes after the deadline + with pytest.raises(LookupFailed, match="rate limit") as failed: + provider.locate(ADDRESS, make_context(http, seconds=10)) + assert failed.value.transient + assert failed.value.sent + assert len(http.calls) == 2 + + def test_dbip_provider_keeps_misses_for_miss_ttl() -> None: """Addresses without place are asked for again after `miss_ttl`.""" clock = Clock() diff --git a/plugins/tes/routing/distance/tests/test_store.py b/plugins/tes/routing/distance/tests/test_store.py new file mode 100644 index 0000000..4622364 --- /dev/null +++ b/plugins/tes/routing/distance/tests/test_store.py @@ -0,0 +1,249 @@ +"""Tests of the plugin store entries of `tes.routing.distance` (`store.py`).""" + +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +import logging +from typing import Any +from unittest.mock import MagicMock + +from foca.plugins import Location, Principal, Store, StoreError +from foca.plugins.store import MemoryStoreBackend, StoreSettings +import pytest + +from foca_plugins.tes.routing.distance.store import StoreCache, UserBudget, _location + +LOG = logging.LoggerAdapter(logging.getLogger("foca.plugins.tes.routing.distance")) + + +def make_store( + backend: Any = None, clock: Callable[[], datetime] | None = None +) -> Store: + """Create a plugin store, as `ctx.store`. + + Args: + backend: Backend; one in memory if not given. + clock: Clock of the store; the real one if not given. + + Returns: + Store, without deadline. + """ + settings = StoreSettings( + namespace="foca-plugin-tes-routing-distance", + mode="local", + backend=MemoryStoreBackend() if backend is None else backend, + max_entry_size=4096, + max_ttl=2_592_000, + ) + return Store(settings) if clock is None else Store(settings, clock=clock) + + +def failing_backend() -> MagicMock: + """Create a store backend whose calls fail. + + Returns: + Backend. + """ + error = StoreError("Plugin store failed: ServerSelectionTimeoutError") + return MagicMock( + **{ + f"{name}.side_effect": error + for name in ("get", "set", "delete", "increment") + } + ) + + +def test_store_cache_hits_and_expires() -> None: + """Locations and misses are kept in the store, for their TTL.""" + now = [datetime(2026, 11, 13, tzinfo=UTC)] + store = make_store(clock=lambda: now[0]) + cache = StoreCache(store, LOG) + here = Location(latitude=1, longitude=2) + assert cache.get("9.9.9.9") == (False, None) + cache.put("9.9.9.9", here, 100) + cache.put("4.4.4.4", None, 10) + assert cache.get("9.9.9.9") == (True, here) + assert cache.get("4.4.4.4") == (True, None) + # another call (or process) with the same store finds them + assert StoreCache(store, LOG).get("9.9.9.9") == (True, here) + now[0] += timedelta(seconds=11) + assert cache.get("4.4.4.4") == (False, None) + assert cache.get("9.9.9.9") == (True, here) + now[0] += timedelta(seconds=90) + assert cache.get("9.9.9.9") == (False, None) + + +def test_store_cache_keys_are_hashes() -> None: + """Addresses aren't kept as keys, only hashes derived by the store.""" + store = make_store() + StoreCache(store, LOG).put("9.9.9.9", Location(latitude=1, longitude=2), 100) + assert store.get("9.9.9.9") is None + assert store.get(store.key_for("location", "9.9.9.9")) == [1, 2] + + +@pytest.mark.parametrize(("enabled", "ttl"), [(False, 100), (True, 0)]) +def test_store_cache_can_be_turned_off(enabled: bool, ttl: int) -> None: + """A turned-off cache keeps nothing, nor does TTL 0. + + Args: + enabled: Whether the cache is on. + ttl: TTL of the value. + """ + store = make_store() + cache = StoreCache(store, LOG, enabled=enabled) + cache.put("9.9.9.9", Location(latitude=1, longitude=1), ttl) + cache.add("4.4.4.4", None, ttl) + assert cache.get("9.9.9.9") == (False, None) + assert store.get(store.key_for("location", "9.9.9.9")) is None + assert store.get(store.key_for("location", "4.4.4.4")) is None + + +def test_store_cache_add_keeps_unexpired_entries() -> None: + """`add()` stores only addresses that aren't cached.""" + store = make_store() + cache = StoreCache(store, LOG) + here = Location(latitude=1, longitude=2) + cache.put("9.9.9.9", here, 100) + cache.add("9.9.9.9", None, 10) + assert cache.get("9.9.9.9") == (True, here) + cache.add("1.1.1.1", None, 10) + assert cache.get("1.1.1.1") == (True, None) + + +@pytest.mark.parametrize("value", [[91, 0], [1], ["x", "y"], {"lat": 1}, 3, None]) +def test_store_cache_ignores_invalid_entries(value: Any) -> None: + """Entries that are neither locations nor misses count as not cached. + + Args: + value: Stored value. + """ + store = make_store() + store.set(store.key_for("location", "9.9.9.9"), value, ttl=100) + assert StoreCache(store, LOG).get("9.9.9.9") == (False, None) + + +@pytest.mark.parametrize( + "value", [[10**400, 0], [0, -(10**400)], [float("nan"), 0], [0, float("inf")]] +) +def test_invalid_numbers_are_no_locations(value: Any) -> None: + """Numbers out of range, also beyond floats (e.g., of other backends), fail. + + Args: + value: Stored value. + """ + assert _location(value) == (False, None) + + +def test_store_cache_survives_store_failures(caplog: pytest.LogCaptureFixture) -> None: + """A failing store leaves addresses uncached; the call asks it no more. + + Args: + caplog: Pytest fixture. + """ + backend = failing_backend() + cache = StoreCache(make_store(backend), LOG) + with caplog.at_level("WARNING"): + assert cache.get("9.9.9.9") == (False, None) + cache.put("9.9.9.9", Location(latitude=1, longitude=1), 100) + cache.add("4.4.4.4", None, 100) + assert cache.get("9.9.9.9") == (False, None) + assert backend.get.call_count == 1 + assert backend.set.call_count == 0 + (record,) = caplog.records + assert "ServerSelectionTimeoutError" in record.getMessage() + assert "9.9.9.9" not in record.getMessage() + + +def test_store_cache_survives_failing_writes() -> None: + """A failing write leaves the address uncached.""" + backend = MagicMock(wraps=MemoryStoreBackend()) + backend.set.side_effect = StoreError("Plugin store failed: WriteError") + cache = StoreCache(make_store(backend), LOG) + cache.put("9.9.9.9", Location(latitude=1, longitude=1), 100) + assert cache.get("9.9.9.9") == (False, None) + assert backend.get.call_count == 0 + + +def test_budget_counts_per_caller() -> None: + """Each caller gets `limit` lookups per window; anonymous ones share one.""" + store = make_store() + alice = UserBudget(store, Principal(subject="alice"), 2, LOG) + assert [alice.take(), alice.take(), alice.take()] == [True, True, False] + # a new call of the same caller: the budget is used up + assert not UserBudget(store, Principal(subject="alice"), 2, LOG).take() + # a service client with the same identifier is another caller + client = Principal(subject="alice", subject_type="client") + assert UserBudget(store, client, 2, LOG).take() + anonymous = [UserBudget(store, Principal(), 2, LOG) for _ in range(3)] + assert [budget.take() for budget in anonymous] == [True, True, False] + + +def test_budget_keys_are_hashes() -> None: + """The counter's key is a hash, not the caller's identifier.""" + store = make_store() + UserBudget(store, Principal(subject="alice"), 2, LOG).take() + key = store.key_for("dbip-lookups", "user", "alice") + assert key.startswith("h:") + assert store.get(key) == 1 + + +def test_budget_is_a_fixed_window() -> None: + """Counts reset a day after the first lookup of the window.""" + now = [datetime(2026, 11, 13, tzinfo=UTC)] + store = make_store(clock=lambda: now[0]) + assert UserBudget(store, Principal(subject="alice"), 1, LOG).take() + now[0] += timedelta(hours=23) + assert not UserBudget(store, Principal(subject="alice"), 1, LOG).take() + now[0] += timedelta(hours=1) + assert UserBudget(store, Principal(subject="alice"), 1, LOG).take() + + +def test_budget_gives_back_unused_lookups() -> None: + """A lookup given back can be taken again.""" + store = make_store() + budget = UserBudget(store, Principal(subject="alice"), 1, LOG) + assert budget.take() + budget.give_back() + assert budget.take() + assert not budget.take() + + +def test_budget_gives_back_nothing_to_expired_counters() -> None: + """Giving back to an expired counter starts no window below 0.""" + now = [datetime(2026, 11, 13, tzinfo=UTC)] + store = make_store(clock=lambda: now[0]) + budget = UserBudget(store, Principal(subject="alice"), 1, LOG) + assert budget.take() + now[0] += timedelta(days=1) + budget.give_back() + assert store.get(budget.key) is None + assert budget.take() + assert not budget.take() + + +def test_budget_fails_closed(caplog: pytest.LogCaptureFixture) -> None: + """Without a working store, no lookup is allowed; it is asked once. + + Args: + caplog: Pytest fixture. + """ + backend = failing_backend() + budget = UserBudget(make_store(backend), Principal(subject="alice"), 5, LOG) + with caplog.at_level("WARNING"): + assert not budget.take() + assert not budget.take() + budget.give_back() + assert backend.increment.call_count == 1 + (record,) = caplog.records + assert "ServerSelectionTimeoutError" in record.getMessage() + assert "alice" not in record.getMessage() + + +def test_budget_survives_failing_give_back() -> None: + """A failed give-back leaves the lookup counted.""" + backend = MagicMock(wraps=MemoryStoreBackend()) + store = make_store(backend) + budget = UserBudget(store, Principal(subject="alice"), 5, LOG) + assert budget.take() + backend.increment.side_effect = StoreError("Plugin store failed: WriteError") + budget.give_back() + assert store.get(store.key_for("dbip-lookups", "user", "alice")) == 1 diff --git a/uv.lock b/uv.lock index 8177900..acb708d 100644 --- a/uv.lock +++ b/uv.lock @@ -1178,11 +1178,11 @@ wheels = [ [[package]] name = "dnspython" -version = "2.9.0" +version = "2.9.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ef/4a/50822184bd67cc6493f0fb6a880749158fcd31ab3fa07409acfd91f9fc85/dnspython-2.9.0.tar.gz", hash = "sha256:b44dc6b18f07a8b1c56676a19fbfdb5209415b046a9cece286baafa87ff3f7f1", size = 423560, upload-time = "2026-10-09T00:07:24.352Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/49/900383b471c645a3c397f47b3211d1fcecb3a540c98f683494cdf86b33de/dnspython-2.9.2.tar.gz", hash = "sha256:8d4d39a513e649d634004ce4d39ade2352efcea472a0247535a5c2d18a9d7ba9", size = 455484, upload-time = "2026-11-03T23:56:53.601Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/02/cdcc9b7c051786a103c3b09e1003a82fa0c66bcb91ffbdabcfbf7b4163b9/dnspython-2.9.0-py3-none-any.whl", hash = "sha256:9a4aedb833c3c1b49214d04d44d3032ab7a9135f7c1d29a549b4ff78fd82fda9", size = 354822, upload-time = "2026-10-09T00:07:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0c/5346358b3a20275d256978c35203ddc2c8617f78c1b23795b79ba22ced6d/dnspython-2.9.2-py3-none-any.whl", hash = "sha256:af15f7d385a6ae2fb35a796fde2138004c8c42c7389090e2addc83e3b3340fd8", size = 360455, upload-time = "2026-11-03T23:56:52.088Z" }, ] [[package]] @@ -1289,7 +1289,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "foca", specifier = ">=0.18.0,<0.23" }, + { name = "foca", specifier = ">=0.19.0,<0.23" }, { name = "pydantic", specifier = ">=2.7,<3" }, ] @@ -1304,7 +1304,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "foca", specifier = ">=0.18.0,<0.23" }, + { name = "foca", specifier = ">=0.19.0,<0.23" }, { name = "pydantic", specifier = ">=2.7,<3" }, ] @@ -1319,7 +1319,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "foca", specifier = ">=0.18.0,<0.23" }, + { name = "foca", specifier = ">=0.19.0,<0.23" }, { name = "pydantic", specifier = ">=2.7,<3" }, ] @@ -1335,7 +1335,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "foca", specifier = ">=0.18.0,<0.23" }, + { name = "foca", specifier = ">=0.19.0,<0.23" }, { name = "maxminddb", specifier = ">=2.6,<4" }, { name = "pydantic", specifier = ">=2.7,<3" }, ] @@ -1368,7 +1368,7 @@ tes-routing = [ [package.metadata] requires-dist = [ - { name = "foca", specifier = ">=0.18.0,<0.23" }, + { name = "foca", specifier = ">=0.19.0,<0.23" }, { name = "foca-plugin-generic-routing-random", marker = "extra == 'experimental'", editable = "plugins/generic/routing/random" }, { name = "foca-plugin-generic-routing-random", marker = "extra == 'generic'", editable = "plugins/generic/routing/random" }, { name = "foca-plugin-generic-routing-random", marker = "extra == 'generic-routing'", editable = "plugins/generic/routing/random" },