Files
jobboard/plugins/readers/_runner.py
T

953 lines
44 KiB
Python

"""The runner: what the app does for every reader, board or search, from what
the reader declares in `READS`.
A reader says how its host answers; this module reads it: the guarded fetch
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
to (`ask_for`) and its series (`series`), the listing whole or paged, one
series per `one`-filter value unioned by id (`read_listing`), completeness,
the sweep of an oversized paged board across pulls, the 10% guard on every
read that is not a sweep (`read`), the per-role detail reads
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
`plugins/search.py` are thin `Source` classes over it: they pass their own
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
as they were. Every reader declares `READS`; a module without it is refused
by the package before it gets here.
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
"seen", "rows", "whole"}`, continued by the next pull
count `{"asks", "count"}`, the roles the last read that was not a
sweep listed, for the 10% guard
failing `{"since", "reason"}`, kept by the Source (`record_failure`);
the guard gives way once `since` is `Limits.stale_days` old
remote_unasked "1" when the reach wanted remote and the read could not
ask for it (`Listing.remote_unasked`), else ""
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
swept "1" when the last stamped listing finished a sweep carried over
from an earlier pull (`Listing.swept`), else "". Offset paging
drifts when roles close between pulls, so `app.jobs.pull` closes
a role of such a board only when two finished sweeps in a row
missed it (it keeps the ids the last one missed in `missing`)
"""
from __future__ import annotations
import ipaddress
import itertools
import json
import logging
import os
import re
import time
from collections.abc import Callable, Iterable, Mapping, MutableMapping
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from types import ModuleType
from typing import Any
import httpx
from desk.sources import USER_AGENT
from ._contract import Ask, Key, Place, Reads
from ._shared import Get, Row
log = logging.getLogger(__name__)
#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed
#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the
#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between
#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its
#: reader's `fallback_url` where it has one.
MAX_BYTES = 96 << 20
#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests
#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read
#: may wait 30 s and a whole request, redirects and body included, 60 s.
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
DEADLINE = 60.0
#: Pages read from one target in one pull at most, and the pause between two
#: requests (pages and detail reads), which keeps Workable from answering 429.
MAX_PAGES = 100
PAGE_DELAY = 0.5
#: Roles read in detail from one board in one pull at most.
DETAIL_CAP = 200
#: The row keys a reader's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value, so what a role's own page
#: said is not wiped the pull after it was read. `title`, `url`, `department`
#: and `salary_text` are left out, since the listing rewrites them every pull.
#: Workday, SuccessFactors and Rippling state `country` and `employment` on a
#: role's own page only.
DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country",
"employment", "currency", "salary_min", "salary_max", "posted")
#: How long a board's `facets` are kept before they are asked again.
FACETS_TTL = timedelta(days=1)
#: A read listing under this share of what the same read listed last time
#: counts as a failure (`read`).
SUSPECT_SHARE = 0.1
#: The last count from which the guard holds every read. A read that is
#: filtered, narrowed by terms or of an `empty="suspect"` reader is held from
#: a count of 1, as it was before every read was; a whole read of a board
#: that listed a handful of roles can drop to none by itself.
SUSPECT_MIN = 10
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
class Refused(Exception):
"""A request or a redirect aimed at a host its allow-list does not cover."""
class TooLarge(ValueError):
"""A response over the byte cap. A `ValueError`, as it was before it had a
type of its own."""
class Suspect(Exception):
"""A read that listed under `SUSPECT_SHARE` of what it listed last time
(see `read`)."""
class MissingKey(ValueError):
"""A reader's `READS.key` has no value in the config or the environment."""
@dataclass(frozen=True, slots=True)
class Limits:
"""What one Source allows a read: pages, the pause between requests, the
detail reads a pull, and the days after which the 10% guard accepts the
answer it keeps refusing (`stale_days`, the pull's `STALE_DAYS`; None,
for a Source run outside a pull, never)."""
max_pages: int = MAX_PAGES
page_delay: float = PAGE_DELAY
detail_cap: int = DETAIL_CAP
stale_days: int | None = None
@dataclass(slots=True)
class Listing:
"""One read of a target's listing. `rows` are this pull's rows, `ids` the
`source_id`s the read listed: this pull's, or the whole sweep's once it
reached the end. `complete` says `ids` are the whole list. `sweep` is the
state to keep for the next pull, None when no sweep is under way.
`filtered` says the read asked the host for the roles in some places,
`narrowed` for the roles matching words (`narrowed()`): the ids of
neither are the whole board. `swept` says a complete listing finished a
sweep carried over from an earlier pull: `ids` were read across pulls,
and roles closing meanwhile may have shifted an offset-paged host's later
roles back past the cursor, so `ids` can miss a role that is still open.
`remote_unasked` says a filtered read was meant to ask for remote roles
as well and could not, so its `ids` say nothing of the remote roles."""
rows: list[Row]
complete: bool
ids: list[str] = field(default_factory=list)
filtered: bool = False
sweep: dict[str, Any] | None = None
asks: str = ""
narrowed: bool = False
swept: bool = False
remote_unasked: bool = False
# --------------------------------------------------------------- modules
def declared(module: ModuleType) -> bool:
"""Whether `module` declares `READS`. Every reader does, so the runner no
longer asks; `app.jobs.reach` still does."""
return isinstance(getattr(module, "READS", None), Reads)
def module_of(name: str) -> ModuleType | None:
"""The reader named `name`, a board host first, else a search provider."""
from plugins import readers
return readers.HOSTS.get(name) or readers.PROVIDERS.get(name)
# ----------------------------------------------------------------- fetch
#: Host name suffixes that name a machine on a private network.
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
def local_host(name: str) -> bool:
"""Whether a host name is an address or a local name rather than a public
DNS name: an IP literal, anything whose last label is a number
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost`
or a private suffix."""
name = name.strip("[]").rstrip(".").lower()
try:
ipaddress.ip_address(name)
return True
except ValueError:
pass
last = name.rsplit(".", 1)[-1]
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None, max_bytes: int | None = None,
deadline: float | None = None, name: str = "") -> bytes:
"""Fetch one URL for one target: https on port 443 only, never an address
or a local name, only hosts `allow` accepts, at most three redirects each
checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the
whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends
`payload` as JSON, `{}` when None; `headers` are added to the
User-Agent and Accept and replace either where they name it.
The body is streamed and counted as it arrives, and the deadline checked
between chunks, so a trickling answer cannot run on. Raises `Refused`,
`TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError`
on a timeout or a connection failure, and ValueError on a broken
redirect."""
cap = MAX_BYTES if max_bytes is None else max_bytes
limit = DEADLINE if deadline is None else deadline
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
ends = time.monotonic() + limit
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if (parsed.scheme != "https" or parsed.port not in (None, 443)
or local_host(parsed.host) or not allow(parsed.host)):
raise Refused(f"{current} is not an allowed host for {name or 'this reader'}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
json=(payload or {}) if method == "POST" else None,
) as response:
if response.is_redirect:
location = response.headers.get("location") or ""
if not location:
raise ValueError("redirect with no location")
current = str(httpx.URL(current).join(location))
continue
response.raise_for_status()
chunks: list[bytes] = []
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > cap:
raise TooLarge(f"response too large (over {cap} bytes)")
if time.monotonic() > ends:
raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s",
request=response.request)
chunks.append(chunk)
return b"".join(chunks)
raise ValueError("too many redirects")
def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get:
"""The `get` a reader is handed: `fetch` bound to one target's allow-list."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
return fetch(client, url, allow, method=method, payload=payload, headers=headers,
name=name)
return get
# ------------------------------------------------------------------- ask
#: A filter dimension of `Filters` -> the field of `Ask` it fills.
_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"}
#: Keys a subscription's params or a reach may use, -> the dimension.
_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place",
"place": "place", "remote": "remote", "query": "query", "q": "query"}
def _values(value: Any) -> tuple[str, ...]:
"""A param's value as a tuple of non-empty strings."""
items = value if isinstance(value, list | tuple | set) else [value]
return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip())
def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]:
"""`dimension -> value` from a params or reach mapping: the generic keys
(`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS`
(saved param -> dimension, e.g. Jobindex's `area` -> `place`)."""
names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {})
out: dict[str, Any] = {}
for key, value in given.items():
dimension = names.get(key)
if dimension == "remote":
if isinstance(value, bool):
out["remote"] = value
elif dimension == "query":
if str(value or "").strip():
out["query"] = str(value).strip()
elif dimension in ("country", "place"):
values = _values(value)
if dimension == "country":
values = tuple(v.upper() for v in values)
if values:
out[dimension] = values
return out
def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
"""A reader's key: each of `key.fields` from the config entry
`sources.search.<key.name>` (`keys` is `sources.search`), else from the
environment variable at the same position in `key.env`. Raises
`MissingKey` naming both places when a field has neither."""
entry = (keys or {}).get(key.name) or {}
out: dict[str, str] = {}
for index, name in enumerate(key.fields):
env = key.env[index] if index < len(key.env) else ""
value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \
or (os.environ.get(env, "") if env else "")
if not value:
where = f"sources.search.{key.name}: {', '.join(key.fields)}"
if key.env:
where += f", or {' and '.join(key.env)}"
raise MissingKey(f"{key.name} needs a key: {where}")
out[name] = value
return out
#: The most series a reach may cost: a reach of more countries on a `one`
#: filter (the wide reach, Europe) reads such a host unfiltered instead, and
#: fetch terms times the reach's series may not cost more either.
MAX_SERIES = 8
def _terms(value: Any) -> tuple[str, ...]:
"""Fetch terms as given in a reach: stripped, empty ones dropped, one of
each regardless of case, in the order given."""
seen: set[str] = set()
out: list[str] = []
for term in _values(value):
if term.casefold() not in seen:
seen.add(term.casefold())
out.append(term)
return tuple(out)
def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
"""How many series the location part of `wanted` takes on this reader:
one per value of a `one` dimension (the product of two), one for a
`many` or `ids` dimension, and one more for remote beside places, which
the runner asks as a series of its own (see `series`)."""
count = 1
for dimension in ("country", "place"):
values = wanted.get(dimension, ())
if values and getattr(reads.filters, dimension) == "one":
count *= len(values)
if wanted.get("remote") and (wanted.get("country") or wanted.get("place")):
count += 1
return count
def _wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None,
reach: Mapping[str, Any] | None) -> tuple[dict[str, Any], bool]:
"""The dimensions `reach` and then the params set, params winning, before
they are narrowed to what the reader declares, and whether the read is
wide. A reach of more than `MAX_SERIES` countries on a `one` country
filter is dropped whole, which makes the read wide; a `remote` of False
asks nothing."""
given = reach or {}
reached = _dimensions(module, given)
wide = bool(given.get("wide"))
if module.READS.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
reached = {}
wide = True
wanted = reached | _dimensions(module, dict(subscription_params or {}))
if wanted.get("remote") is False:
del wanted["remote"]
return wanted, wide
def remote_wanted(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None) -> bool:
"""Whether the reach or the params want remote roles, whether or not the
reader can ask for them. `read_listing` compares it with what the series
asked (`Listing.remote_unasked`)."""
return _wanted(module, subscription_params, reach)[0].get("remote") is True
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask:
"""The `Ask` one target resolves to. `reach` (from the agents, see
`app/jobs/reach.py`) and then the subscription's own `params` (a board's
override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's
saved params) set the dimensions, params winning per dimension, narrowed
to the ones the reader declares in `READS.filters`. A reach of more
than `MAX_SERIES` countries on a reader whose country filter is `one` is
dropped. A `remote` of False asks nothing. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
A remote the reader cannot filter on is dropped here; `remote_wanted`
still says it was wanted.
Fetch terms (`reach["terms"]`) go to a board reader that declares
`query="one"`, on the wide and the narrow read alike. A company set to All
roles, and a board that is small, unknown or overridden, gets no terms in
its reach (see `app/jobs/reach.py`), and a board whose params name a query
of their own takes none. Terms times the reach's series (`_cost`) may cost
at most `MAX_SERIES`. Over it, a wide read (`reach["wide"]`, a company set
to All locations, or a reach this rule just dropped as too many
countries) drops the place part, whole, since remote without its
countries would read only the remote roles, and keeps the terms. A narrow
read keeps the places, which the agent's fetch gate needs, and drops the
terms, so it is not `narrowed`. Terms alone over it are dropped on either
and the board is read unfiltered."""
reads: Reads = module.READS
params = dict(subscription_params or {})
given = reach or {}
wanted, wide = _wanted(module, params, given)
declared_now = reads.filters.declared()
wanted = {name: value for name, value in wanted.items() if name in declared_now}
terms = _terms(given.get("terms"))
if reads.scope != "board" or reads.filters.query != "one" or "query" in wanted:
terms = ()
if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES:
if wide:
wanted = {name: value for name, value in wanted.items()
if name not in ("country", "place", "remote")}
else:
terms = ()
if len(terms) > MAX_SERIES:
terms = ()
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()}
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
if reads.key is not None:
extra["key"] = resolve_key(reads.key, keys)
return Ask(**fields, terms=terms, params=extra)
def filtered(ask: Ask) -> bool:
"""Whether `ask` reads a part of a board by place: a country, a place or
remote. Terms are `narrowed`."""
return bool(ask.countries or ask.places or ask.remote is not None)
def narrowed(module: ModuleType, ask: Ask) -> bool:
"""Whether `ask` reads a board by words: fetch terms, or a board's own
query. Such a read is never the whole board, nor the board within its
places, so it closes nothing. A search's query is what
the search is, and does not count."""
board = module.READS.scope == "board"
return bool(ask.terms) or (board and bool(ask.query))
def series(module: ModuleType, ask: Ask,
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
"""The series of requests `ask` takes on this reader. Remote with a
country or a place is two parts, "in these places, or remote": the
places without remote, and remote alone. Each part is one series per
value of a dimension declared `one` (the product when two are), one
with every value for `many`, and host ids from `facets` for `ids`:
values `facets` does not know are dropped, and a part with none left is
no series, since the host has nothing of it; a dimension `facets` does
not state at all is read unfiltered, and so is remote alone, while the
remote part beside places is then left out. Each of those is then asked
once per fetch term (`ask.terms`), with the term as `query`: the host
intersects a keyword with its facets, as the survey found on every
host, so the union by id is "one of the terms, in these places"."""
if ask.remote and (ask.countries or ask.places):
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
facets, beside=True)]
else:
parts = _series(module, ask, facets)
if not ask.terms:
return parts
return [part.model_copy(update={"query": term}) for term in ask.terms for part in parts]
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
beside: bool = False) -> list[Ask]:
"""`series` for an ask that is places or remote, not both. `beside` marks
the remote part of an ask that also has places."""
reach = module.READS.filters
known_facets = facets or {}
countries, places = ask.countries, ask.places
for dimension in ("country", "place"):
values = countries if dimension == "country" else places
if getattr(reach, dimension) != "ids" or not values:
continue
if dimension not in known_facets:
log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension)
mapped: tuple[str, ...] = ()
else:
known = known_facets.get(dimension) or {}
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
if not mapped:
log.warning("readers: %s knows none of %s %s; nothing asked", module.NAME,
dimension, ", ".join(values))
return []
if dimension == "country":
countries = mapped
else:
places = mapped
if reach.remote == "ids" and ask.remote:
if "remote" not in known_facets:
if beside:
log.info("readers: %s states no remote facet; remote left out", module.NAME)
return []
ask = ask.model_copy(update={"remote": None})
else:
ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values()))
if not ids:
return []
ask = ask.model_copy(update={"remote_ids": ids})
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
else [countries]
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
return [ask.model_copy(update={"countries": c, "places": p})
for c, p in itertools.product(by_country, by_place)]
def fingerprint(asks: Iterable[Ask]) -> str:
"""What a read asked, without its params (which may hold a key): the same
string for the same series, so a sweep or a count is only compared with
a read that asked the same."""
return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks],
sort_keys=True)
# --------------------------------------------------------------- listing
def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
limits: Limits) -> dict[str, Row]:
"""A whole listing, one request a series, unioned by `source_id`. A body
over the cap gets one retry at the reader's `fallback_url`."""
rows: dict[str, Row] = {}
for index, ask in enumerate(asks):
if index:
time.sleep(limits.page_delay)
url = module.feed_url(slug, ask)
try:
body = get(url)
except TooLarge:
if not hasattr(module, "fallback_url"):
raise
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
module.NAME, slug)
body = get(module.fallback_url(slug))
for row in module.parse(body, slug):
rows.setdefault(row["source_id"], row)
return rows
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
can_sweep: bool = False,
facets: Mapping[str, Mapping[str, str]] | None = None,
remote: bool = False) -> Listing:
"""One target's listing, read and parsed. Raises on any failure of the
first request, and on a reader that does not understand an answer.
A whole listing is complete whenever it parses. A paged one reads each
series' pages, `limits.page_delay` apart, until the host has no next
page, the stated total is reached, the read stalls or `limits.max_pages`
pages in all. A stall is a page that brings no new id while the reader
still hands a cursor: any such page on a host that states no total (it
ignores the cursor), and with a total one the reader hands over empty
(it saw the page come again). A series is complete when it read as many distinct result
ids as its stated total, or, without a total, when it ended on the
host's last page; never after a stall, and never for
`complete="never"`. The read is complete when every series is. A 429
after the first page ends the read, incomplete.
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
the `sweep` to keep, and the next pull, handed it back, continues from
its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed, and the listing is `swept`. The
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
read at the reader's `ceiling` is not swept: the host answers nothing
past it.
A filtered read that asks no series, because an `ids` dimension names
no value the host's facets know, asked nothing, which is no evidence of
anything: it is incomplete and closes nothing. `filtered` is a board's
only: a search's places are what the search is, as its query is.
`remote` says the reach or the params wanted remote roles; when no series
asked for them (a reader without a remote filter, or one whose facets
state none beside places) the listing is `remote_unasked`."""
limits = limits or Limits()
ask = ask or Ask()
reads: Reads = module.READS
asks = series(module, ask, facets)
is_filtered = filtered(ask) and reads.scope == "board"
by_words = narrowed(module, ask)
unasked = (is_filtered and (remote or bool(ask.remote))
and not any(a.remote or a.remote_ids for a in asks))
key = fingerprint(asks)
if not asks:
log.warning("readers: %s:%s: the filter maps to nothing the host knows; "
"nothing asked, not closing", module.NAME, slug)
return Listing([], False, [], True, None, key, by_words, remote_unasked=unasked)
if not hasattr(module, "page"):
whole_rows = _whole(module, slug, get, asks, limits)
return Listing(list(whole_rows.values()), True, list(whole_rows), is_filtered, None,
key, by_words, remote_unasked=unasked)
never = reads.complete == "never"
ceiling = reads.ceiling
start, cursor, seen, earlier, whole = 0, None, set(), [], True
resumed = bool(sweep and sweep.get("asks") == key)
if sweep and resumed:
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
whole = bool(sweep.get("whole", True))
rows: dict[str, Row] = {}
pages = 0
stop: str = ""
index = start
for index in range(start, len(asks)):
total: Any = None
ended = stalled = False
while True:
if pages >= limits.max_pages:
stop = "cap"
break
if pages:
time.sleep(limits.page_delay)
try:
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
except httpx.HTTPStatusError as exc:
if not pages or exc.response.status_code != 429:
raise
log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete",
module.NAME, slug, pages + 1, len(rows))
stop = "429"
break
pages += 1
before = len(seen)
seen |= _result_ids(results)
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True
break
# Nothing new while the reader says more follow. Without a total
# the host ignores the cursor, and the read would never end. With
# one, only a page the reader hands over empty is a stall (it saw
# the page come again, as SuccessFactors does): a page of ids
# already read is what a resumed sweep starts on when the list
# grew at the top since, and the read goes on from there.
if len(seen) == before and (total is None or not results):
stalled = True
break
if stop:
break
done = (not never and not stalled
and (len(seen) >= total if isinstance(total, int) else ended))
if not done:
log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug,
len(seen), "an unstated total" if total is None else total)
whole = whole and done
cursor, seen = None, set()
listed = list(dict.fromkeys(earlier + list(rows)))
if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words,
resumed and whole, unasked)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling
if not can_sweep or never or at_ceiling or not _storable(cursor):
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words,
remote_unasked=unasked)
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
"rows": listed, "whole": whole}
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words,
remote_unasked=unasked)
def _storable(cursor: Any) -> bool:
"""Whether a cursor survives a trip through JSON in state. A tuple comes
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
row, first id)`), which counts as surviving: the next pull hands the
reader its cursor as JSON gives it back, and every paged reader takes
the list as it took the tuple."""
try:
return json.loads(json.dumps(cursor)) == _as_json(cursor)
except (TypeError, ValueError):
return False
def _as_json(value: Any) -> Any:
"""`value` with every tuple in it a list, as JSON hands it back."""
if isinstance(value, tuple | list):
return [_as_json(item) for item in value]
if isinstance(value, dict):
return {key: _as_json(item) for key, item in value.items()}
return value
# ------------------------------------------------------------ with state
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
try:
value = json.loads(state.get(key) or "{}")
except (TypeError, ValueError):
return {}
return value if isinstance(value, dict) else {}
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
(a country, a place, or remote),
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
if not hasattr(module, "facets"):
return {}
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
return {}
now = datetime.now(UTC)
if state is not None and key:
kept = _load(state, key)
try:
if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL:
return dict(kept.get("facets") or {})
except (KeyError, TypeError, ValueError):
pass
found = dict(module.facets(get, slug) or {})
if state is not None and key:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"facets": found})
return found
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
slug: str, get: Get, ask: Ask | None = None, *,
limits: Limits | None = None, can_sweep: bool = True, remote: bool = False) -> Listing:
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
the facets it needs, the sweep it continues, and the 10% guard. Nothing
is written here but the facets; `stamp` keeps the rest once the rows are
stored.
The guard holds every read that is not a sweep: one that listed under
`SUSPECT_SHARE` of what the same read listed last time (the same
`fingerprint`, so a term read is only held to a term read with the same
terms and places) raises `Suspect`, and the Source records the target
as failing, so its roles stay open. A host whose answer changed shape
parses to no rows with a 200, and closing a board on that would lose it
whole. It holds from a last count of `SUSPECT_MIN`, or of 1 for a read
that is filtered, narrowed or of an `empty="suspect"` reader. A board
that really shrank is accepted once the target has been failing for
`limits.stale_days` (its `<prefix>:failing:<target>` record), and its
stamp then resets the count; a Recheck resets it at once
(`plugins.ats.keep_size`)."""
limits = limits or Limits()
ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets,
remote=remote)
held = listing.filtered or listing.narrowed or module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
if (listing.sweep is None and count >= (1 if held else SUSPECT_MIN)
and len(listing.ids) < SUSPECT_SHARE * count):
why = (f"listed {len(listing.ids)} roles, under "
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
if _failing_for(state, f"{prefix}:failing:{target}", limits.stale_days):
log.warning("readers: %s:%s: %s, accepted after %d days of failing",
module.NAME, slug, why, limits.stale_days)
return listing
raise Suspect(why)
return listing
def _failing_for(state: Mapping[str, str], key: str, days: int | None) -> bool:
"""Whether the failure kept under `key` began at least `days` ago; False
for `days` None or no record."""
since = str(_load(state, key).get("since") or "")
if days is None or not since:
return False
try:
began = datetime.fromisoformat(since)
except ValueError:
return False
return datetime.now(UTC) - began >= timedelta(days=days)
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
listing: Listing, ids: list[str]) -> None:
"""Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, `narrowed` (a read by terms, which the pull never closes
against), whether it finished a sweep from earlier pulls (`swept`),
whether a filtered read left remote unasked (`remote_unasked`), the
sweep to continue or none, and the count for the 10% guard once a read
is not a sweep. A read that asked nothing keeps the count it had."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:swept:{target}"] = "1" if listing.swept else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
state[f"{prefix}:remote_unasked:{target}"] = "1" if listing.remote_unasked else ""
if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
return
state.pop(f"{prefix}:sweep:{target}", None)
if listing.asks != fingerprint([]):
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
"count": len(listing.ids)})
# ---------------------------------------------------------------- details
def id_set(state: Mapping[str, str], key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is none
or it does not parse."""
try:
return {str(i) for i in json.loads(state.get(key) or "[]")}
except (TypeError, ValueError):
return set()
def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row],
detailed: set[str], failed: set[str], *, listed: set[str] | None = None,
limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]:
"""The rows with the roles not yet read in detail filled from their own
pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the
roles never tried, then those in `failed`. Ids are compound,
`<name>:<slug>:<source_id>`. Returns the rows and the ids read and
failed, to keep as the caller's `detailed` and `failed` sets.
Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location`
without `places` becomes the one place. Every row not read now goes out
with an empty description, so one read earlier stays, and a row read on
an earlier pull without the listing's place and remote, so what its page
said stays. A 404 or 410 counts as read, a 429 stops the reads for this
pull, anything else keeps the role in `failed`. `listed` is every id of a
complete listing: ids outside it leave both sets."""
limits = limits or Limits()
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{name}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:limits.detail_cap]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(limits.page_delay)
try:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug)
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("readers: %s:%s: detail of %s answered %s", name, slug,
row["source_id"], code)
failing.add(key)
continue
except Exception as exc:
log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug,
row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
name, slug, row["source_id"], ", ".join(ignored))
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
if taken.get("location") and "places" not in taken:
taken["places"] = [{"text": taken["location"], "remote": None}]
filled[key] = taken
done.add(key)
failing.discard(key)
out: list[Row] = []
for row in rows:
key = compound(row)
if key in filled:
out.append(row | {"description": ""} | filled[key])
elif key in detailed:
out.append(row | {"description": "", "location": "", "places": [],
"remote": None})
else:
out.append(row | {"description": ""})
if listed is not None:
done &= listed
failing &= listed
return out, done, failing
# -------------------------------------------------------------------- raw
def places_of(row: Mapping[str, Any]) -> list[Place]:
"""A row's places: its `places`, or, for a row without (a raw row stored
before readers returned places), its `location` as the one place, split
nothing."""
places = row.get("places")
if not isinstance(places, list):
location = str(row.get("location") or "").strip()
return [{"text": location, "remote": None}] if location else []
out: list[Place] = []
for place in places:
if isinstance(place, Mapping) and str(place.get("text") or "").strip():
out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")})
return out
def raw_of(row: Mapping[str, Any]) -> dict[str, Any]:
"""The `raw` of an `Item` for one row: every key the row states except
those the `Item` carries itself (`ITEM_KEYS`), with `places` always a
list, `place` the display line of the places joined with "; ", and the
keys storage reads always present (`department`, `remote`,
`salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`)."""
places = places_of(row)
raw = {k: v for k, v in row.items() if k not in ITEM_KEYS}
return raw | {
"places": places,
"place": "; ".join(place["text"] for place in places),
"department": row.get("department") or "",
"remote": row.get("remote"),
"salary_text": row.get("salary_text") or "",
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency") or "",
"teaser": row.get("teaser") or "",
}
# --------------------------------------------------------------- failures
def failure_reason(exc: BaseException) -> str:
"""One short line for why a target did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None:
"""Keep a failure under `key` as `{"since", "reason"}`: the first failure
in a row sets `since`, later ones only the reason. The pull closes a
target's roles once `since` is old enough; its next answer removes the
key."""
since = _load(state, key).get("since")
state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})