app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
759 lines
34 KiB
Python
759 lines
34 KiB
Python
"""The runner: what the app does for every reader, board or search, from what
|
|
the reader declares in `READS`.
|
|
|
|
A reader says how its host answers; this module reads it: the guarded fetch
|
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
|
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
|
the sweep of an oversized paged board across pulls, the 10% guard on a
|
|
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
|
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
|
as they were.
|
|
|
|
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
|
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
|
|
|
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
|
|
|
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
|
"seen", "rows", "whole"}`, continued by the next pull
|
|
count `{"asks", "count"}`, the roles the last whole read listed, for
|
|
the 10% guard
|
|
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import USER_AGENT
|
|
|
|
from ._contract import Ask, Key, Place, Reads
|
|
from ._shared import Get, Row
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed
|
|
#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the
|
|
#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between
|
|
#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its
|
|
#: reader's `fallback_url` where it has one.
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests
|
|
#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read
|
|
#: may wait 30 s and a whole request, redirects and body included, 60 s.
|
|
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
|
DEADLINE = 60.0
|
|
|
|
#: Pages read from one target in one pull at most, and the pause between two
|
|
#: requests (pages and detail reads), which keeps Workable from answering 429.
|
|
MAX_PAGES = 100
|
|
PAGE_DELAY = 0.5
|
|
|
|
#: Roles read in detail from one board in one pull at most.
|
|
DETAIL_CAP = 200
|
|
|
|
#: The row keys a reader's `detail` may fill: those whose column survives a
|
|
#: later pull that sends the listing's empty value, so what a role's own page
|
|
#: said is not wiped the pull after it was read. `title`, `url`, `department`
|
|
#: and `salary_text` are left out, since the listing rewrites them every pull.
|
|
#: Workday, SuccessFactors and Rippling state `country` and `employment` on a
|
|
#: role's own page only.
|
|
DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country",
|
|
"employment", "currency", "salary_min", "salary_max", "posted")
|
|
|
|
#: How long a board's `facets` are kept before they are asked again.
|
|
FACETS_TTL = timedelta(days=1)
|
|
|
|
#: A filtered or `empty="suspect"` read listing under this share of what the
|
|
#: same read listed last time counts as a failure.
|
|
SUSPECT_SHARE = 0.1
|
|
|
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
|
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
|
|
|
|
|
class Refused(Exception):
|
|
"""A request or a redirect aimed at a host its allow-list does not cover."""
|
|
|
|
|
|
class TooLarge(ValueError):
|
|
"""A response over the byte cap. A `ValueError`, as it was before it had a
|
|
type of its own."""
|
|
|
|
|
|
class Suspect(Exception):
|
|
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
|
of what it listed last time."""
|
|
|
|
|
|
class MissingKey(ValueError):
|
|
"""A reader's `READS.key` has no value in the config or the environment."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Limits:
|
|
"""What one Source allows a read: pages, the pause between requests, and
|
|
the detail reads a pull."""
|
|
|
|
max_pages: int = MAX_PAGES
|
|
page_delay: float = PAGE_DELAY
|
|
detail_cap: int = DETAIL_CAP
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Listing:
|
|
"""One read of a target's listing. `rows` are this pull's rows, `ids` the
|
|
`source_id`s the read listed: this pull's, or the whole sweep's once it
|
|
reached the end. `complete` says `ids` are the whole list. `sweep` is the
|
|
state to keep for the next pull, None when no sweep is under way.
|
|
`filtered` says the read asked the host for a part of the list."""
|
|
|
|
rows: list[Row]
|
|
complete: bool
|
|
ids: list[str] = field(default_factory=list)
|
|
filtered: bool = False
|
|
sweep: dict[str, Any] | None = None
|
|
asks: str = ""
|
|
|
|
|
|
# --------------------------------------------------------------- modules
|
|
|
|
def declared(module: ModuleType) -> bool:
|
|
"""Whether `module` declares `READS`, and so takes `ask`."""
|
|
return isinstance(getattr(module, "READS", None), Reads)
|
|
|
|
|
|
def module_of(name: str) -> ModuleType | None:
|
|
"""The reader named `name`, a board host first, else a search provider."""
|
|
from plugins import readers
|
|
|
|
return readers.HOSTS.get(name) or readers.PROVIDERS.get(name)
|
|
|
|
|
|
# ----------------------------------------------------------------- fetch
|
|
|
|
#: Host name suffixes that name a machine on a private network.
|
|
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
|
|
|
|
|
def local_host(name: str) -> bool:
|
|
"""Whether a host name is an address or a local name rather than a public
|
|
DNS name: an IP literal, anything whose last label is a number
|
|
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost`
|
|
or a private suffix."""
|
|
name = name.strip("[]").rstrip(".").lower()
|
|
try:
|
|
ipaddress.ip_address(name)
|
|
return True
|
|
except ValueError:
|
|
pass
|
|
last = name.rsplit(".", 1)[-1]
|
|
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
|
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
|
|
|
|
|
def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None, max_bytes: int | None = None,
|
|
deadline: float | None = None, name: str = "") -> bytes:
|
|
"""Fetch one URL for one target: https on port 443 only, never an address
|
|
or a local name, only hosts `allow` accepts, at most three redirects each
|
|
checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the
|
|
whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends
|
|
`payload` as JSON, `{}` when None; `headers` are added to the
|
|
User-Agent and Accept and replace either where they name it.
|
|
|
|
The body is streamed and counted as it arrives, and the deadline checked
|
|
between chunks, so a trickling answer cannot run on. Raises `Refused`,
|
|
`TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError`
|
|
on a timeout or a connection failure, and ValueError on a broken
|
|
redirect."""
|
|
cap = MAX_BYTES if max_bytes is None else max_bytes
|
|
limit = DEADLINE if deadline is None else deadline
|
|
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
|
ends = time.monotonic() + limit
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
|
or local_host(parsed.host) or not allow(parsed.host)):
|
|
raise Refused(f"{current} is not an allowed host for {name or 'this reader'}")
|
|
with client.stream(
|
|
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
|
json=(payload or {}) if method == "POST" else None,
|
|
) as response:
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
chunks: list[bytes] = []
|
|
size = 0
|
|
for chunk in response.iter_bytes():
|
|
size += len(chunk)
|
|
if size > cap:
|
|
raise TooLarge(f"response too large (over {cap} bytes)")
|
|
if time.monotonic() > ends:
|
|
raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s",
|
|
request=response.request)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get:
|
|
"""The `get` a reader is handed: `fetch` bound to one target's allow-list."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
return fetch(client, url, allow, method=method, payload=payload, headers=headers,
|
|
name=name)
|
|
return get
|
|
|
|
|
|
# ------------------------------------------------------------------- ask
|
|
|
|
#: A filter dimension of `Filters` -> the field of `Ask` it fills.
|
|
_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"}
|
|
|
|
#: Keys a subscription's params or a reach may use, -> the dimension.
|
|
_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place",
|
|
"place": "place", "remote": "remote", "query": "query", "q": "query"}
|
|
|
|
|
|
def _values(value: Any) -> tuple[str, ...]:
|
|
"""A param's value as a tuple of non-empty strings."""
|
|
items = value if isinstance(value, list | tuple | set) else [value]
|
|
return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip())
|
|
|
|
|
|
def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""`dimension -> value` from a params or reach mapping: the generic keys
|
|
(`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS`
|
|
(saved param -> dimension, e.g. Jobindex's `area` -> `place`)."""
|
|
names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {})
|
|
out: dict[str, Any] = {}
|
|
for key, value in given.items():
|
|
dimension = names.get(key)
|
|
if dimension == "remote":
|
|
if isinstance(value, bool):
|
|
out["remote"] = value
|
|
elif dimension == "query":
|
|
if str(value or "").strip():
|
|
out["query"] = str(value).strip()
|
|
elif dimension in ("country", "place"):
|
|
values = _values(value)
|
|
if dimension == "country":
|
|
values = tuple(v.upper() for v in values)
|
|
if values:
|
|
out[dimension] = values
|
|
return out
|
|
|
|
|
|
def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
|
|
"""A reader's key: each of `key.fields` from the config entry
|
|
`sources.search.<key.name>` (`keys` is `sources.search`), else from the
|
|
environment variable at the same position in `key.env`. Raises
|
|
`MissingKey` naming both places when a field has neither."""
|
|
entry = (keys or {}).get(key.name) or {}
|
|
out: dict[str, str] = {}
|
|
for index, name in enumerate(key.fields):
|
|
env = key.env[index] if index < len(key.env) else ""
|
|
value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \
|
|
or (os.environ.get(env, "") if env else "")
|
|
if not value:
|
|
where = f"sources.search.{key.name}: {', '.join(key.fields)}"
|
|
if key.env:
|
|
where += f", or {' and '.join(key.env)}"
|
|
raise MissingKey(f"{key.name} needs a key: {where}")
|
|
out[name] = value
|
|
return out
|
|
|
|
|
|
#: The most series a reach may cost on a `one` filter: a reach of more
|
|
#: countries (a favourite's Europe) reads such a host unfiltered instead.
|
|
MAX_SERIES = 8
|
|
|
|
|
|
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
|
reach: Mapping[str, Any] | None = None, *,
|
|
keys: Mapping[str, Any] | None = None) -> Ask:
|
|
"""The `Ask` one target resolves to. `reach` (from the agents, see
|
|
`app/jobs/reach.py`) and then the subscription's own `params` (a board's
|
|
override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's
|
|
saved params) set the dimensions, params winning per dimension, narrowed
|
|
to the ones the reader declares in `READS.filters`. `{"reach": "all"}` in
|
|
the params drops the reach, and so does a reach of more than
|
|
`MAX_SERIES` countries on a reader whose country filter is `one`. A
|
|
`remote` of False asks nothing. A search's params go whole into
|
|
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
|
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
|
A module without `READS` gets `Ask()`."""
|
|
if not declared(module):
|
|
return Ask()
|
|
reads: Reads = module.READS
|
|
params = dict(subscription_params or {})
|
|
reached = {} if params.get("reach") == "all" else _dimensions(module, reach or {})
|
|
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
|
reached = {}
|
|
wanted = reached | _dimensions(module, params)
|
|
if wanted.get("remote") is False:
|
|
del wanted["remote"]
|
|
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
|
|
if name in reads.filters.declared()}
|
|
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
|
|
if reads.key is not None:
|
|
extra["key"] = resolve_key(reads.key, keys)
|
|
return Ask(**fields, params=extra)
|
|
|
|
|
|
def filtered(ask: Ask) -> bool:
|
|
"""Whether `ask` reads a part of a board: a country, a place or remote."""
|
|
return bool(ask.countries or ask.places or ask.remote is not None)
|
|
|
|
|
|
def series(module: ModuleType, ask: Ask,
|
|
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
|
|
"""The series of requests `ask` takes on this reader. Remote with a
|
|
country or a place is two parts, "in these places, or remote": the
|
|
places without remote, and remote alone. Each part is one series per
|
|
value of a dimension declared `one` (the product when two are), one
|
|
with every value for `many`, and host ids from `facets` for `ids`:
|
|
values `facets` does not know are dropped, and a part with none left is
|
|
no series, since the host has nothing of it; a dimension `facets` does
|
|
not state at all is read unfiltered, and so is remote alone, while the
|
|
remote part beside places is then left out. `[ask]` for a module
|
|
without `READS`."""
|
|
if not declared(module):
|
|
return [ask]
|
|
if ask.remote and (ask.countries or ask.places):
|
|
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
|
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
|
facets, beside=True)]
|
|
return _series(module, ask, facets)
|
|
|
|
|
|
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
|
beside: bool = False) -> list[Ask]:
|
|
"""`series` for an ask that is places or remote, not both. `beside` marks
|
|
the remote part of an ask that also has places."""
|
|
reach = module.READS.filters
|
|
known_facets = facets or {}
|
|
countries, places = ask.countries, ask.places
|
|
for dimension in ("country", "place"):
|
|
values = countries if dimension == "country" else places
|
|
if getattr(reach, dimension) != "ids" or not values:
|
|
continue
|
|
if dimension not in known_facets:
|
|
log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension)
|
|
mapped: tuple[str, ...] = ()
|
|
else:
|
|
known = known_facets.get(dimension) or {}
|
|
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
|
if not mapped:
|
|
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
|
", ".join(values))
|
|
return []
|
|
if dimension == "country":
|
|
countries = mapped
|
|
else:
|
|
places = mapped
|
|
if reach.remote == "ids" and ask.remote:
|
|
if "remote" not in known_facets:
|
|
if beside:
|
|
log.info("readers: %s states no remote facet; remote left out", module.NAME)
|
|
return []
|
|
ask = ask.model_copy(update={"remote": None})
|
|
else:
|
|
ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values()))
|
|
if not ids:
|
|
return []
|
|
ask = ask.model_copy(update={"remote_ids": ids})
|
|
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
|
|
else [countries]
|
|
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
|
|
return [ask.model_copy(update={"countries": c, "places": p})
|
|
for c, p in itertools.product(by_country, by_place)]
|
|
|
|
|
|
def fingerprint(asks: Iterable[Ask]) -> str:
|
|
"""What a read asked, without its params (which may hold a key): the same
|
|
string for the same series, so a sweep or a count is only compared with
|
|
a read that asked the same."""
|
|
return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks],
|
|
sort_keys=True)
|
|
|
|
|
|
# --------------------------------------------------------------- listing
|
|
|
|
def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
|
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
|
|
|
|
|
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|
limits: Limits) -> dict[str, Row]:
|
|
"""A whole listing, one request a series, unioned by `source_id`. A body
|
|
over the cap gets one retry at the reader's `fallback_url`."""
|
|
rows: dict[str, Row] = {}
|
|
for index, ask in enumerate(asks):
|
|
if index:
|
|
time.sleep(limits.page_delay)
|
|
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
|
try:
|
|
body = get(url)
|
|
except TooLarge:
|
|
if not hasattr(module, "fallback_url"):
|
|
raise
|
|
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
|
module.NAME, slug)
|
|
body = get(module.fallback_url(slug))
|
|
for row in module.parse(body, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
return rows
|
|
|
|
|
|
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
|
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
|
can_sweep: bool = False,
|
|
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
|
"""One target's listing, read and parsed. Raises on any failure of the
|
|
first request, and on a reader that does not understand an answer.
|
|
|
|
A whole listing is complete whenever it parses. A paged one reads each
|
|
series' pages, `limits.page_delay` apart, until the host has no next
|
|
page, the stated total is reached, a page brings nothing new (a host
|
|
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
|
A series is complete when it read as many distinct result ids as its
|
|
stated total, or, without a total, when it ended on the host's last
|
|
page; never for `complete="never"`. The read is complete when every
|
|
series is. A 429 after the first page ends the read, incomplete.
|
|
|
|
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
|
the `sweep` to keep, and the next pull, handed it back, continues from
|
|
its series and cursor; the sweep is complete when it reaches the end,
|
|
and `ids` then hold every id it listed. A read at the reader's `ceiling`
|
|
is not swept: the host answers nothing past it."""
|
|
limits = limits or Limits()
|
|
ask = ask or Ask()
|
|
asks = series(module, ask, facets)
|
|
is_filtered = filtered(ask)
|
|
key = fingerprint(asks)
|
|
if not asks:
|
|
return Listing([], True, [], is_filtered, None, key)
|
|
if not hasattr(module, "page"):
|
|
rows = _whole(module, slug, get, asks, limits)
|
|
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
|
|
|
|
reads: Reads | None = module.READS if declared(module) else None
|
|
never = reads is not None and reads.complete == "never"
|
|
ceiling = reads.ceiling if reads is not None else None
|
|
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
|
if sweep and sweep.get("asks") == key:
|
|
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
|
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
|
whole = bool(sweep.get("whole", True))
|
|
rows: dict[str, Row] = {}
|
|
pages = 0
|
|
stop: str = ""
|
|
index = start
|
|
for index in range(start, len(asks)):
|
|
total: Any = None
|
|
ended = stalled = False
|
|
while True:
|
|
if pages >= limits.max_pages:
|
|
stop = "cap"
|
|
break
|
|
if pages:
|
|
time.sleep(limits.page_delay)
|
|
try:
|
|
if reads is not None:
|
|
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
|
else:
|
|
data, results, total, cursor = module.page(get, slug, cursor)
|
|
except httpx.HTTPStatusError as exc:
|
|
if not pages or exc.response.status_code != 429:
|
|
raise
|
|
log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete",
|
|
module.NAME, slug, pages + 1, len(rows))
|
|
stop = "429"
|
|
break
|
|
pages += 1
|
|
before = len(seen)
|
|
seen |= _result_ids(results)
|
|
for row in module.rows(data, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
|
ended = True
|
|
break
|
|
if total is None and len(seen) == before:
|
|
stalled = True
|
|
break
|
|
if stop:
|
|
break
|
|
done = (not never and not stalled
|
|
and (len(seen) >= total if isinstance(total, int) else ended))
|
|
if not done:
|
|
log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug,
|
|
len(seen), "an unstated total" if total is None else total)
|
|
whole = whole and done
|
|
cursor, seen = None, set()
|
|
|
|
listed = list(dict.fromkeys(earlier + list(rows)))
|
|
if not stop:
|
|
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
|
|
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
|
module.NAME, slug, pages, stop)
|
|
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
|
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
|
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
|
|
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
|
"rows": listed, "whole": whole}
|
|
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
|
|
|
|
|
|
def _storable(cursor: Any) -> bool:
|
|
"""Whether a cursor survives a trip through JSON in state."""
|
|
try:
|
|
return json.loads(json.dumps(cursor)) == cursor
|
|
except (TypeError, ValueError):
|
|
return False
|
|
|
|
|
|
# ------------------------------------------------------------ with state
|
|
|
|
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(state.get(key) or "{}")
|
|
except (TypeError, ValueError):
|
|
return {}
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
|
|
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
|
(a country, a place, or remote),
|
|
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
|
if not declared(module) or not hasattr(module, "facets"):
|
|
return {}
|
|
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
|
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
|
return {}
|
|
now = datetime.now(UTC)
|
|
if state is not None and key:
|
|
kept = _load(state, key)
|
|
try:
|
|
if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL:
|
|
return dict(kept.get("facets") or {})
|
|
except (KeyError, TypeError, ValueError):
|
|
pass
|
|
found = dict(module.facets(get, slug) or {})
|
|
if state is not None and key:
|
|
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
|
"facets": found})
|
|
return found
|
|
|
|
|
|
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
|
slug: str, get: Get, ask: Ask | None = None, *,
|
|
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
|
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
|
the facets it needs, the sweep it continues, and the 10% guard. A read
|
|
that is filtered or whose reader declares `empty="suspect"`, and that
|
|
listed under `SUSPECT_SHARE` of what the same read listed last time,
|
|
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
|
the rest once the rows are stored."""
|
|
ask = ask or Ask()
|
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
|
suspect = declared(module) and module.READS.empty == "suspect"
|
|
last = _load(state, f"{prefix}:count:{target}")
|
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
|
if ((listing.filtered or suspect) and listing.sweep is None and count
|
|
and len(listing.ids) < SUSPECT_SHARE * count):
|
|
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
|
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
|
return listing
|
|
|
|
|
|
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|
listing: Listing, ids: list[str]) -> None:
|
|
"""Record a read whose rows are stored: `ok` (the run's marker),
|
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
|
`filtered`, the sweep to continue or none, and the count for the 10%
|
|
guard once a read is whole."""
|
|
state[f"{prefix}:ok:{target}"] = run
|
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
|
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
|
if listing.sweep is not None:
|
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
|
else:
|
|
state.pop(f"{prefix}:sweep:{target}", None)
|
|
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
|
"count": len(listing.ids)})
|
|
|
|
|
|
# ---------------------------------------------------------------- details
|
|
|
|
def id_set(state: Mapping[str, str], key: str) -> set[str]:
|
|
"""A JSON list of ids kept in state, as a set; empty when there is none
|
|
or it does not parse."""
|
|
try:
|
|
return {str(i) for i in json.loads(state.get(key) or "[]")}
|
|
except (TypeError, ValueError):
|
|
return set()
|
|
|
|
|
|
def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row],
|
|
detailed: set[str], failed: set[str], *, listed: set[str] | None = None,
|
|
limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]:
|
|
"""The rows with the roles not yet read in detail filled from their own
|
|
pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the
|
|
roles never tried, then those in `failed`. Ids are compound,
|
|
`<name>:<slug>:<source_id>`. Returns the rows and the ids read and
|
|
failed, to keep as the caller's `detailed` and `failed` sets.
|
|
|
|
Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location`
|
|
without `places` becomes the one place. Every row not read now goes out
|
|
with an empty description, so one read earlier stays, and a row read on
|
|
an earlier pull without the listing's place and remote, so what its page
|
|
said stays. A 404 or 410 counts as read, a 429 stops the reads for this
|
|
pull, anything else keeps the role in `failed`. `listed` is every id of a
|
|
complete listing: ids outside it leave both sets."""
|
|
limits = limits or Limits()
|
|
done, failing = set(detailed), set(failed) - set(detailed)
|
|
|
|
def compound(row: Row) -> str:
|
|
return f"{name}:{slug}:{row['source_id']}"
|
|
|
|
unread = [row for row in rows if compound(row) not in done]
|
|
queue = ([row for row in unread if compound(row) not in failing]
|
|
+ [row for row in unread if compound(row) in failing])[:limits.detail_cap]
|
|
filled: dict[str, Row] = {}
|
|
for row in queue:
|
|
key = compound(row)
|
|
time.sleep(limits.page_delay)
|
|
try:
|
|
answer = module.detail(get, slug, row)
|
|
except httpx.HTTPStatusError as exc:
|
|
code = exc.response.status_code
|
|
if code == 429:
|
|
log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug)
|
|
break
|
|
if code in (404, 410):
|
|
done.add(key)
|
|
failing.discard(key)
|
|
else:
|
|
log.warning("readers: %s:%s: detail of %s answered %s", name, slug,
|
|
row["source_id"], code)
|
|
failing.add(key)
|
|
continue
|
|
except Exception as exc:
|
|
log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug,
|
|
row["source_id"], exc)
|
|
failing.add(key)
|
|
continue
|
|
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
|
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
|
name, slug, row["source_id"], ", ".join(ignored))
|
|
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
|
if taken.get("location") and "places" not in taken:
|
|
taken["places"] = [{"text": taken["location"], "remote": None}]
|
|
filled[key] = taken
|
|
done.add(key)
|
|
failing.discard(key)
|
|
out: list[Row] = []
|
|
for row in rows:
|
|
key = compound(row)
|
|
if key in filled:
|
|
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
|
elif key in detailed:
|
|
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
|
"remote": None})
|
|
else:
|
|
out.append(row | {"description": ""}) # type: ignore[operator]
|
|
if listed is not None:
|
|
done &= listed
|
|
failing &= listed
|
|
return out, done, failing
|
|
|
|
|
|
# -------------------------------------------------------------------- raw
|
|
|
|
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
|
"""A row's places: its `places`, or, for a row without (an undeclared
|
|
reader's), its `location` as the one place, split nothing."""
|
|
places = row.get("places")
|
|
if not isinstance(places, list):
|
|
location = str(row.get("location") or "").strip()
|
|
return [{"text": location, "remote": None}] if location else []
|
|
out: list[Place] = []
|
|
for place in places:
|
|
if isinstance(place, Mapping) and str(place.get("text") or "").strip():
|
|
out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")})
|
|
return out
|
|
|
|
|
|
def raw_of(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""The `raw` of an `Item` for one row: every key the row states except
|
|
those the `Item` carries itself (`ITEM_KEYS`), with `places` always a
|
|
list, `place` the display line of the places joined with "; ", and the
|
|
keys storage reads always present (`department`, `remote`,
|
|
`salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`)."""
|
|
places = places_of(row)
|
|
raw = {k: v for k, v in row.items() if k not in ITEM_KEYS}
|
|
return raw | {
|
|
"places": places,
|
|
"place": "; ".join(place["text"] for place in places),
|
|
"department": row.get("department") or "",
|
|
"remote": row.get("remote"),
|
|
"salary_text": row.get("salary_text") or "",
|
|
"salary_min": row.get("salary_min"),
|
|
"salary_max": row.get("salary_max"),
|
|
"currency": row.get("currency") or "",
|
|
"teaser": row.get("teaser") or "",
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------- failures
|
|
|
|
def failure_reason(exc: BaseException) -> str:
|
|
"""One short line for why a target did not answer: the status of an HTTP
|
|
error, else the exception's class and the first line of its message."""
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
|
lines = str(exc).strip().splitlines()
|
|
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
|
return text[:200]
|
|
|
|
|
|
def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None:
|
|
"""Keep a failure under `key` as `{"since", "reason"}`: the first failure
|
|
in a row sets `since`, later ones only the reason. The pull closes a
|
|
target's roles once `since` is old enough; its next answer removes the
|
|
key."""
|
|
since = _load(state, key).get("since")
|
|
state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
|
"reason": failure_reason(exc)})
|