Migration 21 adds companies.all_roles, tags every favourite of any agent and each company whose board had params {"reach": "all"} (logged per board, the param removed). reach.plan reads a tagged company's boards with the union of the enabled agents' require reach and the base reach; favourite no longer widens the fetch. POST /api/companies/{key}/all-roles sets it and queues a pull when on. The Companies list, the reader and the company strip draw the toggle beside the affinity buttons, the list filters on it, and the board's reach editor drops its All roles option.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
758 lines
34 KiB
Python
758 lines
34 KiB
Python
"""The runner: what the app does for every reader, board or search, from what
|
|
the reader declares in `READS`.
|
|
|
|
A reader says how its host answers; this module reads it: the guarded fetch
|
|
(`fetch`, bound to one target by `getter`), the `Ask` a subscription resolves
|
|
to (`ask_for`) and its series (`series`), the listing whole or paged, one
|
|
series per `one`-filter value unioned by id (`read_listing`), completeness,
|
|
the sweep of an oversized paged board across pulls, the 10% guard on a
|
|
filtered or `empty="suspect"` read (`read`), the per-role detail reads
|
|
(`read_details`), the `raw` an `Item` carries (`raw_of`), and the failure
|
|
bookkeeping (`record_failure`, `failure_reason`). `plugins/ats.py` and
|
|
`plugins/search.py` are thin `Source` classes over it: they pass their own
|
|
limits (`Limits`) and state prefix (`ats`, `search`), so their state keys stay
|
|
as they were.
|
|
|
|
A module without `READS` is read with the older signatures, `feed_url(slug)`
|
|
and `page(get, slug, cursor)`, and never gets an `Ask`.
|
|
|
|
State keys, under the Source's prefix and the target (`ats:sweep:<target>`):
|
|
|
|
sweep a paged read cut at `max_pages`: `{"asks", "series", "cursor",
|
|
"seen", "rows", "whole"}`, continued by the next pull
|
|
count `{"asks", "count"}`, the roles the last whole read listed, for
|
|
the 10% guard
|
|
facets `{"checked", "facets"}`, a board's `facets`, kept `FACETS_TTL`
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import itertools
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import time
|
|
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime, timedelta
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import USER_AGENT
|
|
|
|
from ._contract import Ask, Key, Place, Reads
|
|
from ._shared import Get, Row
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: openings.go's 8 MB body cap is not kept: every URL fetched here is a feed
|
|
#: on a fixed allow-list, and the real run of 2026-09-23 hit 8 MB on the
|
|
#: biggest employers (Anduril at 41 MB, Anthropic, OpenAI, Databricks between
|
|
#: 9 and 19 MB). 96 MB fits them; a board still over it gets one retry at its
|
|
#: reader's `fallback_url` where it has one.
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: The first byte of a Greenhouse board waited up to 16.6 s in 84 requests
|
|
#: measured on 2026-09-29, and the body then arrived in under 1 s. So a read
|
|
#: may wait 30 s and a whole request, redirects and body included, 60 s.
|
|
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
|
DEADLINE = 60.0
|
|
|
|
#: Pages read from one target in one pull at most, and the pause between two
|
|
#: requests (pages and detail reads), which keeps Workable from answering 429.
|
|
MAX_PAGES = 100
|
|
PAGE_DELAY = 0.5
|
|
|
|
#: Roles read in detail from one board in one pull at most.
|
|
DETAIL_CAP = 200
|
|
|
|
#: The row keys a reader's `detail` may fill: those whose column survives a
|
|
#: later pull that sends the listing's empty value, so what a role's own page
|
|
#: said is not wiped the pull after it was read. `title`, `url`, `department`
|
|
#: and `salary_text` are left out, since the listing rewrites them every pull.
|
|
#: Workday, SuccessFactors and Rippling state `country` and `employment` on a
|
|
#: role's own page only.
|
|
DETAIL_FIELDS = ("description", "location", "places", "teaser", "remote", "country",
|
|
"employment", "currency", "salary_min", "salary_max", "posted")
|
|
|
|
#: How long a board's `facets` are kept before they are asked again.
|
|
FACETS_TTL = timedelta(days=1)
|
|
|
|
#: A filtered or `empty="suspect"` read listing under this share of what the
|
|
#: same read listed last time counts as a failure.
|
|
SUSPECT_SHARE = 0.1
|
|
|
|
#: The keys of a row that an `Item` carries in its own fields, not in `raw`.
|
|
ITEM_KEYS = ("source_id", "title", "url", "description", "posted")
|
|
|
|
|
|
class Refused(Exception):
|
|
"""A request or a redirect aimed at a host its allow-list does not cover."""
|
|
|
|
|
|
class TooLarge(ValueError):
|
|
"""A response over the byte cap. A `ValueError`, as it was before it had a
|
|
type of its own."""
|
|
|
|
|
|
class Suspect(Exception):
|
|
"""A filtered or `empty="suspect"` read that listed under `SUSPECT_SHARE`
|
|
of what it listed last time."""
|
|
|
|
|
|
class MissingKey(ValueError):
|
|
"""A reader's `READS.key` has no value in the config or the environment."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Limits:
|
|
"""What one Source allows a read: pages, the pause between requests, and
|
|
the detail reads a pull."""
|
|
|
|
max_pages: int = MAX_PAGES
|
|
page_delay: float = PAGE_DELAY
|
|
detail_cap: int = DETAIL_CAP
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Listing:
|
|
"""One read of a target's listing. `rows` are this pull's rows, `ids` the
|
|
`source_id`s the read listed: this pull's, or the whole sweep's once it
|
|
reached the end. `complete` says `ids` are the whole list. `sweep` is the
|
|
state to keep for the next pull, None when no sweep is under way.
|
|
`filtered` says the read asked the host for a part of the list."""
|
|
|
|
rows: list[Row]
|
|
complete: bool
|
|
ids: list[str] = field(default_factory=list)
|
|
filtered: bool = False
|
|
sweep: dict[str, Any] | None = None
|
|
asks: str = ""
|
|
|
|
|
|
# --------------------------------------------------------------- modules
|
|
|
|
def declared(module: ModuleType) -> bool:
|
|
"""Whether `module` declares `READS`, and so takes `ask`."""
|
|
return isinstance(getattr(module, "READS", None), Reads)
|
|
|
|
|
|
def module_of(name: str) -> ModuleType | None:
|
|
"""The reader named `name`, a board host first, else a search provider."""
|
|
from plugins import readers
|
|
|
|
return readers.HOSTS.get(name) or readers.PROVIDERS.get(name)
|
|
|
|
|
|
# ----------------------------------------------------------------- fetch
|
|
|
|
#: Host name suffixes that name a machine on a private network.
|
|
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
|
|
|
|
|
def local_host(name: str) -> bool:
|
|
"""Whether a host name is an address or a local name rather than a public
|
|
DNS name: an IP literal, anything whose last label is a number
|
|
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1), `localhost`
|
|
or a private suffix."""
|
|
name = name.strip("[]").rstrip(".").lower()
|
|
try:
|
|
ipaddress.ip_address(name)
|
|
return True
|
|
except ValueError:
|
|
pass
|
|
last = name.rsplit(".", 1)[-1]
|
|
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
|
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
|
|
|
|
|
def fetch(client: httpx.Client, url: str, allow: Callable[[str], bool], *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None, max_bytes: int | None = None,
|
|
deadline: float | None = None, name: str = "") -> bytes:
|
|
"""Fetch one URL for one target: https on port 443 only, never an address
|
|
or a local name, only hosts `allow` accepts, at most three redirects each
|
|
checked the same way, the body cut at `max_bytes` (`MAX_BYTES`) and the
|
|
whole request ended within `deadline` (`DEADLINE`) seconds. A POST sends
|
|
`payload` as JSON, `{}` when None; `headers` are added to the
|
|
User-Agent and Accept and replace either where they name it.
|
|
|
|
The body is streamed and counted as it arrives, and the deadline checked
|
|
between chunks, so a trickling answer cannot run on. Raises `Refused`,
|
|
`TooLarge`, `httpx.HTTPStatusError` on a non-2xx answer, `httpx.HTTPError`
|
|
on a timeout or a connection failure, and ValueError on a broken
|
|
redirect."""
|
|
cap = MAX_BYTES if max_bytes is None else max_bytes
|
|
limit = DEADLINE if deadline is None else deadline
|
|
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
|
ends = time.monotonic() + limit
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
|
or local_host(parsed.host) or not allow(parsed.host)):
|
|
raise Refused(f"{current} is not an allowed host for {name or 'this reader'}")
|
|
with client.stream(
|
|
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
|
json=(payload or {}) if method == "POST" else None,
|
|
) as response:
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
chunks: list[bytes] = []
|
|
size = 0
|
|
for chunk in response.iter_bytes():
|
|
size += len(chunk)
|
|
if size > cap:
|
|
raise TooLarge(f"response too large (over {cap} bytes)")
|
|
if time.monotonic() > ends:
|
|
raise httpx.ReadTimeout(f"no whole answer within {limit:.0f} s",
|
|
request=response.request)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def getter(client: httpx.Client, allow: Callable[[str], bool], name: str = "") -> Get:
|
|
"""The `get` a reader is handed: `fetch` bound to one target's allow-list."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
return fetch(client, url, allow, method=method, payload=payload, headers=headers,
|
|
name=name)
|
|
return get
|
|
|
|
|
|
# ------------------------------------------------------------------- ask
|
|
|
|
#: A filter dimension of `Filters` -> the field of `Ask` it fills.
|
|
_ASK_FIELD = {"country": "countries", "place": "places", "remote": "remote", "query": "query"}
|
|
|
|
#: Keys a subscription's params or a reach may use, -> the dimension.
|
|
_PARAM_DIMENSION = {"countries": "country", "country": "country", "places": "place",
|
|
"place": "place", "remote": "remote", "query": "query", "q": "query"}
|
|
|
|
|
|
def _values(value: Any) -> tuple[str, ...]:
|
|
"""A param's value as a tuple of non-empty strings."""
|
|
items = value if isinstance(value, list | tuple | set) else [value]
|
|
return tuple(str(v).strip() for v in items if v not in (None, "", 0) and str(v).strip())
|
|
|
|
|
|
def _dimensions(module: ModuleType, given: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""`dimension -> value` from a params or reach mapping: the generic keys
|
|
(`countries`, `places`, `remote`, `q`) and the reader's own `PARAMS`
|
|
(saved param -> dimension, e.g. Jobindex's `area` -> `place`)."""
|
|
names = _PARAM_DIMENSION | dict(getattr(module, "PARAMS", {}) or {})
|
|
out: dict[str, Any] = {}
|
|
for key, value in given.items():
|
|
dimension = names.get(key)
|
|
if dimension == "remote":
|
|
if isinstance(value, bool):
|
|
out["remote"] = value
|
|
elif dimension == "query":
|
|
if str(value or "").strip():
|
|
out["query"] = str(value).strip()
|
|
elif dimension in ("country", "place"):
|
|
values = _values(value)
|
|
if dimension == "country":
|
|
values = tuple(v.upper() for v in values)
|
|
if values:
|
|
out[dimension] = values
|
|
return out
|
|
|
|
|
|
def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
|
|
"""A reader's key: each of `key.fields` from the config entry
|
|
`sources.search.<key.name>` (`keys` is `sources.search`), else from the
|
|
environment variable at the same position in `key.env`. Raises
|
|
`MissingKey` naming both places when a field has neither."""
|
|
entry = (keys or {}).get(key.name) or {}
|
|
out: dict[str, str] = {}
|
|
for index, name in enumerate(key.fields):
|
|
env = key.env[index] if index < len(key.env) else ""
|
|
value = str((entry.get(name) if isinstance(entry, Mapping) else "") or "") \
|
|
or (os.environ.get(env, "") if env else "")
|
|
if not value:
|
|
where = f"sources.search.{key.name}: {', '.join(key.fields)}"
|
|
if key.env:
|
|
where += f", or {' and '.join(key.env)}"
|
|
raise MissingKey(f"{key.name} needs a key: {where}")
|
|
out[name] = value
|
|
return out
|
|
|
|
|
|
#: The most series a reach may cost on a `one` filter: a reach of more
|
|
#: countries (the wide reach, Europe) reads such a host unfiltered instead.
|
|
MAX_SERIES = 8
|
|
|
|
|
|
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
|
reach: Mapping[str, Any] | None = None, *,
|
|
keys: Mapping[str, Any] | None = None) -> Ask:
|
|
"""The `Ask` one target resolves to. `reach` (from the agents, see
|
|
`app/jobs/reach.py`) and then the subscription's own `params` (a board's
|
|
override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's
|
|
saved params) set the dimensions, params winning per dimension, narrowed
|
|
to the ones the reader declares in `READS.filters`. A reach of more
|
|
than `MAX_SERIES` countries on a reader whose country filter is `one` is
|
|
dropped. A `remote` of False asks nothing. A search's params go whole into
|
|
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
|
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
|
A module without `READS` gets `Ask()`."""
|
|
if not declared(module):
|
|
return Ask()
|
|
reads: Reads = module.READS
|
|
params = dict(subscription_params or {})
|
|
reached = _dimensions(module, reach or {})
|
|
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
|
reached = {}
|
|
wanted = reached | _dimensions(module, params)
|
|
if wanted.get("remote") is False:
|
|
del wanted["remote"]
|
|
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
|
|
if name in reads.filters.declared()}
|
|
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
|
|
if reads.key is not None:
|
|
extra["key"] = resolve_key(reads.key, keys)
|
|
return Ask(**fields, params=extra)
|
|
|
|
|
|
def filtered(ask: Ask) -> bool:
|
|
"""Whether `ask` reads a part of a board: a country, a place or remote."""
|
|
return bool(ask.countries or ask.places or ask.remote is not None)
|
|
|
|
|
|
def series(module: ModuleType, ask: Ask,
|
|
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
|
|
"""The series of requests `ask` takes on this reader. Remote with a
|
|
country or a place is two parts, "in these places, or remote": the
|
|
places without remote, and remote alone. Each part is one series per
|
|
value of a dimension declared `one` (the product when two are), one
|
|
with every value for `many`, and host ids from `facets` for `ids`:
|
|
values `facets` does not know are dropped, and a part with none left is
|
|
no series, since the host has nothing of it; a dimension `facets` does
|
|
not state at all is read unfiltered, and so is remote alone, while the
|
|
remote part beside places is then left out. `[ask]` for a module
|
|
without `READS`."""
|
|
if not declared(module):
|
|
return [ask]
|
|
if ask.remote and (ask.countries or ask.places):
|
|
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
|
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
|
facets, beside=True)]
|
|
return _series(module, ask, facets)
|
|
|
|
|
|
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
|
beside: bool = False) -> list[Ask]:
|
|
"""`series` for an ask that is places or remote, not both. `beside` marks
|
|
the remote part of an ask that also has places."""
|
|
reach = module.READS.filters
|
|
known_facets = facets or {}
|
|
countries, places = ask.countries, ask.places
|
|
for dimension in ("country", "place"):
|
|
values = countries if dimension == "country" else places
|
|
if getattr(reach, dimension) != "ids" or not values:
|
|
continue
|
|
if dimension not in known_facets:
|
|
log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension)
|
|
mapped: tuple[str, ...] = ()
|
|
else:
|
|
known = known_facets.get(dimension) or {}
|
|
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
|
|
if not mapped:
|
|
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
|
|
", ".join(values))
|
|
return []
|
|
if dimension == "country":
|
|
countries = mapped
|
|
else:
|
|
places = mapped
|
|
if reach.remote == "ids" and ask.remote:
|
|
if "remote" not in known_facets:
|
|
if beside:
|
|
log.info("readers: %s states no remote facet; remote left out", module.NAME)
|
|
return []
|
|
ask = ask.model_copy(update={"remote": None})
|
|
else:
|
|
ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values()))
|
|
if not ids:
|
|
return []
|
|
ask = ask.model_copy(update={"remote_ids": ids})
|
|
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
|
|
else [countries]
|
|
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
|
|
return [ask.model_copy(update={"countries": c, "places": p})
|
|
for c, p in itertools.product(by_country, by_place)]
|
|
|
|
|
|
def fingerprint(asks: Iterable[Ask]) -> str:
|
|
"""What a read asked, without its params (which may hold a key): the same
|
|
string for the same series, so a sweep or a count is only compared with
|
|
a read that asked the same."""
|
|
return json.dumps([ask.model_dump(exclude={"params"}, mode="json") for ask in asks],
|
|
sort_keys=True)
|
|
|
|
|
|
# --------------------------------------------------------------- listing
|
|
|
|
def _result_ids(results: Iterable[dict[str, Any]]) -> set[str]:
|
|
return {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
|
|
|
|
|
def _whole(module: ModuleType, slug: str, get: Get, asks: list[Ask],
|
|
limits: Limits) -> dict[str, Row]:
|
|
"""A whole listing, one request a series, unioned by `source_id`. A body
|
|
over the cap gets one retry at the reader's `fallback_url`."""
|
|
rows: dict[str, Row] = {}
|
|
for index, ask in enumerate(asks):
|
|
if index:
|
|
time.sleep(limits.page_delay)
|
|
url = module.feed_url(slug, ask) if declared(module) else module.feed_url(slug)
|
|
try:
|
|
body = get(url)
|
|
except TooLarge:
|
|
if not hasattr(module, "fallback_url"):
|
|
raise
|
|
log.warning("readers: %s:%s: body over the cap; reading the smaller feed",
|
|
module.NAME, slug)
|
|
body = get(module.fallback_url(slug))
|
|
for row in module.parse(body, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
return rows
|
|
|
|
|
|
def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None, *,
|
|
limits: Limits | None = None, sweep: Mapping[str, Any] | None = None,
|
|
can_sweep: bool = False,
|
|
facets: Mapping[str, Mapping[str, str]] | None = None) -> Listing:
|
|
"""One target's listing, read and parsed. Raises on any failure of the
|
|
first request, and on a reader that does not understand an answer.
|
|
|
|
A whole listing is complete whenever it parses. A paged one reads each
|
|
series' pages, `limits.page_delay` apart, until the host has no next
|
|
page, the stated total is reached, a page brings nothing new (a host
|
|
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
|
A series is complete when it read as many distinct result ids as its
|
|
stated total, or, without a total, when it ended on the host's last
|
|
page; never for `complete="never"`. The read is complete when every
|
|
series is. A 429 after the first page ends the read, incomplete.
|
|
|
|
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
|
the `sweep` to keep, and the next pull, handed it back, continues from
|
|
its series and cursor; the sweep is complete when it reaches the end,
|
|
and `ids` then hold every id it listed. A read at the reader's `ceiling`
|
|
is not swept: the host answers nothing past it."""
|
|
limits = limits or Limits()
|
|
ask = ask or Ask()
|
|
asks = series(module, ask, facets)
|
|
is_filtered = filtered(ask)
|
|
key = fingerprint(asks)
|
|
if not asks:
|
|
return Listing([], True, [], is_filtered, None, key)
|
|
if not hasattr(module, "page"):
|
|
rows = _whole(module, slug, get, asks, limits)
|
|
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
|
|
|
|
reads: Reads | None = module.READS if declared(module) else None
|
|
never = reads is not None and reads.complete == "never"
|
|
ceiling = reads.ceiling if reads is not None else None
|
|
start, cursor, seen, earlier, whole = 0, None, set(), [], True
|
|
if sweep and sweep.get("asks") == key:
|
|
start, cursor = int(sweep.get("series") or 0), sweep.get("cursor")
|
|
seen, earlier = set(sweep.get("seen") or []), list(sweep.get("rows") or [])
|
|
whole = bool(sweep.get("whole", True))
|
|
rows: dict[str, Row] = {}
|
|
pages = 0
|
|
stop: str = ""
|
|
index = start
|
|
for index in range(start, len(asks)):
|
|
total: Any = None
|
|
ended = stalled = False
|
|
while True:
|
|
if pages >= limits.max_pages:
|
|
stop = "cap"
|
|
break
|
|
if pages:
|
|
time.sleep(limits.page_delay)
|
|
try:
|
|
if reads is not None:
|
|
data, results, total, cursor = module.page(get, slug, cursor, asks[index])
|
|
else:
|
|
data, results, total, cursor = module.page(get, slug, cursor)
|
|
except httpx.HTTPStatusError as exc:
|
|
if not pages or exc.response.status_code != 429:
|
|
raise
|
|
log.warning("readers: %s:%s: 429 on page %d; kept %d, not complete",
|
|
module.NAME, slug, pages + 1, len(rows))
|
|
stop = "429"
|
|
break
|
|
pages += 1
|
|
before = len(seen)
|
|
seen |= _result_ids(results)
|
|
for row in module.rows(data, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
|
ended = True
|
|
break
|
|
if total is None and len(seen) == before:
|
|
stalled = True
|
|
break
|
|
if stop:
|
|
break
|
|
done = (not never and not stalled
|
|
and (len(seen) >= total if isinstance(total, int) else ended))
|
|
if not done:
|
|
log.warning("readers: %s:%s: read %d of %s; not complete", module.NAME, slug,
|
|
len(seen), "an unstated total" if total is None else total)
|
|
whole = whole and done
|
|
cursor, seen = None, set()
|
|
|
|
listed = list(dict.fromkeys(earlier + list(rows)))
|
|
if not stop:
|
|
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
|
|
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
|
module.NAME, slug, pages, stop)
|
|
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
|
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
|
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
|
|
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
|
"rows": listed, "whole": whole}
|
|
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
|
|
|
|
|
|
def _storable(cursor: Any) -> bool:
|
|
"""Whether a cursor survives a trip through JSON in state."""
|
|
try:
|
|
return json.loads(json.dumps(cursor)) == cursor
|
|
except (TypeError, ValueError):
|
|
return False
|
|
|
|
|
|
# ------------------------------------------------------------ with state
|
|
|
|
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
|
try:
|
|
value = json.loads(state.get(key) or "{}")
|
|
except (TypeError, ValueError):
|
|
return {}
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
|
|
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
|
|
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
|
|
(a country, a place, or remote),
|
|
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
|
|
if not declared(module) or not hasattr(module, "facets"):
|
|
return {}
|
|
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
|
|
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
|
|
return {}
|
|
now = datetime.now(UTC)
|
|
if state is not None and key:
|
|
kept = _load(state, key)
|
|
try:
|
|
if now - datetime.fromisoformat(kept["checked"]) < FACETS_TTL:
|
|
return dict(kept.get("facets") or {})
|
|
except (KeyError, TypeError, ValueError):
|
|
pass
|
|
found = dict(module.facets(get, slug) or {})
|
|
if state is not None and key:
|
|
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
|
"facets": found})
|
|
return found
|
|
|
|
|
|
def read(state: MutableMapping[str, str], prefix: str, target: str, module: ModuleType,
|
|
slug: str, get: Get, ask: Ask | None = None, *,
|
|
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
|
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
|
the facets it needs, the sweep it continues, and the 10% guard. A read
|
|
that is filtered or whose reader declares `empty="suspect"`, and that
|
|
listed under `SUSPECT_SHARE` of what the same read listed last time,
|
|
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
|
the rest once the rows are stored."""
|
|
ask = ask or Ask()
|
|
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
|
listing = read_listing(module, slug, get, ask, limits=limits, can_sweep=can_sweep,
|
|
sweep=_load(state, f"{prefix}:sweep:{target}"), facets=facets)
|
|
suspect = declared(module) and module.READS.empty == "suspect"
|
|
last = _load(state, f"{prefix}:count:{target}")
|
|
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
|
if ((listing.filtered or suspect) and listing.sweep is None and count
|
|
and len(listing.ids) < SUSPECT_SHARE * count):
|
|
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
|
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
|
return listing
|
|
|
|
|
|
def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
|
listing: Listing, ids: list[str]) -> None:
|
|
"""Record a read whose rows are stored: `ok` (the run's marker),
|
|
`complete`, `ids` (the compound ids, what the pull closes against),
|
|
`filtered`, the sweep to continue or none, and the count for the 10%
|
|
guard once a read is whole."""
|
|
state[f"{prefix}:ok:{target}"] = run
|
|
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
|
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
|
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
|
if listing.sweep is not None:
|
|
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
|
else:
|
|
state.pop(f"{prefix}:sweep:{target}", None)
|
|
state[f"{prefix}:count:{target}"] = json.dumps({"asks": listing.asks,
|
|
"count": len(listing.ids)})
|
|
|
|
|
|
# ---------------------------------------------------------------- details
|
|
|
|
def id_set(state: Mapping[str, str], key: str) -> set[str]:
|
|
"""A JSON list of ids kept in state, as a set; empty when there is none
|
|
or it does not parse."""
|
|
try:
|
|
return {str(i) for i in json.loads(state.get(key) or "[]")}
|
|
except (TypeError, ValueError):
|
|
return set()
|
|
|
|
|
|
def read_details(module: ModuleType, name: str, slug: str, get: Get, rows: list[Row],
|
|
detailed: set[str], failed: set[str], *, listed: set[str] | None = None,
|
|
limits: Limits | None = None) -> tuple[list[Row], set[str], set[str]]:
|
|
"""The rows with the roles not yet read in detail filled from their own
|
|
pages, at most `limits.detail_cap`, `limits.page_delay` apart: first the
|
|
roles never tried, then those in `failed`. Ids are compound,
|
|
`<name>:<slug>:<source_id>`. Returns the rows and the ids read and
|
|
failed, to keep as the caller's `detailed` and `failed` sets.
|
|
|
|
Only `DETAIL_FIELDS` are taken from what `detail` returns. A `location`
|
|
without `places` becomes the one place. Every row not read now goes out
|
|
with an empty description, so one read earlier stays, and a row read on
|
|
an earlier pull without the listing's place and remote, so what its page
|
|
said stays. A 404 or 410 counts as read, a 429 stops the reads for this
|
|
pull, anything else keeps the role in `failed`. `listed` is every id of a
|
|
complete listing: ids outside it leave both sets."""
|
|
limits = limits or Limits()
|
|
done, failing = set(detailed), set(failed) - set(detailed)
|
|
|
|
def compound(row: Row) -> str:
|
|
return f"{name}:{slug}:{row['source_id']}"
|
|
|
|
unread = [row for row in rows if compound(row) not in done]
|
|
queue = ([row for row in unread if compound(row) not in failing]
|
|
+ [row for row in unread if compound(row) in failing])[:limits.detail_cap]
|
|
filled: dict[str, Row] = {}
|
|
for row in queue:
|
|
key = compound(row)
|
|
time.sleep(limits.page_delay)
|
|
try:
|
|
answer = module.detail(get, slug, row)
|
|
except httpx.HTTPStatusError as exc:
|
|
code = exc.response.status_code
|
|
if code == 429:
|
|
log.warning("readers: %s:%s: 429 on a detail read; the rest wait", name, slug)
|
|
break
|
|
if code in (404, 410):
|
|
done.add(key)
|
|
failing.discard(key)
|
|
else:
|
|
log.warning("readers: %s:%s: detail of %s answered %s", name, slug,
|
|
row["source_id"], code)
|
|
failing.add(key)
|
|
continue
|
|
except Exception as exc:
|
|
log.warning("readers: %s:%s: detail of %s failed (%s)", name, slug,
|
|
row["source_id"], exc)
|
|
failing.add(key)
|
|
continue
|
|
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
|
log.debug("readers: %s:%s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
|
name, slug, row["source_id"], ", ".join(ignored))
|
|
taken: Row = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} # type: ignore[assignment]
|
|
if taken.get("location") and "places" not in taken:
|
|
taken["places"] = [{"text": taken["location"], "remote": None}]
|
|
filled[key] = taken
|
|
done.add(key)
|
|
failing.discard(key)
|
|
out: list[Row] = []
|
|
for row in rows:
|
|
key = compound(row)
|
|
if key in filled:
|
|
out.append(row | {"description": ""} | filled[key]) # type: ignore[operator]
|
|
elif key in detailed:
|
|
out.append(row | {"description": "", "location": "", "places": [], # type: ignore[operator]
|
|
"remote": None})
|
|
else:
|
|
out.append(row | {"description": ""}) # type: ignore[operator]
|
|
if listed is not None:
|
|
done &= listed
|
|
failing &= listed
|
|
return out, done, failing
|
|
|
|
|
|
# -------------------------------------------------------------------- raw
|
|
|
|
def places_of(row: Mapping[str, Any]) -> list[Place]:
|
|
"""A row's places: its `places`, or, for a row without (an undeclared
|
|
reader's), its `location` as the one place, split nothing."""
|
|
places = row.get("places")
|
|
if not isinstance(places, list):
|
|
location = str(row.get("location") or "").strip()
|
|
return [{"text": location, "remote": None}] if location else []
|
|
out: list[Place] = []
|
|
for place in places:
|
|
if isinstance(place, Mapping) and str(place.get("text") or "").strip():
|
|
out.append({"text": str(place["text"]).strip(), "remote": place.get("remote")})
|
|
return out
|
|
|
|
|
|
def raw_of(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
"""The `raw` of an `Item` for one row: every key the row states except
|
|
those the `Item` carries itself (`ITEM_KEYS`), with `places` always a
|
|
list, `place` the display line of the places joined with "; ", and the
|
|
keys storage reads always present (`department`, `remote`,
|
|
`salary_text`, `salary_min`, `salary_max`, `currency`, `teaser`)."""
|
|
places = places_of(row)
|
|
raw = {k: v for k, v in row.items() if k not in ITEM_KEYS}
|
|
return raw | {
|
|
"places": places,
|
|
"place": "; ".join(place["text"] for place in places),
|
|
"department": row.get("department") or "",
|
|
"remote": row.get("remote"),
|
|
"salary_text": row.get("salary_text") or "",
|
|
"salary_min": row.get("salary_min"),
|
|
"salary_max": row.get("salary_max"),
|
|
"currency": row.get("currency") or "",
|
|
"teaser": row.get("teaser") or "",
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------- failures
|
|
|
|
def failure_reason(exc: BaseException) -> str:
|
|
"""One short line for why a target did not answer: the status of an HTTP
|
|
error, else the exception's class and the first line of its message."""
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
|
lines = str(exc).strip().splitlines()
|
|
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
|
return text[:200]
|
|
|
|
|
|
def record_failure(state: MutableMapping[str, str], key: str, exc: BaseException) -> None:
|
|
"""Keep a failure under `key` as `{"since", "reason"}`: the first failure
|
|
in a row sets `since`, later ones only the reason. The pull closes a
|
|
target's roles once `since` is old enough; its next answer removes the
|
|
key."""
|
|
since = _load(state, key).get("since")
|
|
state[key] = json.dumps({"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
|
"reason": failure_reason(exc)})
|