Files
jobboard/plugins/ats.py
T

646 lines
32 KiB
Python

"""ATS boards: a company's own careers page on one of the hosts in
`plugins/readers/`, one module per host.
Every one of these hosts serves a company's own list of roles as public,
unauthenticated JSON, XML or RSS, and a role that drops off the board is
*closed*, which is the cheapest staleness check there is. What a host says
and how its feed reads is its module's, what every reader shares is
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
the classifier, and the names the app and the tests reach for on `ats`.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
`classify`'s nine functions and their keywords.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
whose read lists under a tenth of what it listed last time (`_runner.read`'s
guard), until it has failed that way for the pull's `stale_days`. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:<target>` with the day it was checked under
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
so a Recheck accepts a board that shrank); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
whose reader cannot narrow that reach is read whole anyway, and sized by
that read once it is complete, rather than by a second request.
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/fetch_text.py` after
scoring, for the roles in the agents' cut, through `read_board_details`. A search provider
whose reader has `detail` (Jobbank) is read the same way, under its search
target `provider:name`; `read_detail` is the one read the Fetch button makes.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
"""
from __future__ import annotations
import json
import logging
import re
import sys
from collections.abc import Callable, Iterator
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
)
try:
from plugins import readers
except ModuleNotFoundError:
# `desk.sources.registry` loads this file by its path, under a name of its
# own, and the app puts the repo root on `sys.path` before it does; a
# registry run without the app (a script, another working directory)
# does not, and the host modules are found as the package beside this
# file. So the root goes on the path then, and only then.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
from plugins.readers import _runner
from plugins.readers._shared import Get, Row
# The names the parsers had here before they moved to their host modules,
# kept for the callers and the tests that still reach for them on `ats`.
from plugins.readers._shared import ( # noqa: F401
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
)
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
log = logging.getLogger(__name__)
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
MAX_BYTES = _runner.MAX_BYTES
TIMEOUTS = _runner.TIMEOUTS
DEADLINE = _runner.DEADLINE
MAX_PAGES = _runner.MAX_PAGES
PAGE_DELAY = _runner.PAGE_DELAY
DETAIL_CAP = _runner.DETAIL_CAP
DETAIL_FIELDS = _runner.DETAIL_FIELDS
#: The runner's errors, under the names they had here.
_Refused = _runner.Refused
_TooLarge = _runner.TooLarge
_local_host = _runner.local_host
#: host -> the name the host gives itself: the hosts this plugin reads, in
#: the order a select offers them. What the Sources page's host select is
#: rendered from, so it cannot offer a host `is_known_target` refuses.
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
#: host -> parser, for the hosts whose module parses a whole body.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
def _limits(stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
stale_days=stale_days)
def reader_of(target: str) -> ModuleType | None:
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
None for a name that is neither. The two never share a name."""
host = target.partition(":")[0]
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board or search:
what its module's `allowed` says. Anything else is refused, at the first
hop and at every redirect."""
module = reader_of(host)
return module.allowed(slug) if module is not None else lambda h: False
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
`DEADLINE`."""
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
payload=payload, headers=headers, max_bytes=MAX_BYTES,
deadline=DEADLINE, name=host)
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
"""The `get` a host module is handed: `_get` bound to one board. `_get`
is looked up when a request is made, so a test that replaces `ats._get`
reaches every host; `payload` and `headers` are passed on only when
given, which keeps a replacement that predates them working."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
extra: dict[str, Any] = {}
if payload is not None:
extra["payload"] = payload
if headers:
extra["headers"] = headers
return _get(client, url, host, slug, method=method, **extra)
return get
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a title
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
#: twice: the first, narrower entry is checked before
#: recruiting/design/marketing/support/sales/data, and the second, which also
#: matches the bare word "product", is checked only after all of those, so a
#: title like "Data Product Manager" keeps classifying as data/product by the
#: same rule order the Go tool used.
#:
#: Not a straight port any more: the Go tool matched substrings, which let
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
#: "Italia" — a European department like "Commercial Italy" filed as
#: engineering. `_classify_text` now matches on word boundaries. A handful of
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
#: meant to also catch "management", "engineering", "recruiting", and a
#: trailing boundary on those would stop them matching the plural and
#: gerund forms real postings use ("Software Developers", "Account
#: Managers").
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
"customer success/support", "recruiting/people", "finance/legal/ops")
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("product", ("product manag", "product management", "product operations",
"program manag", "technical program")),
("recruiting/people", ("recruit", "talent", "people ops", "people &",
"people team", "people operations", "human resources",
"hr", "hrbp")),
("design", ("design", "ux", "user experience", "creative")),
("marketing", ("marketing", "growth", "content", "communications", "brand",
"seo", "social media", "demand gen", "events")),
("customer success/support", ("customer success", "customer support",
"customer experience", "customer service",
"support", "success", "implementation",
"onboarding", "professional services",
"solutions consult", "technical account",
"customer engineer", "deployment")),
("sales", ("sales", "account executive", "account manager", "account director",
"business development", "bdr", "sdr", "revenue", "partnership",
"go-to-market", "gtm", "solutions engineer", "solution engineer",
"pre-sales", "presales", "channel",
# "Commercial" is what French, Italian and Spanish org charts
# call the sales function ("Commercial Italy" is a sales
# department; see the docstring above), and the rest are the
# sales words of the seed's other languages that "sales"
# itself does not already cover (German, Dutch, Danish,
# Norwegian, Swedish, French) — a modest set, not a full port;
# `extract` is the rest.
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
"försäljning", "vente")),
("data", ("data", "analytics", "machine learning", "ml",
"applied science", "research scientist", "statistic")),
("product", ("product manag", "product management", "program manag",
"product operations", "technical program", "product")),
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
"compliance", "operations", "ops", "administrat",
"office", "procurement", "strategy", "corporate",
"facilities", "risk", "audit", "tax", "treasury",
"payroll", "executive assistant", "chief of staff",
"workplace", "supply chain", "logistics", "fraud",
"trust & safety", "trust and safety", "policy")),
("engineering", ("engineer", "software", "developer", "infrastructure",
"platform", "devops", "sre", "site reliability", "security",
"information technology", "it", "technology",
"technical staff", "hardware", "qa", "quality assurance",
"architect", "research", "science", "scientist", "systems",
"network", "cloud", "mobile", "frontend", "backend",
"full stack", "fullstack", "web",
# The developer/engineer words of the seed's other
# languages, same reasoning as sales' above.
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
"utvikler", "utvecklare", "ingenjör", "ingeniør",
"développeur", "developpeur", "ingénieur")),
)
#: Keywords that must match a whole word, not just its start: each one is
#: itself the first letters of an unrelated word that would otherwise false-
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
#: not listed here keeps the old prefix behaviour on purpose — see the
#: `_RULES` docstring above.
_WHOLE_WORD = frozenset({
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
"sre", "bdr", "sdr", "gtm",
})
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
trailing = r"\b" if keyword in _WHOLE_WORD else ""
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
for function, keywords in _RULES)
def _classify_text(text: str) -> str:
stripped = (text or "").strip()
if not stripped:
return ""
for function, patterns in _COMPILED_RULES:
if any(pattern.search(stripped) for pattern in patterns):
return function
return ""
def classify(department: str, title: str) -> str:
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
when neither names one — `''` and not `"other"`, which would claim a
knowledge the classifier does not have (`extract` may still fill it from
reading the posting; see `Store.set_extracted`).
Title before department: a department is often a generic umbrella
("Operations", "Engineering & Product") that says less about the role
than its title does. A hand-checked sample of 60 roles where the two
orders disagreed (docs/decisions.md) found title-first right about twice
as often as department-first, mostly because a specific title beats a
broad department name, so title now wins; department is the fallback
when the title matches nothing."""
return _classify_text(title or "") or _classify_text(department or "") or ""
def is_known_target(target: str) -> bool:
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
`slug_ok` accepts. What the target validator refuses: `app companies add`
and the pull job both call this before touching the network."""
host, sep, slug = target.partition(":")
module = readers.HOSTS.get(host)
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
def careers_url(target: str) -> str:
"""The company's own careers page for `host:slug`, or "" for an unknown
host. What `app companies add` stores alongside the board."""
host, _, slug = target.partition(":")
module = readers.HOSTS.get(host)
return module.careers_url(slug) if module is not None else ""
def _fetch_rows(client: httpx.Client, target: str,
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
"""One board's listing, read and parsed, unfiltered unless `ask` says:
its rows and whether they are the whole board. Raises on any failure.
No sweep and no state: what `fetch_board` and the host tests read."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":")
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
limits=_limits())
return listing.rows, listing.complete
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str], failed: set[str]
) -> tuple[list[Row], set[str], set[str]]:
"""`_runner.read_details` for one board, with `rows` as the whole listing
when `complete`."""
host, _, slug = target.partition(":")
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
rows, detailed, failed, listed=listed, limits=_limits())
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no state, just "does
this board answer, what does it list, and is that all of it". For `app
companies check`. The listing only, unfiltered, no detail reads.
Raises the same way `_fetch_rows` does."""
with httpx.Client() as client:
return _fetch_rows(client, target)
#: Pages read at most to size a board whose host states no total on page 1:
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
SIZE_PAGES = 100
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
"""How many roles board `target` lists, unfiltered and without terms: a
whole host's listing, the total a `complete="stated"` host gives on page
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
as `_fetch_rows` does, so a board that does not answer is refused."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
if client is None:
with httpx.Client() as own:
return board_size(target, client=own)
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
if module.READS.listing == "paged" and module.READS.complete == "stated":
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int):
return total
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
def keep_size(state: Any, target: str, size: int, *, today: str = "",
reset: bool = True) -> None:
"""Keep board `target`'s size and the day it was checked (ISO,
`today`, else today in UTC). With `reset`, the default and what a
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
the next read of a board that really shrank is taken as it is instead of
failing as suspect; the plugin sizing a board as it reads it passes
False, since its count is the read it just stamped."""
state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
if reset:
state.pop(f"ats:count:{target}", None)
failure_reason = _runner.failure_reason
# -------------------------------------------------------------- the plugin
class ATSSource(Source):
name = "ats"
description = f"Company job boards: {', '.join(HOSTS.values())}."
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
#: company, "params": {...}}`: `url` carries the `host:slug` target,
#: `label` the company name and `params` the board's filter.
subscription_field = "boards"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="ATS boards",
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
# run on its own (a test, a dry run) makes one up.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
yield from self._board(client, target, label, run_marker)
def _board(self, client: httpx.Client, target: str, label: str,
run_marker: str) -> Iterator[Item]:
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask, unsized, remote = self._ask(client, target, module)
stale_days = self.options.get("stale_days")
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits(int(stale_days) if stale_days else None),
remote=remote)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
def compound(source_id: str) -> str:
# Matches `roles.source_id`, which `Store.mark_closed` compares.
return f"{host}:{slug}:{source_id}"
# A host with `detail` states a role's text on the role's own page,
# which the `fetch_text` stage reads after scoring (`read_board_details`),
# never the pull. Its rows go out without a description, so the text
# a role's page gave stays, and a role read on its own page goes out
# without the listing's place and remote as well, so what its page
# said stays too (see `_runner.read_details`).
rows = listing.rows
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(module, "detail"):
detailed, failed = (_runner.id_set(self.state, key)
for key in detail_keys(target)[:2])
rows = [row | {"description": ""}
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
for row in rows]
if listing.complete and not listing.narrowed:
listed = {compound(i) for i in listing.ids}
detail_sets = (detailed & listed, failed & listed)
for row in rows:
yield item_of(row, target, label)
# Stamped only once every row went out, so a board that blew up half
# way through is neither marked ok nor has a partial id list treated
# as the whole of it, and it stays failing until then: a row that
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
self.state.pop(failing, None)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids), reset=False)
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
if key in self.state:
self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str,
module: Any) -> tuple[readers.Ask, bool, bool]:
"""What `target` is asked for, from the reach its batch handed it and
its params, whether it still needs a size, and whether that reach
and params want remote roles (`_runner.remote_wanted`, which the
runner compares with what it could ask). A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read,
once it is complete."""
given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"):
return (_runner.ask_for(module, params, given, keys=keys), False,
_runner.remote_wanted(module, params, given))
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
ask = _runner.ask_for(module, params, reach, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
return ask, True, _runner.remote_wanted(module, params, reach)
size = board_size(target, client=client)
keep_size(self.state, target, size, reset=False)
if size < int(given.get("small") or 0):
reach = None
ask = _runner.ask_for(module, params, None, keys=keys)
return ask, False, _runner.remote_wanted(module, params, reach)
# ------------------------------------------------------ the per-role reads
#: What a pull withholds for a role already read on its own page: the
#: listing's place and remote, which would replace what the page stated.
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
#: Days before a role whose own page gave no text is read again: an Oracle
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
#: 410 counts the same, since the listing closes a role that is gone.
EMPTY_RETRY_DAYS = 7
def detail_keys(target: str) -> tuple[str, str, str]:
"""The state keys of one board's per-role reads: the ids read, the ids
whose read failed, and `{id: date}` of the reads that gave no text."""
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
f"ats:detailempty:{target}")
def _dates(state: Any, key: str) -> dict[str, str]:
try:
value = json.loads(state.get(key) or "{}")
except (TypeError, ValueError):
return {}
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
on their own page, in the order given: never read, failed, or read with
no text at least `retry_days` ago. A role read before its date was kept
counts as read that long ago."""
detailed = _runner.id_set(state, detail_keys(target)[0])
empty = _dates(state, detail_keys(target)[2])
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
timespec="seconds")
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
cap: int | None = None, now: datetime | None = None,
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
"""Read `rows` of one board on their own pages, in the order given with
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
(`detail_keys`). The rows are the stored roles without text, each with
its host's own `source_id`. Returns the rows whose page filled a field,
each merged with what it filled.
A read that gave no text is kept with its date and read again once it is
`retry_days` old (`detail_due`). The sets are read again just before they
are written and changed by what this read did, so a pull that dropped ids
meanwhile keeps its drop."""
host, _, slug = target.partition(":")
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
keys = detail_keys(target)
now = now or datetime.now(UTC)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
retry_days=retry_days))
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
# Roles whose read failed go after the rest, so they cannot starve them.
wanted = sorted((row for row in rows if compound(row) in due),
key=lambda row: compound(row) in failed)
wanted = wanted[:DETAIL_CAP if cap is None else cap]
if not wanted:
return []
again = {compound(row) for row in wanted} & detailed
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
out, done, failing = _runner.read_details(
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
limits=limits)
read = done - (detailed - again)
filled: list[Row] = []
empty = _dates(state, keys[2])
for before, after in zip(wanted, out, strict=True):
key = compound(before)
if key in read:
if after.get("description"):
empty.pop(key, None)
else:
empty[key] = now.isoformat(timespec="seconds")
if after != before | {"description": ""}:
filled.append(after)
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
state[keys[0]] = json.dumps(sorted(detailed_now))
state[keys[1]] = json.dumps(sorted(failed_now))
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
return filled
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
"""One role of board or search `target` read on its own page: what the
reader's `detail` returns, for the Fetch button. No state is kept and no
field is filtered. Raises what `detail` raises, and ValueError for a
target whose reader has none."""
host, _, slug = target.partition(":")
module = reader_of(target)
if module is None or not hasattr(module, "detail"):
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
return module.detail(_getter(client, host, slug), slug, row)
def item_of(row: Row, target: str, label: str) -> Item:
"""One row of board `target` as the `Item` the store takes: its id
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
date is dated now, as `Item` itself would date it. A search's row is
stored as `plugins/search.py` stores it: `provider:id`, the employer from
`company` and `label` the search's own."""
host, _, slug = target.partition(":")
description = row.get("description") or ""
if host not in readers.HOSTS:
return Item(
source_id=f"{host}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
"search": target},
)
return Item(
source_id=f"{host}:{slug}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or datetime.now(UTC),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": label,
"function": classify(row.get("department") or "", row["title"]),
"board": host,
"slug": slug,
},
)