Migration 21 adds companies.all_roles, tags every favourite of any agent and each company whose board had params {"reach": "all"} (logged per board, the param removed). reach.plan reads a tagged company's boards with the union of the enabled agents' require reach and the base reach; favourite no longer widens the fetch. POST /api/companies/{key}/all-roles sets it and queues a pull when on. The Companies list, the reader and the company strip draw the toggle beside the affinity buttons, the list filters on it, and the board's reach editor drops its All roles option.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
528 lines
25 KiB
Python
528 lines
25 KiB
Python
"""ATS boards: a company's own careers page on one of the hosts in
|
|
`plugins/readers/`, one module per host.
|
|
|
|
Every one of these hosts serves a company's own list of roles as public,
|
|
unauthenticated JSON, XML or RSS, and a role that drops off the board is
|
|
*closed*, which is the cheapest staleness check there is. What a host says
|
|
and how its feed reads is its module's, what every reader shares is
|
|
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
|
|
the classifier, and the names the app and the tests reach for on `ats`.
|
|
|
|
The allow-list and the classifier are ported from `openings.go` and
|
|
`cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
|
its slug is. Every board is read independently; one that fails is logged,
|
|
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
|
filter is the `reach` option the pull hands it (the agents', see
|
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
|
(`_runner.ask_for`). An unfiltered read records the board's size under
|
|
`ats:size:<target>`, which the reach reads to leave small boards whole.
|
|
|
|
A pull reads listings only. A host that states a role's text on the role's
|
|
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
|
scoring, best roles first, through `read_board_details`.
|
|
|
|
The limits below are read when a board is read, so a test that replaces one
|
|
on this module reaches the runner.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
|
|
)
|
|
|
|
try:
|
|
from plugins import readers
|
|
except ModuleNotFoundError:
|
|
# `desk.sources.registry` loads this file by its path, under a name of its
|
|
# own, and the app puts the repo root on `sys.path` before it does; a
|
|
# registry run without the app (a script, another working directory)
|
|
# does not, and the host modules are found as the package beside this
|
|
# file. So the root goes on the path then, and only then.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
from plugins.readers import _runner
|
|
from plugins.readers._shared import Get, Row
|
|
|
|
# The names the parsers had here before they moved to their host modules,
|
|
# kept for the callers and the tests that still reach for them on `ats`.
|
|
from plugins.readers._shared import ( # noqa: F401
|
|
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
|
)
|
|
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
|
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
|
|
MAX_BYTES = _runner.MAX_BYTES
|
|
TIMEOUTS = _runner.TIMEOUTS
|
|
DEADLINE = _runner.DEADLINE
|
|
MAX_PAGES = _runner.MAX_PAGES
|
|
PAGE_DELAY = _runner.PAGE_DELAY
|
|
DETAIL_CAP = _runner.DETAIL_CAP
|
|
DETAIL_FIELDS = _runner.DETAIL_FIELDS
|
|
|
|
#: The runner's errors, under the names they had here.
|
|
_Refused = _runner.Refused
|
|
_TooLarge = _runner.TooLarge
|
|
_local_host = _runner.local_host
|
|
|
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
|
#: the order a select offers them. What the Sources page's host select is
|
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
|
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
|
|
|
#: host -> parser, for the hosts whose module parses a whole body.
|
|
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
|
|
|
|
|
def _limits() -> _runner.Limits:
|
|
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
|
|
|
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
"""The hosts a request or a redirect may land on for one board: what its
|
|
host module's `allowed` says. Anything else is refused, at the first hop
|
|
and at every redirect."""
|
|
module = readers.HOSTS.get(host)
|
|
return module.allowed(slug) if module is not None else lambda h: False
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
|
|
`DEADLINE`."""
|
|
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
|
|
payload=payload, headers=headers, max_bytes=MAX_BYTES,
|
|
deadline=DEADLINE, name=host)
|
|
|
|
|
|
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
|
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
|
is looked up when a request is made, so a test that replaces `ats._get`
|
|
reaches every host; `payload` and `headers` are passed on only when
|
|
given, which keeps a replacement that predates them working."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
extra: dict[str, Any] = {}
|
|
if payload is not None:
|
|
extra["payload"] = payload
|
|
if headers:
|
|
extra["headers"] = headers
|
|
return _get(client, url, host, slug, method=method, **extra)
|
|
return get
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
|
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
|
and the pull job both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return module.careers_url(slug) if module is not None else ""
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str,
|
|
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
|
|
"""One board's listing, read and parsed, unfiltered unless `ask` says:
|
|
its rows and whether they are the whole board. Raises on any failure.
|
|
No sweep and no state: what `fetch_board` and the host tests read."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
|
|
limits=_limits())
|
|
return listing.rows, listing.complete
|
|
|
|
|
|
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
|
detailed: set[str], failed: set[str]
|
|
) -> tuple[list[Row], set[str], set[str]]:
|
|
"""`_runner.read_details` for one board, with `rows` as the whole listing
|
|
when `complete`."""
|
|
host, _, slug = target.partition(":")
|
|
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
|
|
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
|
|
rows, detailed, failed, listed=listed, limits=_limits())
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
|
this board answer, what does it list, and is that all of it". For `app
|
|
companies add|check`. The listing only, unfiltered, no detail reads.
|
|
Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
failure_reason = _runner.failure_reason
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company, "params": {...}}`: `url` carries the `host:slug` target,
|
|
#: `label` the company name and `params` the board's filter.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
|
|
# run on its own (a test, a dry run) makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
yield from self._board(client, target, label, run_marker)
|
|
|
|
def _board(self, client: httpx.Client, target: str, label: str,
|
|
run_marker: str) -> Iterator[Item]:
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
module = readers.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
|
|
keys=self.options.get("keys"))
|
|
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
|
limits=_limits())
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
_runner.record_failure(self.state, failing, exc)
|
|
return
|
|
self.state.pop(failing, None)
|
|
|
|
def compound(source_id: str) -> str:
|
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
|
return f"{host}:{slug}:{source_id}"
|
|
|
|
# A host with `detail` states a role's text on the role's own page,
|
|
# which the `enrich` stage reads after scoring (`read_board_details`),
|
|
# never the pull. Its rows go out without a description, so the text
|
|
# a role's page gave stays, and a role read on its own page goes out
|
|
# without the listing's place and remote as well, so what its page
|
|
# said stays too (see `_runner.read_details`).
|
|
rows = listing.rows
|
|
detail_sets: tuple[set[str], set[str]] | None = None
|
|
if hasattr(module, "detail"):
|
|
detailed, failed = (_runner.id_set(self.state, key)
|
|
for key in detail_keys(target)[:2])
|
|
rows = [row | {"description": ""}
|
|
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
|
|
for row in rows]
|
|
if listing.complete:
|
|
listed = {compound(i) for i in listing.ids}
|
|
detail_sets = (detailed & listed, failed & listed)
|
|
|
|
for row in rows:
|
|
yield item_of(row, target, label)
|
|
# Stamped only once every row went out, so a board that blew up half
|
|
# way through is neither marked ok nor has a partial id list treated
|
|
# as the whole of it.
|
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
|
[compound(i) for i in listing.ids])
|
|
if not listing.filtered:
|
|
size = len(listing.ids)
|
|
if not listing.complete:
|
|
kept = self.state.get(f"ats:size:{target}", "")
|
|
size = max(size, int(kept) if kept.isdigit() else 0)
|
|
self.state[f"ats:size:{target}"] = str(size)
|
|
if detail_sets is not None:
|
|
# A complete listing drops the ids it no longer lists.
|
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
|
if key in self.state:
|
|
self.state[key] = json.dumps(sorted(ids))
|
|
|
|
|
|
# ------------------------------------------------------ the per-role reads
|
|
|
|
#: What a pull withholds for a role already read on its own page: the
|
|
#: listing's place and remote, which would replace what the page stated.
|
|
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
|
|
|
|
#: Days before a role whose own page gave no text is read again: an Oracle
|
|
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
|
|
#: 410 counts the same, since the listing closes a role that is gone.
|
|
EMPTY_RETRY_DAYS = 7
|
|
|
|
|
|
def detail_keys(target: str) -> tuple[str, str, str]:
|
|
"""The state keys of one board's per-role reads: the ids read, the ids
|
|
whose read failed, and `{id: date}` of the reads that gave no text."""
|
|
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
|
|
f"ats:detailempty:{target}")
|
|
|
|
|
|
def _dates(state: Any, key: str) -> dict[str, str]:
|
|
try:
|
|
value = json.loads(state.get(key) or "{}")
|
|
except (TypeError, ValueError):
|
|
return {}
|
|
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
|
|
|
|
|
|
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
|
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
|
|
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
|
|
on their own page, in the order given: never read, failed, or read with
|
|
no text at least `retry_days` ago. A role read before its date was kept
|
|
counts as read that long ago."""
|
|
detailed = _runner.id_set(state, detail_keys(target)[0])
|
|
empty = _dates(state, detail_keys(target)[2])
|
|
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
|
|
timespec="seconds")
|
|
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
|
|
|
|
|
|
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
|
|
cap: int | None = None, now: datetime | None = None,
|
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
|
|
"""Read `rows` of one board on their own pages, in the order given with
|
|
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
|
|
(`detail_keys`). The rows are the stored roles without text, each with
|
|
its host's own `source_id`. Returns the rows whose page filled a field,
|
|
each merged with what it filled.
|
|
|
|
A read that gave no text is kept with its date and read again once it is
|
|
`retry_days` old (`detail_due`). The sets are read again just before they
|
|
are written and changed by what this read did, so a pull that dropped ids
|
|
meanwhile keeps its drop."""
|
|
host, _, slug = target.partition(":")
|
|
module = readers.HOSTS[host]
|
|
keys = detail_keys(target)
|
|
now = now or datetime.now(UTC)
|
|
|
|
def compound(row: Row) -> str:
|
|
return f"{host}:{slug}:{row['source_id']}"
|
|
|
|
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
|
|
retry_days=retry_days))
|
|
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
|
|
# Roles whose read failed go after the rest, so they cannot starve them.
|
|
wanted = sorted((row for row in rows if compound(row) in due),
|
|
key=lambda row: compound(row) in failed)
|
|
wanted = wanted[:DETAIL_CAP if cap is None else cap]
|
|
if not wanted:
|
|
return []
|
|
again = {compound(row) for row in wanted} & detailed
|
|
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
|
|
out, done, failing = _runner.read_details(
|
|
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
|
|
limits=limits)
|
|
read = done - (detailed - again)
|
|
filled: list[Row] = []
|
|
empty = _dates(state, keys[2])
|
|
for before, after in zip(wanted, out, strict=True):
|
|
key = compound(before)
|
|
if key in read:
|
|
if after.get("description"):
|
|
empty.pop(key, None)
|
|
else:
|
|
empty[key] = now.isoformat(timespec="seconds")
|
|
if after != before | {"description": ""}:
|
|
filled.append(after)
|
|
|
|
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
|
|
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
|
|
state[keys[0]] = json.dumps(sorted(detailed_now))
|
|
state[keys[1]] = json.dumps(sorted(failed_now))
|
|
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
|
|
return filled
|
|
|
|
|
|
def item_of(row: Row, target: str, label: str) -> Item:
|
|
"""One row of board `target` as the `Item` the store takes: its id
|
|
compound, `host:slug:id`, as `roles.source_id` has it."""
|
|
host, _, slug = target.partition(":")
|
|
description = row.get("description") or ""
|
|
return Item(
|
|
source_id=f"{host}:{slug}:{row['source_id']}",
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted"),
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {
|
|
"company": label,
|
|
"function": classify(row.get("department") or "", row["title"]),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|