646 lines
32 KiB
Python
646 lines
32 KiB
Python
"""ATS boards: a company's own careers page on one of the hosts in
|
|
`plugins/readers/`, one module per host.
|
|
|
|
Every one of these hosts serves a company's own list of roles as public,
|
|
unauthenticated JSON, XML or RSS, and a role that drops off the board is
|
|
*closed*, which is the cheapest staleness check there is. What a host says
|
|
and how its feed reads is its module's, what every reader shares is
|
|
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
|
|
the classifier, and the names the app and the tests reach for on `ats`.
|
|
|
|
The allow-list and the classifier are ported from `openings.go` and
|
|
`cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
|
its slug is. Every board is read independently; one that fails is logged,
|
|
recorded as failing under `ats:failing:<target>`, and skipped, and so is one
|
|
whose read lists under a tenth of what it listed last time (`_runner.read`'s
|
|
guard), until it has failed that way for the pull's `stale_days`. A board's
|
|
filter is the `reach` option the pull hands it (the agents', see
|
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
|
["DK"], "remote": true}`) win, narrowed to what its reader declares
|
|
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
|
|
the whole board: it prunes no per-role state.
|
|
|
|
A board's size (`board_size`) is set once, when the board is added
|
|
(`app.jobs.sources.set_board`) or rechecked, and kept under
|
|
`ats:size:<target>` with the day it was checked under
|
|
`ats:sizechecked:<target>` (`keep_size`, which also resets the guard's count,
|
|
so a Recheck accepts a board that shrank); the reach reads it to leave small
|
|
boards whole. A pull does not rewrite it. A board with no size yet is sized
|
|
in its batch before it is read: its reach then says `unsized` and the size
|
|
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
|
|
whose reader cannot narrow that reach is read whole anyway, and sized by
|
|
that read once it is complete, rather than by a second request.
|
|
|
|
A pull reads listings only. A host that states a role's text on the role's
|
|
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
|
|
scoring, best roles first, through `read_board_details`. A search provider
|
|
whose reader has `detail` (Jobbank) is read the same way, under its search
|
|
target `provider:name`; `read_detail` is the one read the Fetch button makes.
|
|
|
|
The limits below are read when a board is read, so a test that replaces one
|
|
on this module reaches the runner.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
|
|
)
|
|
|
|
try:
|
|
from plugins import readers
|
|
except ModuleNotFoundError:
|
|
# `desk.sources.registry` loads this file by its path, under a name of its
|
|
# own, and the app puts the repo root on `sys.path` before it does; a
|
|
# registry run without the app (a script, another working directory)
|
|
# does not, and the host modules are found as the package beside this
|
|
# file. So the root goes on the path then, and only then.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
from plugins.readers import _runner
|
|
from plugins.readers._shared import Get, Row
|
|
|
|
# The names the parsers had here before they moved to their host modules,
|
|
# kept for the callers and the tests that still reach for them on `ats`.
|
|
from plugins.readers._shared import ( # noqa: F401
|
|
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
|
)
|
|
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
|
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
|
|
MAX_BYTES = _runner.MAX_BYTES
|
|
TIMEOUTS = _runner.TIMEOUTS
|
|
DEADLINE = _runner.DEADLINE
|
|
MAX_PAGES = _runner.MAX_PAGES
|
|
PAGE_DELAY = _runner.PAGE_DELAY
|
|
DETAIL_CAP = _runner.DETAIL_CAP
|
|
DETAIL_FIELDS = _runner.DETAIL_FIELDS
|
|
|
|
#: The runner's errors, under the names they had here.
|
|
_Refused = _runner.Refused
|
|
_TooLarge = _runner.TooLarge
|
|
_local_host = _runner.local_host
|
|
|
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
|
#: the order a select offers them. What the Sources page's host select is
|
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
|
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
|
|
|
#: host -> parser, for the hosts whose module parses a whole body.
|
|
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
|
|
|
|
|
def _limits(stale_days: int | None = None) -> _runner.Limits:
|
|
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP,
|
|
stale_days=stale_days)
|
|
|
|
|
|
def reader_of(target: str) -> ModuleType | None:
|
|
"""The reader of a board (`host:slug`) or of a search (`provider:name`),
|
|
None for a name that is neither. The two never share a name."""
|
|
host = target.partition(":")[0]
|
|
return readers.HOSTS.get(host) or readers.PROVIDERS.get(host)
|
|
|
|
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
"""The hosts a request or a redirect may land on for one board or search:
|
|
what its module's `allowed` says. Anything else is refused, at the first
|
|
hop and at every redirect."""
|
|
module = reader_of(host)
|
|
return module.allowed(slug) if module is not None else lambda h: False
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
|
|
`DEADLINE`."""
|
|
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
|
|
payload=payload, headers=headers, max_bytes=MAX_BYTES,
|
|
deadline=DEADLINE, name=host)
|
|
|
|
|
|
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
|
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
|
is looked up when a request is made, so a test that replaces `ats._get`
|
|
reaches every host; `payload` and `headers` are passed on only when
|
|
given, which keeps a replacement that predates them working."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
extra: dict[str, Any] = {}
|
|
if payload is not None:
|
|
extra["payload"] = payload
|
|
if headers:
|
|
extra["headers"] = headers
|
|
return _get(client, url, host, slug, method=method, **extra)
|
|
return get
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
|
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
|
and the pull job both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return module.careers_url(slug) if module is not None else ""
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str,
|
|
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
|
|
"""One board's listing, read and parsed, unfiltered unless `ask` says:
|
|
its rows and whether they are the whole board. Raises on any failure.
|
|
No sweep and no state: what `fetch_board` and the host tests read."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
|
|
limits=_limits())
|
|
return listing.rows, listing.complete
|
|
|
|
|
|
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
|
detailed: set[str], failed: set[str]
|
|
) -> tuple[list[Row], set[str], set[str]]:
|
|
"""`_runner.read_details` for one board, with `rows` as the whole listing
|
|
when `complete`."""
|
|
host, _, slug = target.partition(":")
|
|
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
|
|
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
|
|
rows, detailed, failed, listed=listed, limits=_limits())
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
|
this board answer, what does it list, and is that all of it". For `app
|
|
companies check`. The listing only, unfiltered, no detail reads.
|
|
Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
#: Pages read at most to size a board whose host states no total on page 1:
|
|
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
|
|
SIZE_PAGES = 100
|
|
|
|
|
|
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
|
|
"""How many roles board `target` lists, unfiltered and without terms: a
|
|
whole host's listing, the total a `complete="stated"` host gives on page
|
|
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
|
|
as `_fetch_rows` does, so a board that does not answer is refused."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
if client is None:
|
|
with httpx.Client() as own:
|
|
return board_size(target, client=own)
|
|
host, _, slug = target.partition(":")
|
|
module = readers.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
if module.READS.listing == "paged" and module.READS.complete == "stated":
|
|
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
|
|
if isinstance(total, int):
|
|
return total
|
|
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
|
|
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
|
|
|
|
|
|
def keep_size(state: Any, target: str, size: int, *, today: str = "",
|
|
reset: bool = True) -> None:
|
|
"""Keep board `target`'s size and the day it was checked (ISO,
|
|
`today`, else today in UTC). With `reset`, the default and what a
|
|
Recheck does, the 10% guard's count goes too (`ats:count:<target>`), so
|
|
the next read of a board that really shrank is taken as it is instead of
|
|
failing as suspect; the plugin sizing a board as it reads it passes
|
|
False, since its count is the read it just stamped."""
|
|
state[f"ats:size:{target}"] = str(size)
|
|
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
|
|
if reset:
|
|
state.pop(f"ats:count:{target}", None)
|
|
|
|
|
|
failure_reason = _runner.failure_reason
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company, "params": {...}}`: `url` carries the `host:slug` target,
|
|
#: `label` the company name and `params` the board's filter.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
|
|
# run on its own (a test, a dry run) makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
yield from self._board(client, target, label, run_marker)
|
|
|
|
def _board(self, client: httpx.Client, target: str, label: str,
|
|
run_marker: str) -> Iterator[Item]:
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
module = readers.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
ask, unsized, remote = self._ask(client, target, module)
|
|
stale_days = self.options.get("stale_days")
|
|
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
|
limits=_limits(int(stale_days) if stale_days else None),
|
|
remote=remote)
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
_runner.record_failure(self.state, failing, exc)
|
|
return
|
|
|
|
def compound(source_id: str) -> str:
|
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
|
return f"{host}:{slug}:{source_id}"
|
|
|
|
# A host with `detail` states a role's text on the role's own page,
|
|
# which the `enrich` stage reads after scoring (`read_board_details`),
|
|
# never the pull. Its rows go out without a description, so the text
|
|
# a role's page gave stays, and a role read on its own page goes out
|
|
# without the listing's place and remote as well, so what its page
|
|
# said stays too (see `_runner.read_details`).
|
|
rows = listing.rows
|
|
detail_sets: tuple[set[str], set[str]] | None = None
|
|
if hasattr(module, "detail"):
|
|
detailed, failed = (_runner.id_set(self.state, key)
|
|
for key in detail_keys(target)[:2])
|
|
rows = [row | {"description": ""}
|
|
| (_KEPT_FROM_DETAIL if compound(row["source_id"]) in detailed else {})
|
|
for row in rows]
|
|
if listing.complete and not listing.narrowed:
|
|
listed = {compound(i) for i in listing.ids}
|
|
detail_sets = (detailed & listed, failed & listed)
|
|
|
|
for row in rows:
|
|
yield item_of(row, target, label)
|
|
# Stamped only once every row went out, so a board that blew up half
|
|
# way through is neither marked ok nor has a partial id list treated
|
|
# as the whole of it, and it stays failing until then: a row that
|
|
# fails to store (`app.jobs.pull.run_batch`) leaves the record in place.
|
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
|
[compound(i) for i in listing.ids])
|
|
self.state.pop(failing, None)
|
|
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
|
|
keep_size(self.state, target, len(listing.ids), reset=False)
|
|
if detail_sets is not None:
|
|
# A complete listing drops the ids it no longer lists.
|
|
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
|
|
if key in self.state:
|
|
self.state[key] = json.dumps(sorted(ids))
|
|
|
|
def _ask(self, client: httpx.Client, target: str,
|
|
module: Any) -> tuple[readers.Ask, bool, bool]:
|
|
"""What `target` is asked for, from the reach its batch handed it and
|
|
its params, whether it still needs a size, and whether that reach
|
|
and params want remote roles (`_runner.remote_wanted`, which the
|
|
runner compares with what it could ask). A board with no size
|
|
yet (`unsized` in its reach) whose reach would narrow the read is
|
|
sized first (`board_size`, kept by `keep_size`), and read whole under
|
|
`small` roles. One whose read is whole anyway is sized by that read,
|
|
once it is complete."""
|
|
given = self.options.get("reach")
|
|
params, keys = self.params_for(target), self.options.get("keys")
|
|
if not isinstance(given, dict) or not given.get("unsized"):
|
|
return (_runner.ask_for(module, params, given, keys=keys), False,
|
|
_runner.remote_wanted(module, params, given))
|
|
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} or None
|
|
ask = _runner.ask_for(module, params, reach, keys=keys)
|
|
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
|
|
return ask, True, _runner.remote_wanted(module, params, reach)
|
|
size = board_size(target, client=client)
|
|
keep_size(self.state, target, size, reset=False)
|
|
if size < int(given.get("small") or 0):
|
|
reach = None
|
|
ask = _runner.ask_for(module, params, None, keys=keys)
|
|
return ask, False, _runner.remote_wanted(module, params, reach)
|
|
|
|
|
|
# ------------------------------------------------------ the per-role reads
|
|
|
|
#: What a pull withholds for a role already read on its own page: the
|
|
#: listing's place and remote, which would replace what the page stated.
|
|
_KEPT_FROM_DETAIL: dict[str, Any] = {"location": "", "places": [], "remote": None}
|
|
|
|
#: Days before a role whose own page gave no text is read again: an Oracle
|
|
#: requisition not yet published, an empty SuccessFactors block. A 404 or a
|
|
#: 410 counts the same, since the listing closes a role that is gone.
|
|
EMPTY_RETRY_DAYS = 7
|
|
|
|
|
|
def detail_keys(target: str) -> tuple[str, str, str]:
|
|
"""The state keys of one board's per-role reads: the ids read, the ids
|
|
whose read failed, and `{id: date}` of the reads that gave no text."""
|
|
return (f"ats:detailed:{target}", f"ats:detailfailed:{target}",
|
|
f"ats:detailempty:{target}")
|
|
|
|
|
|
def _dates(state: Any, key: str) -> dict[str, str]:
|
|
try:
|
|
value = json.loads(state.get(key) or "{}")
|
|
except (TypeError, ValueError):
|
|
return {}
|
|
return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {}
|
|
|
|
|
|
def detail_due(state: Any, target: str, ids: list[str], *, now: datetime | None = None,
|
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[str]:
|
|
"""Of `ids` (compound, `host:slug:id`, roles without text), those to read
|
|
on their own page, in the order given: never read, failed, or read with
|
|
no text at least `retry_days` ago. A role read before its date was kept
|
|
counts as read that long ago."""
|
|
detailed = _runner.id_set(state, detail_keys(target)[0])
|
|
empty = _dates(state, detail_keys(target)[2])
|
|
cutoff = ((now or datetime.now(UTC)) - timedelta(days=retry_days)).isoformat(
|
|
timespec="seconds")
|
|
return [i for i in ids if i not in detailed or empty.get(i, "") <= cutoff]
|
|
|
|
|
|
def read_board_details(state: Any, target: str, rows: list[Row], *, client: httpx.Client,
|
|
cap: int | None = None, now: datetime | None = None,
|
|
retry_days: int = EMPTY_RETRY_DAYS) -> list[Row]:
|
|
"""Read `rows` of one board on their own pages, in the order given with
|
|
the roles whose read failed last, at most `cap` (`DETAIL_CAP`), and keep the board's per-role state
|
|
(`detail_keys`). The rows are the stored roles without text, each with
|
|
its host's own `source_id`. Returns the rows whose page filled a field,
|
|
each merged with what it filled.
|
|
|
|
A read that gave no text is kept with its date and read again once it is
|
|
`retry_days` old (`detail_due`). The sets are read again just before they
|
|
are written and changed by what this read did, so a pull that dropped ids
|
|
meanwhile keeps its drop."""
|
|
host, _, slug = target.partition(":")
|
|
module = reader_of(target)
|
|
if module is None or not hasattr(module, "detail"):
|
|
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
|
keys = detail_keys(target)
|
|
now = now or datetime.now(UTC)
|
|
|
|
def compound(row: Row) -> str:
|
|
return f"{host}:{slug}:{row['source_id']}"
|
|
|
|
due = set(detail_due(state, target, [compound(r) for r in rows], now=now,
|
|
retry_days=retry_days))
|
|
detailed, failed = (_runner.id_set(state, key) for key in keys[:2])
|
|
# Roles whose read failed go after the rest, so they cannot starve them.
|
|
wanted = sorted((row for row in rows if compound(row) in due),
|
|
key=lambda row: compound(row) in failed)
|
|
wanted = wanted[:DETAIL_CAP if cap is None else cap]
|
|
if not wanted:
|
|
return []
|
|
again = {compound(row) for row in wanted} & detailed
|
|
limits = _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=len(wanted))
|
|
out, done, failing = _runner.read_details(
|
|
module, host, slug, _getter(client, host, slug), wanted, detailed - again, failed,
|
|
limits=limits)
|
|
read = done - (detailed - again)
|
|
filled: list[Row] = []
|
|
empty = _dates(state, keys[2])
|
|
for before, after in zip(wanted, out, strict=True):
|
|
key = compound(before)
|
|
if key in read:
|
|
if after.get("description"):
|
|
empty.pop(key, None)
|
|
else:
|
|
empty[key] = now.isoformat(timespec="seconds")
|
|
if after != before | {"description": ""}:
|
|
filled.append(after)
|
|
|
|
detailed_now = (_runner.id_set(state, keys[0]) - again) | read
|
|
failed_now = (_runner.id_set(state, keys[1]) | (failing - failed)) - read
|
|
state[keys[0]] = json.dumps(sorted(detailed_now))
|
|
state[keys[1]] = json.dumps(sorted(failed_now))
|
|
state[keys[2]] = json.dumps({k: v for k, v in sorted(empty.items()) if k in detailed_now})
|
|
return filled
|
|
|
|
|
|
def read_detail(client: httpx.Client, target: str, row: Row) -> dict[str, Any]:
|
|
"""One role of board or search `target` read on its own page: what the
|
|
reader's `detail` returns, for the Fetch button. No state is kept and no
|
|
field is filtered. Raises what `detail` raises, and ValueError for a
|
|
target whose reader has none."""
|
|
host, _, slug = target.partition(":")
|
|
module = reader_of(target)
|
|
if module is None or not hasattr(module, "detail"):
|
|
raise ValueError(f"{target!r} has no reader that reads a role on its own page")
|
|
return module.detail(_getter(client, host, slug), slug, row)
|
|
|
|
|
|
def item_of(row: Row, target: str, label: str) -> Item:
|
|
"""One row of board `target` as the `Item` the store takes: its id
|
|
compound, `host:slug:id`, as `roles.source_id` has it. A row without a
|
|
date is dated now, as `Item` itself would date it. A search's row is
|
|
stored as `plugins/search.py` stores it: `provider:id`, the employer from
|
|
`company` and `label` the search's own."""
|
|
host, _, slug = target.partition(":")
|
|
description = row.get("description") or ""
|
|
if host not in readers.HOSTS:
|
|
return Item(
|
|
source_id=f"{host}:{row['source_id']}",
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted") or datetime.now(UTC),
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {"company": row.get("company") or "", "provider": host,
|
|
"search": target},
|
|
)
|
|
return Item(
|
|
source_id=f"{host}:{slug}:{row['source_id']}",
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted") or datetime.now(UTC),
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {
|
|
"company": label,
|
|
"function": classify(row.get("department") or "", row["title"]),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|