Files
jobboard/app/jobs/enrich.py
T

189 lines
8.1 KiB
Python

"""`enrich`: the best roles without text, read on their own pages.
Some hosts list a role without its text and state it on the role's own page
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
with a `detail`, and a search provider such as Jobbank). The pull reads
listings only; this stage reads those pages after scoring, so the calls go
where the ranking is. A saved search whose reader has `detail` is treated as
a board under its target `provider:name`: the same bar, budgets and state
keys, and its failing record is `search:failing:<target>`.
1. `enrich_waiting` hands over the open roles without text that an enabled
agent with a prompt ranks ungated among its best `enrich.top` and at
`enrich.min_score` or above, best first by the highest such score. A role
below that bar for every agent gets no call.
2. They are taken in that order, those whose read failed last, on boards and
searches whose reader has `detail`, at most `enrich.per_board` from one
board and `enrich.per_run` in all, leaving out the roles already read,
unless their read failed or gave no text `enrich.retry_days` ago
(`plugins.ats.detail_due`).
3. One batch per board or search, on the `http` pool, reads them
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
what each page filled. `Store.upsert_role` drops the vector of a role
whose text improved, so the next `embed` encodes those roles only.
In the pull cascade `embed` and `score` follow; `cascade.step` passes over
both when no role was filled. The same stage follows a rescore after an agent
is saved or a company's affinity changes (`cascade.enrich_after`), never after
a vote. Whether it runs is `stages.stage_on(config, store, "enrich")`.
"""
from __future__ import annotations
import json
import logging
from collections import defaultdict
from datetime import datetime
from typing import Any
import httpx
from app.config import Config
from app.jobs import stages
from app.models import Subscription
from app.storage import Store
from pipeline import NULL, Batch, RunContext, Tally, Task, job
log = logging.getLogger(__name__)
KIND = "enrich_batch"
SETTLE = "app.jobs.enrich:settle"
def _ats() -> Any:
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
from plugins import ats
return ats
def _native(found: dict[str, Any], sub: Subscription) -> str:
"""The host's own id of a stored role: a board's roles are stored as
`host:slug:id`, a search's as `provider:id`."""
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
return str(found["source_id"]).removeprefix(f"{prefix}:")
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
"""A stored role back as the row its reader listed: the stored `raw`
with the host's own id, the title, the link and the date."""
try:
raw = json.loads(found.get("raw") or "{}")
except ValueError:
raw = {}
raw = raw if isinstance(raw, dict) else {}
try:
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
except ValueError:
posted = None
return raw | {"source_id": _native(found, sub),
"title": found["title"], "url": found["url"], "description": "",
"posted": posted}
def _readable(store: Store) -> dict[int, Subscription]:
"""The fetchable boards and searches whose reader has `detail`, by
subscription id."""
ats = _ats()
return {sub.id: sub for source in ("ats", "search")
for sub in store.subscriptions(source=source, fetchable=True)
if hasattr(ats.reader_of(sub.target), "detail")}
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
"""`(target, uids)` per board or search, best first, within the budgets:
the roles `enrich_waiting` ranks, on fetchable boards and searches whose
reader has `detail` and that are not failing, and due for a read."""
settings = config.enrich
ats = _ats()
subs = _readable(store)
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
for found in store.enrich_waiting(settings.top, settings.min_score):
sub = subs.get(found["subscription_id"])
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
continue
# The id the board's per-role state keeps: `host:slug:id`.
found["compound"] = f"{sub.target}:{_native(found, sub)}"
by_board[sub.target].append(found)
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
retry_days=settings.retry_days))
for target, found in by_board.items()}
# Best first; roles whose read failed after every other, so a page that
# keeps failing cannot take the budget of the roles never tried.
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
for target in by_board}
ranked = sorted((f for found in by_board.values() for f in found),
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
-float(f["score"] or 0), f["uid"]))
picked: dict[str, list[str]] = {}
total = 0
for found in ranked:
target = subs[found["subscription_id"]].target
if found["compound"] not in due[target]:
continue
if total >= settings.per_run:
break
uids = picked.setdefault(target, [])
if len(uids) >= settings.per_board:
continue
uids.append(found["uid"])
total += 1
return list(picked.items())
def batches(config: Config, store: Store) -> list[Batch]:
"""The finder: one batch per board or search, nothing while the stage is
off."""
if not stages.stage_on(config, store, "enrich"):
return []
return [Batch({"target": target, "uids": uids},
label=f"Reading {len(uids)} roles on {target}")
for target, uids in plan(config, store)]
@job(KIND, "Reading roles on their own pages", resource="http")
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
context: RunContext = NULL) -> dict[str, Any]:
"""Read one board's or search's roles in `uids` on their own pages, in
that order, and store what each page filled. `filled` is how many roles
changed, `texts` how many of them now have text."""
ats = _ats()
subs = [s for s in _readable(store).values() if s.target == target]
if not subs or not uids:
return {"target": target, "filled": 0, "texts": 0}
sub = subs[0]
found = {f["uid"]: f for f in store.enrich_waiting(0, 0.0, uids)}
rows = [_row(found[uid], sub) for uid in uids if uid in found]
context.check()
with httpx.Client() as client:
filled = ats.read_board_details(store.state, target, rows, client=client,
cap=len(rows), retry_days=config.enrich.retry_days)
label = sub.company or sub.label
texts = 0
for row in filled:
item = ats.item_of(row, target, label)
item.source = sub.source
store.upsert_role(item, sub)
texts += bool(row.get("description"))
log.info("enrich: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
texts)
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
def filled(tally: Tally) -> int:
"""How many roles the batches behind `tally` filled."""
return sum(int(result.get("filled") or 0) for result in tally.results)
def settle(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's sum: roles filled, and how many of them now have text."""
return {"filled": filled(tally),
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
def enrich_task() -> Task:
return Task(kind=KIND, finder=batches, name="enrich", settle=SETTLE,
label="Reading the best roles on their own pages")