195 lines
8.3 KiB
Python
195 lines
8.3 KiB
Python
"""`fetch_text`: the roles in the cut without text, read on their own pages.
|
|
|
|
Some hosts list a role without its text and state it on the role's own page
|
|
only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest
|
|
with a `detail`, and a search provider such as Jobbank). The pull reads
|
|
listings only; this stage reads those pages after scoring, for the roles the
|
|
model pass will read, so each has its text first. A saved search whose
|
|
reader has `detail` is treated as a board under its target `provider:name`:
|
|
the same budgets and state keys, and its failing record is
|
|
`search:failing:<target>`. No model is asked.
|
|
|
|
1. `Store.fetch_text_waiting` hands over the open roles without text in the
|
|
cut of an enabled agent (`scores.cut`, see `app.jobs.score`), best first
|
|
by the highest final score. A role in no cut gets no call.
|
|
2. They are taken in that order, those whose read failed last, on boards and
|
|
searches whose reader has `detail`, at most `fetch_text.per_board` from
|
|
one board and `fetch_text.per_run` in all, leaving out the roles already
|
|
read, unless their read failed or gave no text `fetch_text.retry_days`
|
|
ago (`plugins.ats.detail_due`).
|
|
3. One batch per board or search, on the `http` pool, reads them
|
|
(`plugins.ats.read_board_details`, which keeps `ats:detailed:<target>`,
|
|
`ats:detailfailed:<target>` and `ats:detailempty:<target>`) and stores
|
|
what each page filled. `Store.upsert_role` drops the vector of a role
|
|
whose text improved, so the next `embed` encodes those roles only.
|
|
|
|
In the pull cascade `prep`, `embed` and `score` follow, so the filled roles
|
|
are embedded again and their levels and the cut recomputed; `cascade.step`
|
|
passes over the three when no role was filled. The same stage starts the
|
|
cascade that follows a rescore after an agent is saved or a company's
|
|
affinity or overrides change (`cascade.fetch_text_after`), never after a
|
|
vote. Whether it runs is `stages.stage_on(config, store, "fetch_text")`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.config import Config
|
|
from app.jobs import stages
|
|
from app.models import Subscription
|
|
from app.storage import Store
|
|
from pipeline import NULL, Batch, RunContext, Tally, Task, job
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
KIND = "fetch_text_batch"
|
|
SETTLE = "app.jobs.fetch_text:settle"
|
|
|
|
|
|
def _ats() -> Any:
|
|
"""`plugins.ats`, imported when needed, the way `pull.run_batch` does."""
|
|
from plugins import ats
|
|
|
|
return ats
|
|
|
|
|
|
def _native(found: dict[str, Any], sub: Subscription) -> str:
|
|
"""The host's own id of a stored role: a board's roles are stored as
|
|
`host:slug:id`, a search's as `provider:id`."""
|
|
prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0]
|
|
return str(found["source_id"]).removeprefix(f"{prefix}:")
|
|
|
|
|
|
def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]:
|
|
"""A stored role back as the row its reader listed: the stored `raw`
|
|
with the host's own id, the title, the link and the date."""
|
|
try:
|
|
raw = json.loads(found.get("raw") or "{}")
|
|
except ValueError:
|
|
raw = {}
|
|
raw = raw if isinstance(raw, dict) else {}
|
|
try:
|
|
posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or ""))
|
|
except ValueError:
|
|
posted = None
|
|
return raw | {"source_id": _native(found, sub),
|
|
"title": found["title"], "url": found["url"], "description": "",
|
|
"posted": posted}
|
|
|
|
|
|
def _readable(store: Store) -> dict[int, Subscription]:
|
|
"""The fetchable boards and searches whose reader has `detail`, by
|
|
subscription id."""
|
|
ats = _ats()
|
|
return {sub.id: sub for source in ("ats", "search")
|
|
for sub in store.subscriptions(source=source, fetchable=True)
|
|
if hasattr(ats.reader_of(sub.target), "detail")}
|
|
|
|
|
|
def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]:
|
|
"""`(target, uids)` per board or search, best first, within the budgets:
|
|
the roles `fetch_text_waiting` hands over, on fetchable boards and
|
|
searches whose reader has `detail` and that are not failing, and due for
|
|
a read."""
|
|
settings = config.fetch_text
|
|
ats = _ats()
|
|
subs = _readable(store)
|
|
by_board: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for found in store.fetch_text_waiting():
|
|
sub = subs.get(found["subscription_id"])
|
|
if sub is None or f"{sub.source}:failing:{sub.target}" in store.state:
|
|
continue
|
|
# The id the board's per-role state keeps: `host:slug:id`.
|
|
found["compound"] = f"{sub.target}:{_native(found, sub)}"
|
|
by_board[sub.target].append(found)
|
|
due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found],
|
|
retry_days=settings.retry_days))
|
|
for target, found in by_board.items()}
|
|
# Best first; roles whose read failed after every other, so a page that
|
|
# keeps failing cannot take the budget of the roles never tried.
|
|
failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1])
|
|
for target in by_board}
|
|
ranked = sorted((f for found in by_board.values() for f in found),
|
|
key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target],
|
|
-float(f["score"] or 0), f["uid"]))
|
|
picked: dict[str, list[str]] = {}
|
|
total = 0
|
|
for found in ranked:
|
|
target = subs[found["subscription_id"]].target
|
|
if found["compound"] not in due[target]:
|
|
continue
|
|
if total >= settings.per_run:
|
|
break
|
|
uids = picked.setdefault(target, [])
|
|
if len(uids) >= settings.per_board:
|
|
continue
|
|
uids.append(found["uid"])
|
|
total += 1
|
|
return list(picked.items())
|
|
|
|
|
|
def batches(config: Config, store: Store) -> list[Batch]:
|
|
"""The finder: one batch per board or search, nothing while the stage is
|
|
off."""
|
|
if not stages.stage_on(config, store, "fetch_text"):
|
|
return []
|
|
return [Batch({"target": target, "uids": uids},
|
|
label=f"Reading {len(uids)} roles on {target}")
|
|
for target, uids in plan(config, store)]
|
|
|
|
|
|
@job(KIND, "Reading roles on their own pages", resource="http")
|
|
def run_batch(config: Config, store: Store, *, target: str, uids: list[str],
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Read one board's or search's roles in `uids` on their own pages, in
|
|
that order, and store what each page filled. `filled` is how many roles
|
|
changed, `texts` how many of them now have text."""
|
|
ats = _ats()
|
|
subs = [s for s in _readable(store).values() if s.target == target]
|
|
if not subs or not uids:
|
|
return {"target": target, "filled": 0, "texts": 0}
|
|
sub = subs[0]
|
|
found = {f["uid"]: f for f in store.fetch_text_waiting(uids)}
|
|
rows = [_row(found[uid], sub) for uid in uids if uid in found]
|
|
context.check()
|
|
with httpx.Client() as client:
|
|
filled = ats.read_board_details(store.state, target, rows, client=client,
|
|
cap=len(rows), retry_days=config.fetch_text.retry_days)
|
|
label = sub.company or sub.label
|
|
texts = 0
|
|
for row in filled:
|
|
item = ats.item_of(row, target, label)
|
|
item.source = sub.source
|
|
store.upsert_role(item, sub)
|
|
texts += bool(row.get("description"))
|
|
log.info("fetch_text: %s: %d read, %d filled, %d with text", target, len(rows), len(filled),
|
|
texts)
|
|
return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts}
|
|
|
|
|
|
def filled(tally: Tally) -> int:
|
|
"""How many roles the batches behind `tally` filled."""
|
|
return sum(int(result.get("filled") or 0) for result in tally.results)
|
|
|
|
|
|
def settle(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's sum: roles filled, and how many of them now have text."""
|
|
return {"filled": filled(tally),
|
|
"texts": sum(int(result.get("texts") or 0) for result in tally.results)}
|
|
|
|
|
|
#: What the task and the cascade's job are called on the Runs page.
|
|
LABEL = "Fetching the text of the roles in the cut"
|
|
|
|
|
|
def fetch_text_task() -> Task:
|
|
return Task(kind=KIND, finder=batches, name="fetch_text", settle=SETTLE, label=LABEL)
|