"""`fetch_text`: the roles in the cut without text, read on their own pages. Some hosts list a role without its text and state it on the role's own page only (`text="detail"`: Workday, SuccessFactors, Oracle, Phenom and the rest with a `detail`, and a search provider such as Jobbank). The pull reads listings only; this stage reads those pages after scoring, for the roles the model pass will read, so each has its text first. A saved search whose reader has `detail` is treated as a board under its target `provider:name`: the same budgets and state keys, and its failing record is `search:failing:`. No model is asked. 1. `Store.fetch_text_waiting` hands over the open roles without text in the cut of an enabled agent (`scores.cut`, see `app.jobs.score`), best first by the highest final score. A role in no cut gets no call. 2. They are taken in that order, those whose read failed last, on boards and searches whose reader has `detail`, at most `fetch_text.per_board` from one board and `fetch_text.per_run` in all, leaving out the roles already read, unless their read failed or gave no text `fetch_text.retry_days` ago (`plugins.ats.detail_due`). 3. One batch per board or search, on the `http` pool, reads them (`plugins.ats.read_board_details`, which keeps `ats:detailed:`, `ats:detailfailed:` and `ats:detailempty:`) and stores what each page filled. `Store.upsert_role` drops the vector of a role whose text improved, so the next `embed` encodes those roles only. In the pull cascade `prep`, `embed` and `score` follow, so the filled roles are embedded again and their levels and the cut recomputed; `cascade.step` passes over the three when no role was filled. The same stage starts the cascade that follows a rescore after an agent is saved or a company's affinity or overrides change (`cascade.fetch_text_after`), never after a vote. Whether it runs is `stages.stage_on(config, store, "fetch_text")`. """ from __future__ import annotations import json import logging from collections import defaultdict from datetime import datetime from typing import Any import httpx from app.config import Config from app.jobs import stages from app.models import Subscription from app.storage import Store from pipeline import NULL, Batch, RunContext, Tally, Task, job log = logging.getLogger(__name__) KIND = "fetch_text_batch" SETTLE = "app.jobs.fetch_text:settle" def _ats() -> Any: """`plugins.ats`, imported when needed, the way `pull.run_batch` does.""" from plugins import ats return ats def _native(found: dict[str, Any], sub: Subscription) -> str: """The host's own id of a stored role: a board's roles are stored as `host:slug:id`, a search's as `provider:id`.""" prefix = sub.target if sub.source == "ats" else sub.target.partition(":")[0] return str(found["source_id"]).removeprefix(f"{prefix}:") def _row(found: dict[str, Any], sub: Subscription) -> dict[str, Any]: """A stored role back as the row its reader listed: the stored `raw` with the host's own id, the title, the link and the date.""" try: raw = json.loads(found.get("raw") or "{}") except ValueError: raw = {} raw = raw if isinstance(raw, dict) else {} try: posted: datetime | None = datetime.fromisoformat(str(found.get("published_at") or "")) except ValueError: posted = None return raw | {"source_id": _native(found, sub), "title": found["title"], "url": found["url"], "description": "", "posted": posted} def _readable(store: Store) -> dict[int, Subscription]: """The fetchable boards and searches whose reader has `detail`, by subscription id.""" ats = _ats() return {sub.id: sub for source in ("ats", "search") for sub in store.subscriptions(source=source, fetchable=True) if hasattr(ats.reader_of(sub.target), "detail")} def plan(config: Config, store: Store) -> list[tuple[str, list[str]]]: """`(target, uids)` per board or search, best first, within the budgets: the roles `fetch_text_waiting` hands over, on fetchable boards and searches whose reader has `detail` and that are not failing, and due for a read.""" settings = config.fetch_text ats = _ats() subs = _readable(store) by_board: dict[str, list[dict[str, Any]]] = defaultdict(list) for found in store.fetch_text_waiting(): sub = subs.get(found["subscription_id"]) if sub is None or f"{sub.source}:failing:{sub.target}" in store.state: continue # The id the board's per-role state keeps: `host:slug:id`. found["compound"] = f"{sub.target}:{_native(found, sub)}" by_board[sub.target].append(found) due = {target: set(ats.detail_due(store.state, target, [f["compound"] for f in found], retry_days=settings.retry_days)) for target, found in by_board.items()} # Best first; roles whose read failed after every other, so a page that # keeps failing cannot take the budget of the roles never tried. failed = {target: ats._runner.id_set(store.state, ats.detail_keys(target)[1]) for target in by_board} ranked = sorted((f for found in by_board.values() for f in found), key=lambda f: (f["compound"] in failed[subs[f["subscription_id"]].target], -float(f["score"] or 0), f["uid"])) picked: dict[str, list[str]] = {} total = 0 for found in ranked: target = subs[found["subscription_id"]].target if found["compound"] not in due[target]: continue if total >= settings.per_run: break uids = picked.setdefault(target, []) if len(uids) >= settings.per_board: continue uids.append(found["uid"]) total += 1 return list(picked.items()) def batches(config: Config, store: Store) -> list[Batch]: """The finder: one batch per board or search, nothing while the stage is off.""" if not stages.stage_on(config, store, "fetch_text"): return [] return [Batch({"target": target, "uids": uids}, label=f"Reading {len(uids)} roles on {target}") for target, uids in plan(config, store)] @job(KIND, "Reading roles on their own pages", resource="http") def run_batch(config: Config, store: Store, *, target: str, uids: list[str], context: RunContext = NULL) -> dict[str, Any]: """Read one board's or search's roles in `uids` on their own pages, in that order, and store what each page filled. `filled` is how many roles changed, `texts` how many of them now have text.""" ats = _ats() subs = [s for s in _readable(store).values() if s.target == target] if not subs or not uids: return {"target": target, "filled": 0, "texts": 0} sub = subs[0] found = {f["uid"]: f for f in store.fetch_text_waiting(uids)} rows = [_row(found[uid], sub) for uid in uids if uid in found] context.check() with httpx.Client() as client: filled = ats.read_board_details(store.state, target, rows, client=client, cap=len(rows), retry_days=config.fetch_text.retry_days) label = sub.company or sub.label texts = 0 for row in filled: item = ats.item_of(row, target, label) item.source = sub.source store.upsert_role(item, sub) texts += bool(row.get("description")) log.info("fetch_text: %s: %d read, %d filled, %d with text", target, len(rows), len(filled), texts) return {"target": target, "read": len(rows), "filled": len(filled), "texts": texts} def filled(tally: Tally) -> int: """How many roles the batches behind `tally` filled.""" return sum(int(result.get("filled") or 0) for result in tally.results) def settle(config: Config, store: Store, tally: Tally, *, context: RunContext = NULL) -> dict[str, Any]: """The run's sum: roles filled, and how many of them now have text.""" return {"filled": filled(tally), "texts": sum(int(result.get("texts") or 0) for result in tally.results)} #: What the task and the cascade's job are called on the Runs page. LABEL = "Fetching the text of the roles in the cut" def fetch_text_task() -> Task: return Task(kind=KIND, finder=batches, name="fetch_text", settle=SETTLE, label=LABEL)