diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index dc078d9..8a235ac 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -15,14 +15,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. discover.py which ATS a company's careers site runs on, from its pages posting.py one role's posting page fetched as text, on demand reach.py which countries a pull asks each big board for, from the agents + limits.py what limits a board's read, in words, for the Companies list """ from __future__ import annotations from typing import Any -from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources, - stages, summarise) +from app.jobs import (cascade, discover, embed, extract, limits, posting, pull, reach, score, + sources, stages, summarise) from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents @@ -41,5 +42,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: return config, Store(config.database) -__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting", +__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "limits", "posting", "pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] diff --git a/app/jobs/limits.py b/app/jobs/limits.py new file mode 100644 index 0000000..3b9e362 --- /dev/null +++ b/app/jobs/limits.py @@ -0,0 +1,158 @@ +"""What a board's read cannot do, in words: the Companies list shows them +after the reach line and filters on them, so the owner can go through the +companies read only under a limitation. + +A limit comes from two places. The reader declares what it can do (`READS`: +where the text is, what it states, what it filters, its ceiling), and the +board's `state` says what the last pull did (`plugins/readers/_runner.py`, +`stamp`): failed, read cut short, read by reach or by terms. `words` joins +both; nothing here decides how a board is read, so the words never disagree +with the pull as long as they read the keys the pull writes. + +A word with a number carries it after a colon ("without text: 12"), and the +filter matches the phrase before the colon (`WORDS`). +""" + +from __future__ import annotations + +from collections.abc import Mapping +from types import ModuleType +from typing import NamedTuple + +from app.jobs.reach import SMALL, Plan +from app.models import Subscription +from app.storage import Store +from plugins.readers import Reads, _runner + +#: Every phrase, in the order `words` gives them, with what it means. The +#: Companies filter offers one option per entry (`settings/sources.html` +#: repeats them and a test compares the two). +WORDS: dict[str, str] = { + "failing": "the last read failed; the roles stay open until it has failed for 21 days", + "read incomplete": "the last read answered but did not list the whole board, so it closed nothing", + "swept": "the board is over the page cap and is read across several pulls", + "filtered by reach": "the read asked the host for the agents' countries and remote only", + "remote not asked": "filtered by country on a host with no remote filter while the agents want remote", + "narrowed by terms": "read by the agents' fetch terms, so a role outside them is never listed", + "nothing asked": "the host knows none of the places asked, so the read asked nothing and closed nothing", + "size 0": "the board was sized at no roles", + "over the ceiling": "the board is at or over what the host answers at all, so the read is incomplete", + "text per role": "the list has no text; enrich reads each role's page, ranked roles first", + "no text": "the host gives no text", + "excerpt only": "the host gives a teaser and nothing more", + "no pay": "the host states no pay", + "date not trusted": "the host's date is not the posting date", + "without text": "open roles with no description, teaser or page text (the number follows)", + "text failed": "roles whose page read failed and is tried again (the number follows)", + "read whole": "under 300 roles or a host that cannot filter, so the reach does not narrow it", +} + +#: The `state` keys `words` reads, `::`. +KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked", + "narrowed", "detailfailed") + + +class Texts(NamedTuple): + """One board's open roles: how many, and how many have no text at all.""" + + open: int + without_text: int + + +def state_of(store: Store) -> dict[str, str]: + """The keys `words` needs for every board, one query a key rather than one + a board and key.""" + out: dict[str, str] = {} + for prefix in ("ats", "search"): + for key in KEYS: + out.update(store.state_items(f"{prefix}:{key}:")) + return out + + +def texts(store: Store) -> dict[int, Texts]: + """`subscription id -> Texts` over the open roles of every subscription, + copies left out as the feed leaves them. One query, for all boards.""" + with store.transaction() as conn: + return {int(sub): Texts(int(total), int(bare or 0)) for sub, total, bare in conn.execute( + """SELECT subscription_id, COUNT(*), + SUM(description = '' AND feed_text = '' AND article_text = '') + FROM roles + WHERE closed_at IS NULL AND dup_of IS NULL AND subscription_id IS NOT NULL + GROUP BY subscription_id""")} + + +def words(module: ModuleType, state: Mapping[str, str], target: str, + sub: Subscription | None, size: int | None, *, plan: Plan | None = None, + line: str = "", counts: Texts | None = None) -> list[str]: + """The limits of one board, in the order of `WORDS`, those that apply. + + `state` holds the board's keys (`state_of`), `size` its kept size, + `plan` and `sub` the reach it is read with (for "read whole"), `line` the + reach line the row shows (`reach.line`: only it knows the site listed + none of the places asked) and `counts` the board's roles (`texts`). A + reader that declares nothing gives the words its state gives. A key the + pull has not written yet, such as `remote_unasked` on a database from + before it, counts as not set.""" + reads = getattr(module, "READS", None) + declared = _runner.declared(module) + prefix = "search" if declared and reads.scope == "search" else "ats" + + def key(name: str) -> str: + return f"{prefix}:{name}:{target}" + + out: list[str] = [] + if key("failing") in state: + out.append("failing") + if state.get(key("ok")) and not state.get(key("complete")): + out.append("read incomplete") + if state.get(key("swept")) == "1" or key("sweep") in state: + out.append("swept") + if state.get(key("filtered")) == "1": + out.append("filtered by reach") + if state.get(key("remote_unasked")) == "1": + out.append("remote not asked") + if state.get(key("narrowed")) == "1": + out.append("narrowed by terms") + if "nothing asked" in line: + out.append("nothing asked") + if size == 0: + out.append("size 0") + if declared and reads.ceiling is not None and size is not None and size >= reads.ceiling: + out.append("over the ceiling") + if declared: + out += _declared(reads) + if counts is not None and counts.without_text: + out.append(f"without text: {counts.without_text}") + if failed := len(_runner.id_set(state, key("detailfailed"))): + out.append(f"text failed: {failed}") + if declared and reads.scope == "board" and _whole(reads, sub, size, plan): + out.append("read whole") + return out + + +def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool: + """Whether the reach leaves the board unread by place: its reader cannot + filter, or the board is under `SMALL` roles (the line's two "read whole" + cases; `reach.line` has the rest).""" + if not reads.filters.declared(): + return True + if plan is not None and sub is not None: + return plan.why(sub)[1] == "small" + return size is not None and size < SMALL + + +def _declared(reads: Reads) -> list[str]: + """The words a reader's declaration gives, whatever the board.""" + out: list[str] = [] + match reads.text: + case "detail": + out.append("text per role") + case "none": + out.append("no text") + case "excerpt": + out.append("excerpt only") + if reads.fields.pay == "none": + out.append("no pay") + if reads.fields.posted != "trusted": + out.append("date not trusted") + return out