Add the limit words of a board
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
09468455d1
commit
8b07f9a602
2 files changed
+162
-3
No files matched your search
@@ -15,14 +15,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
|
||||
discover.py which ATS a company's careers site runs on, from its pages
|
||||
posting.py one role's posting page fetched as text, on demand
|
||||
reach.py which countries a pull asks each big board for, from the agents
|
||||
limits.py what limits a board's read, in words, for the Companies list
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.jobs import (cascade, discover, embed, extract, posting, pull, reach, score, sources,
|
||||
stages, summarise)
|
||||
from app.jobs import (cascade, discover, embed, extract, limits, posting, pull, reach, score,
|
||||
sources, stages, summarise)
|
||||
from app.jobs.pull import search_target, seed_searches
|
||||
from app.jobs.score import seed_agents
|
||||
|
||||
@@ -41,5 +42,5 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
|
||||
return config, Store(config.database)
|
||||
|
||||
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "posting",
|
||||
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "limits", "posting",
|
||||
"pull", "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
|
||||
@@ -0,0 +1,158 @@
|
||||
"""What a board's read cannot do, in words: the Companies list shows them
|
||||
after the reach line and filters on them, so the owner can go through the
|
||||
companies read only under a limitation.
|
||||
|
||||
A limit comes from two places. The reader declares what it can do (`READS`:
|
||||
where the text is, what it states, what it filters, its ceiling), and the
|
||||
board's `state` says what the last pull did (`plugins/readers/_runner.py`,
|
||||
`stamp`): failed, read cut short, read by reach or by terms. `words` joins
|
||||
both; nothing here decides how a board is read, so the words never disagree
|
||||
with the pull as long as they read the keys the pull writes.
|
||||
|
||||
A word with a number carries it after a colon ("without text: 12"), and the
|
||||
filter matches the phrase before the colon (`WORDS`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import ModuleType
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.jobs.reach import SMALL, Plan
|
||||
from app.models import Subscription
|
||||
from app.storage import Store
|
||||
from plugins.readers import Reads, _runner
|
||||
|
||||
#: Every phrase, in the order `words` gives them, with what it means. The
|
||||
#: Companies filter offers one option per entry (`settings/sources.html`
|
||||
#: repeats them and a test compares the two).
|
||||
WORDS: dict[str, str] = {
|
||||
"failing": "the last read failed; the roles stay open until it has failed for 21 days",
|
||||
"read incomplete": "the last read answered but did not list the whole board, so it closed nothing",
|
||||
"swept": "the board is over the page cap and is read across several pulls",
|
||||
"filtered by reach": "the read asked the host for the agents' countries and remote only",
|
||||
"remote not asked": "filtered by country on a host with no remote filter while the agents want remote",
|
||||
"narrowed by terms": "read by the agents' fetch terms, so a role outside them is never listed",
|
||||
"nothing asked": "the host knows none of the places asked, so the read asked nothing and closed nothing",
|
||||
"size 0": "the board was sized at no roles",
|
||||
"over the ceiling": "the board is at or over what the host answers at all, so the read is incomplete",
|
||||
"text per role": "the list has no text; enrich reads each role's page, ranked roles first",
|
||||
"no text": "the host gives no text",
|
||||
"excerpt only": "the host gives a teaser and nothing more",
|
||||
"no pay": "the host states no pay",
|
||||
"date not trusted": "the host's date is not the posting date",
|
||||
"without text": "open roles with no description, teaser or page text (the number follows)",
|
||||
"text failed": "roles whose page read failed and is tried again (the number follows)",
|
||||
"read whole": "under 300 roles or a host that cannot filter, so the reach does not narrow it",
|
||||
}
|
||||
|
||||
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
||||
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
||||
"narrowed", "detailfailed")
|
||||
|
||||
|
||||
class Texts(NamedTuple):
|
||||
"""One board's open roles: how many, and how many have no text at all."""
|
||||
|
||||
open: int
|
||||
without_text: int
|
||||
|
||||
|
||||
def state_of(store: Store) -> dict[str, str]:
|
||||
"""The keys `words` needs for every board, one query a key rather than one
|
||||
a board and key."""
|
||||
out: dict[str, str] = {}
|
||||
for prefix in ("ats", "search"):
|
||||
for key in KEYS:
|
||||
out.update(store.state_items(f"{prefix}:{key}:"))
|
||||
return out
|
||||
|
||||
|
||||
def texts(store: Store) -> dict[int, Texts]:
|
||||
"""`subscription id -> Texts` over the open roles of every subscription,
|
||||
copies left out as the feed leaves them. One query, for all boards."""
|
||||
with store.transaction() as conn:
|
||||
return {int(sub): Texts(int(total), int(bare or 0)) for sub, total, bare in conn.execute(
|
||||
"""SELECT subscription_id, COUNT(*),
|
||||
SUM(description = '' AND feed_text = '' AND article_text = '')
|
||||
FROM roles
|
||||
WHERE closed_at IS NULL AND dup_of IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id""")}
|
||||
|
||||
|
||||
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
||||
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
||||
line: str = "", counts: Texts | None = None) -> list[str]:
|
||||
"""The limits of one board, in the order of `WORDS`, those that apply.
|
||||
|
||||
`state` holds the board's keys (`state_of`), `size` its kept size,
|
||||
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
||||
reach line the row shows (`reach.line`: only it knows the site listed
|
||||
none of the places asked) and `counts` the board's roles (`texts`). A
|
||||
reader that declares nothing gives the words its state gives. A key the
|
||||
pull has not written yet, such as `remote_unasked` on a database from
|
||||
before it, counts as not set."""
|
||||
reads = getattr(module, "READS", None)
|
||||
declared = _runner.declared(module)
|
||||
prefix = "search" if declared and reads.scope == "search" else "ats"
|
||||
|
||||
def key(name: str) -> str:
|
||||
return f"{prefix}:{name}:{target}"
|
||||
|
||||
out: list[str] = []
|
||||
if key("failing") in state:
|
||||
out.append("failing")
|
||||
if state.get(key("ok")) and not state.get(key("complete")):
|
||||
out.append("read incomplete")
|
||||
if state.get(key("swept")) == "1" or key("sweep") in state:
|
||||
out.append("swept")
|
||||
if state.get(key("filtered")) == "1":
|
||||
out.append("filtered by reach")
|
||||
if state.get(key("remote_unasked")) == "1":
|
||||
out.append("remote not asked")
|
||||
if state.get(key("narrowed")) == "1":
|
||||
out.append("narrowed by terms")
|
||||
if "nothing asked" in line:
|
||||
out.append("nothing asked")
|
||||
if size == 0:
|
||||
out.append("size 0")
|
||||
if declared and reads.ceiling is not None and size is not None and size >= reads.ceiling:
|
||||
out.append("over the ceiling")
|
||||
if declared:
|
||||
out += _declared(reads)
|
||||
if counts is not None and counts.without_text:
|
||||
out.append(f"without text: {counts.without_text}")
|
||||
if failed := len(_runner.id_set(state, key("detailfailed"))):
|
||||
out.append(f"text failed: {failed}")
|
||||
if declared and reads.scope == "board" and _whole(reads, sub, size, plan):
|
||||
out.append("read whole")
|
||||
return out
|
||||
|
||||
|
||||
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
||||
"""Whether the reach leaves the board unread by place: its reader cannot
|
||||
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
||||
cases; `reach.line` has the rest)."""
|
||||
if not reads.filters.declared():
|
||||
return True
|
||||
if plan is not None and sub is not None:
|
||||
return plan.why(sub)[1] == "small"
|
||||
return size is not None and size < SMALL
|
||||
|
||||
|
||||
def _declared(reads: Reads) -> list[str]:
|
||||
"""The words a reader's declaration gives, whatever the board."""
|
||||
out: list[str] = []
|
||||
match reads.text:
|
||||
case "detail":
|
||||
out.append("text per role")
|
||||
case "none":
|
||||
out.append("no text")
|
||||
case "excerpt":
|
||||
out.append("excerpt only")
|
||||
if reads.fields.pay == "none":
|
||||
out.append("no pay")
|
||||
if reads.fields.posted != "trusted":
|
||||
out.append("date not trusted")
|
||||
return out
|
||||
Reference in new issue
Block a user