159 lines
7.0 KiB
Python
159 lines
7.0 KiB
Python
"""What a board's read cannot do, in words: the Companies list shows them
|
|
after the reach line and filters on them, so the owner can go through the
|
|
companies read only under a limitation.
|
|
|
|
A limit comes from two places. The reader declares what it can do (`READS`:
|
|
where the text is, what it states, what it filters, its ceiling), and the
|
|
board's `state` says what the last pull did (`plugins/readers/_runner.py`,
|
|
`stamp`): failed, read cut short, read by reach or by terms. `words` joins
|
|
both; nothing here decides how a board is read, so the words never disagree
|
|
with the pull as long as they read the keys the pull writes.
|
|
|
|
A word with a number carries it after a colon ("without text: 12"), and the
|
|
filter matches the phrase before the colon (`WORDS`).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping
|
|
from types import ModuleType
|
|
from typing import NamedTuple
|
|
|
|
from app.jobs.reach import SMALL, Plan
|
|
from app.models import Subscription
|
|
from app.storage import Store
|
|
from plugins.readers import Reads, _runner
|
|
|
|
#: Every phrase, in the order `words` gives them, with what it means. The
|
|
#: Companies filter offers one option per entry (`settings/sources.html`
|
|
#: repeats them and a test compares the two).
|
|
WORDS: dict[str, str] = {
|
|
"failing": "the last read failed; the roles stay open until it has failed for 21 days",
|
|
"read incomplete": "the last read answered but did not list the whole board, so it closed nothing",
|
|
"swept": "the board is over the page cap and is read across several pulls",
|
|
"filtered by reach": "the read asked the host for the agents' countries and remote only",
|
|
"remote not asked": "filtered by country on a host with no remote filter while the agents want remote",
|
|
"narrowed by terms": "read by the agents' fetch terms, so a role outside them is never listed",
|
|
"nothing asked": "the host knows none of the places asked, so the read asked nothing and closed nothing",
|
|
"size 0": "the board was sized at no roles",
|
|
"over the ceiling": "the board is at or over what the host answers at all, so the read is incomplete",
|
|
"text per role": "the list has no text; fetch text reads each role's page, for the roles in the cut",
|
|
"no text": "the host gives no text",
|
|
"excerpt only": "the host gives a teaser and nothing more",
|
|
"no pay": "the host states no pay",
|
|
"date not trusted": "the host's date is not the posting date",
|
|
"without text": "open roles with no description, teaser or page text (the number follows)",
|
|
"text failed": "roles whose page read failed and is tried again (the number follows)",
|
|
"read whole": "under 300 roles or a host that cannot filter, so the reach does not narrow it",
|
|
}
|
|
|
|
#: The `state` keys `words` reads, `<prefix>:<key>:<target>`.
|
|
KEYS = ("failing", "ok", "complete", "swept", "sweep", "filtered", "remote_unasked",
|
|
"narrowed", "detailfailed")
|
|
|
|
|
|
class Texts(NamedTuple):
|
|
"""One board's open roles: how many, and how many have no text at all."""
|
|
|
|
open: int
|
|
without_text: int
|
|
|
|
|
|
def state_of(store: Store) -> dict[str, str]:
|
|
"""The keys `words` needs for every board, one query a key rather than one
|
|
a board and key."""
|
|
out: dict[str, str] = {}
|
|
for prefix in ("ats", "search"):
|
|
for key in KEYS:
|
|
out.update(store.state_items(f"{prefix}:{key}:"))
|
|
return out
|
|
|
|
|
|
def texts(store: Store) -> dict[int, Texts]:
|
|
"""`subscription id -> Texts` over the open roles of every subscription,
|
|
copies left out as the feed leaves them. One query, for all boards."""
|
|
with store.transaction() as conn:
|
|
return {int(sub): Texts(int(total), int(bare or 0)) for sub, total, bare in conn.execute(
|
|
"""SELECT subscription_id, COUNT(*),
|
|
SUM(description = '' AND feed_text = '' AND article_text = '')
|
|
FROM roles
|
|
WHERE closed_at IS NULL AND dup_of IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id""")}
|
|
|
|
|
|
def words(module: ModuleType, state: Mapping[str, str], target: str,
|
|
sub: Subscription | None, size: int | None, *, plan: Plan | None = None,
|
|
line: str = "", counts: Texts | None = None) -> list[str]:
|
|
"""The limits of one board, in the order of `WORDS`, those that apply.
|
|
|
|
`state` holds the board's keys (`state_of`), `size` its kept size,
|
|
`plan` and `sub` the reach it is read with (for "read whole"), `line` the
|
|
reach line the row shows (`reach.line`: only it knows the site listed
|
|
none of the places asked) and `counts` the board's roles (`texts`). A
|
|
reader that declares nothing gives the words its state gives. A key the
|
|
pull has not written yet, such as `remote_unasked` on a database from
|
|
before it, counts as not set."""
|
|
reads = getattr(module, "READS", None)
|
|
declared = _runner.declared(module)
|
|
prefix = "search" if declared and reads.scope == "search" else "ats"
|
|
|
|
def key(name: str) -> str:
|
|
return f"{prefix}:{name}:{target}"
|
|
|
|
out: list[str] = []
|
|
if key("failing") in state:
|
|
out.append("failing")
|
|
if state.get(key("ok")) and not state.get(key("complete")):
|
|
out.append("read incomplete")
|
|
if state.get(key("swept")) == "1" or key("sweep") in state:
|
|
out.append("swept")
|
|
if state.get(key("filtered")) == "1":
|
|
out.append("filtered by reach")
|
|
if state.get(key("remote_unasked")) == "1":
|
|
out.append("remote not asked")
|
|
if state.get(key("narrowed")) == "1":
|
|
out.append("narrowed by terms")
|
|
if "nothing asked" in line:
|
|
out.append("nothing asked")
|
|
if size == 0:
|
|
out.append("size 0")
|
|
if declared and reads.ceiling is not None and size is not None and size >= reads.ceiling:
|
|
out.append("over the ceiling")
|
|
if declared:
|
|
out += _declared(reads)
|
|
if counts is not None and counts.without_text:
|
|
out.append(f"without text: {counts.without_text}")
|
|
if failed := len(_runner.id_set(state, key("detailfailed"))):
|
|
out.append(f"text failed: {failed}")
|
|
if declared and reads.scope == "board" and _whole(reads, sub, size, plan):
|
|
out.append("read whole")
|
|
return out
|
|
|
|
|
|
def _whole(reads: Reads, sub: Subscription | None, size: int | None, plan: Plan | None) -> bool:
|
|
"""Whether the reach leaves the board unread by place: its reader cannot
|
|
filter, or the board is under `SMALL` roles (the line's two "read whole"
|
|
cases; `reach.line` has the rest)."""
|
|
if not reads.filters.declared():
|
|
return True
|
|
if plan is not None and sub is not None:
|
|
return plan.why(sub)[1] == "small"
|
|
return size is not None and size < SMALL
|
|
|
|
|
|
def _declared(reads: Reads) -> list[str]:
|
|
"""The words a reader's declaration gives, whatever the board."""
|
|
out: list[str] = []
|
|
match reads.text:
|
|
case "detail":
|
|
out.append("text per role")
|
|
case "none":
|
|
out.append("no text")
|
|
case "excerpt":
|
|
out.append("excerpt only")
|
|
if reads.fields.pay == "none":
|
|
out.append("no pay")
|
|
if reads.fields.posted != "trusted":
|
|
out.append("date not trusted")
|
|
return out
|