Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
+37
-3
@@ -13,7 +13,16 @@ on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
||||
still leaves the boards it had already finished stamped, so one bad board in a
|
||||
batch of ten does not cost the other nine their closure. A board that answered
|
||||
but was not read to its stated total (a paged host that hit a 429 or the page
|
||||
cap) stores what it read and closes nothing.
|
||||
cap) stores what it read and closes nothing; a paged board over the page cap
|
||||
is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from its subscription's
|
||||
`params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
@@ -175,7 +184,11 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
@@ -252,7 +265,10 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
incomplete += 1
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if store.state.get(f"{source}:filtered:{sub.target}", ""):
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
else:
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
@@ -263,6 +279,24 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed, "copies": copies}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||
board it did not list, except a role someone acted on, one with a vote
|
||||
(any, neutral included) or an application (which holds the notes). A
|
||||
filtered read cannot tell a role gone from a role outside its filter, and
|
||||
a role acted on stays until the board is read without the filter or
|
||||
fails for `STALE_DAYS`."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""",
|
||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
|
||||
+4
-4
@@ -28,7 +28,7 @@ functions are the classifier's vocabulary and the hosts are what it parses,
|
||||
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
|
||||
app-agnostic and never reaches back into `app`. The
|
||||
Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page
|
||||
come from `plugins.search` for the same reason.
|
||||
come from the search readers in `plugins/readers/` for the same reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,9 +49,9 @@ if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from plugins.ats import FUNCTIONS, HOSTS # noqa: E402
|
||||
from plugins.search import ( # noqa: E402
|
||||
ADZUNA_COUNTRIES, JOBINDEX_AREAS, PLATSBANKEN_REGIONS,
|
||||
)
|
||||
from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402
|
||||
from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402
|
||||
from plugins.readers.platsbanken import REGIONS as PLATSBANKEN_REGIONS # noqa: E402
|
||||
|
||||
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
|
||||
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
|
||||
|
||||
Reference in new issue
Block a user