Guard each target in a batch, close remote-unasked and narrowed roles by age, settle failing searches, and raise one notice per settle naming what needs a look
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
24a165818c
commit
9c8a610787
4 files changed
+576
-115
No files matched your search
+10
-2
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
|||||||
|
|
||||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||||
than report a pull that never happened; `app job pull` claims the run and
|
than report a pull that never happened; `app job pull` claims the run and
|
||||||
starts the chain on a queue of its own instead of calling this.
|
starts the chain on a queue of its own instead of calling this. Anything
|
||||||
|
that raises once the run is claimed, building the chain or handing it
|
||||||
|
over, releases the claim first, or the next pull would wait
|
||||||
|
`pull.ABANDONED` for a run that never started.
|
||||||
"""
|
"""
|
||||||
run = pull.claim_run(store)
|
run = pull.claim_run(store)
|
||||||
if not run:
|
if not run:
|
||||||
since = pull.running(store)
|
since = pull.running(store)
|
||||||
context.notice(already_running(since))
|
context.notice(already_running(since))
|
||||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||||
if not context.task(pull_chain(run)):
|
try:
|
||||||
|
started = context.task(pull_chain(run))
|
||||||
|
except Exception:
|
||||||
|
pull.release_run(store, run)
|
||||||
|
raise
|
||||||
|
if not started:
|
||||||
pull.release_run(store, run)
|
pull.release_run(store, run)
|
||||||
raise RuntimeError("no queue to start the pull cascade on")
|
raise RuntimeError("no queue to start the pull cascade on")
|
||||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||||
|
|||||||
+282
-107
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
|
|||||||
`plugins/readers/_runner.py`).
|
`plugins/readers/_runner.py`).
|
||||||
|
|
||||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
||||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
reach, so its close keeps open the roles a person acted on: those with a vote
|
||||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
||||||
under a tenth of what the same read listed last time is recorded by the
|
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
||||||
runner as a failure, not as a list to close against.
|
says nothing of the remote roles either, so they stay open too and close only
|
||||||
|
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
||||||
|
that lists under a tenth of what the same read listed last time is recorded
|
||||||
|
by the runner as a failure, not as a list to close against (`_runner.read`).
|
||||||
|
|
||||||
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||||
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||||
@@ -33,14 +36,21 @@ not swept clears them.
|
|||||||
|
|
||||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||||
the roles matching a word, which says nothing of the roles it did not list,
|
the roles matching a word, which says nothing of the roles it did not list,
|
||||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
closes.
|
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||||
|
|
||||||
A search closes the same way when it listed everything that matches (see
|
A search closes the same way when it listed everything that matches (see
|
||||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
||||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
||||||
closes all of its roles, and its next answer reopens what it lists.
|
`search:failing:<target>`) closes all of its roles, and its next answer
|
||||||
|
reopens what it lists. A target that raises in its batch after its read, a
|
||||||
|
row that does not store, is recorded as failing the same way and the batch
|
||||||
|
goes on with the next one (`run_batch`).
|
||||||
|
|
||||||
|
The settle says what needs a look in one notice (`settled`): the targets
|
||||||
|
that did not answer or were read incomplete, and the boards that closed more
|
||||||
|
than `CLOSE_NOTICE` roles or more than half of theirs.
|
||||||
|
|
||||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||||
set only while the previous run is settled, in one statement), so the
|
set only while the previous run is settled, in one statement), so the
|
||||||
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pipeline import NULL, RunContext, Tally, Task, job
|
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
||||||
|
|
||||||
from app.config import Config
|
from app.config import Config
|
||||||
from app.jobs.reach import reaches
|
from app.jobs.reach import reaches
|
||||||
@@ -91,6 +101,19 @@ STALE_DAYS = 21
|
|||||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||||
FAILING = "ats:failing:"
|
FAILING = "ats:failing:"
|
||||||
|
|
||||||
|
#: The same for a search, kept by the `search` plugin.
|
||||||
|
SEARCH_FAILING = "search:failing:"
|
||||||
|
|
||||||
|
#: A board that closes more roles than this in one settle is named in the
|
||||||
|
#: settle's notice, and so is one that closes more than half of its open
|
||||||
|
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
||||||
|
#: one is an ordinary day.
|
||||||
|
CLOSE_NOTICE = 50
|
||||||
|
CLOSE_SHARE_MIN = 10
|
||||||
|
|
||||||
|
#: How many targets a notice names in its text; its detail names them all.
|
||||||
|
NOTICE_NAMES = 10
|
||||||
|
|
||||||
#: The `state` key that records the starter searches were imported.
|
#: The `state` key that records the starter searches were imported.
|
||||||
SEARCHES_SEEDED = "searches:seeded"
|
SEARCHES_SEEDED = "searches:seeded"
|
||||||
|
|
||||||
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
|||||||
`close_boards` compares against. `run` is a marker the caller claimed
|
`close_boards` compares against. `run` is a marker the caller claimed
|
||||||
already (the `pull` job, `app job pull`); without one the finder claims
|
already (the `pull` job, `app job pull`); without one the finder claims
|
||||||
its own, and queues nothing when another run is in flight. A run with
|
its own, and queues nothing when another run is in flight. A run with
|
||||||
nothing to read is settled here, since no settle will see a batch of it."""
|
nothing to read is settled here, since no settle will see a batch of it,
|
||||||
seed_searches(config, store)
|
and so is a run whose finder raises: no batch and no settle would follow,
|
||||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
||||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
held = run
|
||||||
if not any(found for _, found in targets):
|
try:
|
||||||
if run:
|
seed_searches(config, store)
|
||||||
release_run(store, run)
|
subs = {source: store.subscriptions(source=source, fetchable=True)
|
||||||
return []
|
for source in SOURCES}
|
||||||
run = run or claim_run(store)
|
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||||
if not run:
|
if not any(found for _, found in targets):
|
||||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
if run:
|
||||||
running(store))
|
release_run(store, run)
|
||||||
return []
|
return []
|
||||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
run = run or claim_run(store)
|
||||||
reach_of = reaches(store, subs["ats"])
|
if not run:
|
||||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
running(store))
|
||||||
for batch in batches:
|
return []
|
||||||
if batch["source"] == "ats":
|
held = run
|
||||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||||
return batches
|
reach_of = reaches(store, subs["ats"])
|
||||||
|
batches: list[dict[str, Any]] = [
|
||||||
|
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||||
|
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||||
|
for batch in batches:
|
||||||
|
if batch["source"] == "ats":
|
||||||
|
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||||
|
return batches
|
||||||
|
except Exception:
|
||||||
|
if held:
|
||||||
|
release_run(store, held)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
@job("pull_batch", "Reading boards", resource="http")
|
@job("pull_batch", "Reading boards", resource="http")
|
||||||
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
|||||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||||
plugin as its `reach` option.
|
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
||||||
|
which the runner's 10% guard accepts the answer it kept refusing.
|
||||||
|
|
||||||
|
Each target is guarded on its own. A read that fails is the plugin's to
|
||||||
|
record; anything raised after it, a row `upsert_role` refuses or a bug
|
||||||
|
past the read, is recorded here under `<source>:failing:<target>` with
|
||||||
|
its reason, and the batch goes on with the next target. Otherwise one
|
||||||
|
bad row would cost the targets after it in the batch their read on every
|
||||||
|
pull, since batches are stable, and leave no trace on any of them. A
|
||||||
|
Kill (`Cancelled`) still ends the batch.
|
||||||
"""
|
"""
|
||||||
from plugins.ats import ATSSource
|
from plugins.ats import ATSSource
|
||||||
|
from plugins.readers import _runner
|
||||||
from plugins.search import SearchSource
|
from plugins.search import SearchSource
|
||||||
|
|
||||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||||
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
|||||||
if sub is None:
|
if sub is None:
|
||||||
continue
|
continue
|
||||||
context.check()
|
context.check()
|
||||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
options = dict(config.sources.get(source) or {}) | {"run": run,
|
||||||
plugin: ATSSource | SearchSource
|
"stale_days": STALE_DAYS}
|
||||||
if source == "search":
|
items: Any = None
|
||||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
try:
|
||||||
plugin = SearchSource(options)
|
plugin: ATSSource | SearchSource
|
||||||
else:
|
if source == "search":
|
||||||
# A board's params are its filter; a reader's key is kept with
|
options["searches"] = [{"url": target, "label": sub.label,
|
||||||
# the searches' (`sources.search.<name>`).
|
"params": sub.params}]
|
||||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
plugin = SearchSource(options)
|
||||||
"params": sub.params}]
|
else:
|
||||||
options["keys"] = dict(config.sources.get("search") or {})
|
# A board's params are its filter; a reader's key is kept with
|
||||||
options["reach"] = (reach or {}).get(target)
|
# the searches' (`sources.search.<name>`).
|
||||||
plugin = ATSSource(options)
|
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||||
plugin.state = store.state
|
"params": sub.params}]
|
||||||
for item in plugin.fetch():
|
options["keys"] = dict(config.sources.get("search") or {})
|
||||||
item.source = source
|
options["reach"] = (reach or {}).get(target)
|
||||||
store.upsert_role(item, sub)
|
plugin = ATSSource(options)
|
||||||
stored += 1
|
plugin.state = store.state
|
||||||
|
items = plugin.fetch()
|
||||||
|
for item in items:
|
||||||
|
item.source = source
|
||||||
|
store.upsert_role(item, sub)
|
||||||
|
stored += 1
|
||||||
|
except Cancelled:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("pull: %s failed after its read (%s); on to the next target",
|
||||||
|
target, exc, exc_info=True)
|
||||||
|
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
||||||
|
finally:
|
||||||
|
# The plugin's generator holds its HTTP client until it is closed.
|
||||||
|
if items is not None and hasattr(items, "close"):
|
||||||
|
items.close()
|
||||||
return {"roles": stored, "run": run}
|
return {"roles": stored, "run": run}
|
||||||
|
|
||||||
|
|
||||||
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
|
|||||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||||
context: RunContext = NULL) -> dict[str, Any]:
|
context: RunContext = NULL) -> dict[str, Any]:
|
||||||
"""The run's settle: close every board this run answered in full and
|
"""The run's settle: close every board this run answered in full and
|
||||||
stopped listing a role from, and tally what happened.
|
stopped listing a role from, tally what happened, and say what needs a
|
||||||
|
look.
|
||||||
|
|
||||||
Reads `self.state` rather than the batch jobs' own results, because a
|
Reads `self.state` rather than the batch jobs' own results, because a
|
||||||
`pull_batch` job that raised partway through still leaves the boards it
|
`pull_batch` job that raised partway through still leaves the boards it
|
||||||
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||||
whole batch job never got to it — is never closed by its list, and a board
|
whole batch job never got to it — is never closed by its list, and a board
|
||||||
that answered incompletely is not either. A search closes by its list only
|
that answered incompletely is not either. A search closes by its list only
|
||||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
||||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
||||||
|
its roles (`failing_boards`). How a read that answered closes is
|
||||||
|
`_close_read`'s.
|
||||||
|
|
||||||
The stamps are compared with this run's own marker, the one its batches
|
The stamps are compared with this run's own marker, the one its batches
|
||||||
carried, never with the stored `pull:run`, which a later run may have
|
carried, never with the stored `pull:run`, which a later run may have
|
||||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
read) closes nothing by list or by age and says nothing. Last, the run is
|
||||||
which lets the next pull start.
|
marked settled, which lets the next pull start.
|
||||||
|
|
||||||
A board or search switched off while its batch was in flight had its
|
A board or search switched off while its batch was in flight had its
|
||||||
roles reopened by that batch, after the switch closed them, so every
|
roles reopened by that batch, after the switch closed them, so every
|
||||||
switched-off one is closed again here.
|
switched-off one is closed again here; those closes are asked for and not
|
||||||
|
named in the notice.
|
||||||
|
|
||||||
|
The counts go into the result (`unanswered` counts boards and searches
|
||||||
|
whose stamp is not this run's), and one notice (`settled`) names every
|
||||||
|
target unanswered, every board read incomplete other than one mid-sweep,
|
||||||
|
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
||||||
|
half the open roles it had, when there is any. The result alone was
|
||||||
|
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
||||||
|
|
||||||
The `prep` stage that follows maps what this run stored and folds the
|
The `prep` stage that follows maps what this run stored and folds the
|
||||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||||
and the roles this run added join theirs.
|
and the roles this run added join theirs.
|
||||||
"""
|
"""
|
||||||
answered = searched = failed = incomplete = closed = 0
|
answered = searched = incomplete = closed = 0
|
||||||
|
unanswered: list[str] = []
|
||||||
|
partial: list[str] = []
|
||||||
|
many: list[tuple[str, int, int]] = []
|
||||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||||
failing = failing_boards(store)
|
failing = failing_boards(store)
|
||||||
run_marker = _run_of(tally, context)
|
run_marker = _run_of(tally, context)
|
||||||
|
open_before = _open_counts(store) if run_marker else {}
|
||||||
for source in SOURCES:
|
for source in SOURCES:
|
||||||
for sub in store.subscriptions(source=source):
|
for sub in store.subscriptions(source=source):
|
||||||
if not (sub.subscribed and sub.fetch):
|
if not (sub.subscribed and sub.fetch):
|
||||||
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
for sub in store.subscriptions(source=source, fetchable=True):
|
for sub in store.subscriptions(source=source, fetchable=True):
|
||||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||||
if not (stamp and stamp == run_marker):
|
if not (stamp and stamp == run_marker):
|
||||||
failed += 1
|
unanswered.append(sub.target)
|
||||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
since = failing.get(sub.target, ("", ""))[0]
|
||||||
if since and since < stale:
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||||
closed += store.close_subscription(sub.id)
|
|
||||||
continue
|
|
||||||
if source == "search":
|
|
||||||
searched += 1
|
|
||||||
else:
|
else:
|
||||||
answered += 1
|
if source == "search":
|
||||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
searched += 1
|
||||||
if source == "ats":
|
else:
|
||||||
incomplete += 1
|
answered += 1
|
||||||
continue
|
read = _close_read(store, source, sub, stale)
|
||||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
if read is None:
|
||||||
continue
|
if source == "ats":
|
||||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
incomplete += 1
|
||||||
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
partial.append(sub.target)
|
||||||
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
continue
|
||||||
continue
|
shut = read
|
||||||
store.state.pop(f"{source}:missing:{sub.target}", None)
|
closed += shut
|
||||||
if is_filtered:
|
before = open_before.get(sub.id, 0)
|
||||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
||||||
else:
|
many.append((sub.target, shut, before))
|
||||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
|
||||||
if searched:
|
if searched:
|
||||||
closed += store.close_unseen("search", stale)
|
closed += store.close_unseen("search", stale)
|
||||||
if run_marker:
|
if run_marker:
|
||||||
|
if unanswered or partial or many:
|
||||||
|
context.notice(settled(unanswered, partial, many, failing,
|
||||||
|
boards=answered, searches=searched, closed=closed))
|
||||||
release_run(store, run_marker)
|
release_run(store, run_marker)
|
||||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||||
return {"boards": answered, "searches": searched, "failed": failed,
|
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
||||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||||
|
|
||||||
|
|
||||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||||
|
"""Close what one target's read of this run says has closed, and return
|
||||||
|
how many; None when the read was incomplete, which closes nothing.
|
||||||
|
|
||||||
|
A read narrowed by terms closes only the roles it has not listed for
|
||||||
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||||
|
two sweeps in a row missed (`_close_swept`); any other complete read
|
||||||
|
clears that record and closes what it did not list, all of it unfiltered
|
||||||
|
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||||
|
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||||
|
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||||
|
age instead."""
|
||||||
|
state, target = store.state, sub.target
|
||||||
|
if not state.get(f"{source}:complete:{target}", ""):
|
||||||
|
return None
|
||||||
|
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
||||||
|
if state.get(f"{source}:narrowed:{target}", ""):
|
||||||
|
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||||
|
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||||
|
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||||
|
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||||
|
if spare_remote else 0)
|
||||||
|
if state.get(f"{source}:swept:{target}", ""):
|
||||||
|
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||||
|
spare_remote=spare_remote)
|
||||||
|
state.pop(f"{source}:missing:{target}", None)
|
||||||
|
if is_filtered:
|
||||||
|
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
||||||
|
spare_remote=spare_remote)
|
||||||
|
return store.mark_closed(source, seen_ids, sub.id)
|
||||||
|
|
||||||
|
|
||||||
|
def _open_counts(store: Store) -> dict[int, int]:
|
||||||
|
"""`subscription id -> open roles`, before the settle closes any, for the
|
||||||
|
share a board closed."""
|
||||||
|
with store.transaction() as conn:
|
||||||
|
return {int(row[0]): int(row[1]) for row in conn.execute(
|
||||||
|
"""SELECT subscription_id, count(*) FROM roles
|
||||||
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||||
|
GROUP BY subscription_id""")}
|
||||||
|
|
||||||
|
|
||||||
|
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
||||||
|
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
||||||
|
closed: int) -> Notice:
|
||||||
|
"""The notice of a settle that has something to look at: the targets
|
||||||
|
that did not answer (with the reason their failing record gives), the
|
||||||
|
boards read incomplete, and the boards that closed many roles, each
|
||||||
|
as `closed of open`, with the totals. The text names the first
|
||||||
|
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
||||||
|
def named(target: str) -> str:
|
||||||
|
reason = failing.get(target, ("", ""))[1]
|
||||||
|
return f"{target} ({reason})" if reason else target
|
||||||
|
|
||||||
|
groups = [("Not answering", [named(t) for t in unanswered]),
|
||||||
|
("Read incomplete", list(incomplete)),
|
||||||
|
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
||||||
|
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
||||||
|
f"{closed:,} roles."]
|
||||||
|
detail: list[str] = []
|
||||||
|
for title, names in groups:
|
||||||
|
if not names:
|
||||||
|
continue
|
||||||
|
shown = ", ".join(names[:NOTICE_NAMES])
|
||||||
|
more = len(names) - NOTICE_NAMES
|
||||||
|
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
||||||
|
else "."))
|
||||||
|
detail += [f"{title}: {name}" for name in names]
|
||||||
|
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
||||||
|
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
||||||
|
|
||||||
|
|
||||||
|
#: The roles a partial read never closes by its list: one with a vote (any,
|
||||||
|
#: neutral included) or an application (which holds the notes).
|
||||||
|
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||||
|
AND uid NOT IN (SELECT role_uid FROM applications
|
||||||
|
WHERE role_uid IS NOT NULL)"""
|
||||||
|
|
||||||
|
#: The remote roles a filtered read that could not ask for them leaves alone.
|
||||||
|
_NOT_REMOTE = " AND remote != 'remote'"
|
||||||
|
|
||||||
|
|
||||||
|
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
||||||
|
*, spare_remote: bool = False) -> int:
|
||||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||||
board it did not list, except a role someone acted on, one with a vote
|
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
||||||
(any, neutral included) or an application (which holds the notes). A
|
|
||||||
filtered read cannot tell a role gone from a role outside its filter, and
|
filtered read cannot tell a role gone from a role outside its filter, and
|
||||||
a role acted on stays until the board is read without the filter or
|
a role acted on stays until the board is read without the filter or
|
||||||
fails for `STALE_DAYS`."""
|
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
||||||
|
well, since the read could not ask for them."""
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
return conn.execute(
|
return conn.execute(
|
||||||
"""UPDATE roles SET closed_at = ?
|
f"""UPDATE roles SET closed_at = ?
|
||||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
||||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
+ (_NOT_REMOTE if spare_remote else ""),
|
||||||
AND uid NOT IN (SELECT role_uid FROM applications
|
|
||||||
WHERE role_uid IS NOT NULL)""",
|
|
||||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|
||||||
|
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||||
|
stale: str, *, remote_only: bool = False) -> int:
|
||||||
|
"""Close the open roles of one board that its read did not list and that
|
||||||
|
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||||
|
writes on every listing): the rule for what a read cannot speak for. A
|
||||||
|
read narrowed by terms closes all such roles, as a search does; with
|
||||||
|
`remote_only`, a filtered read that could not ask for remote closes only
|
||||||
|
its remote roles so, sparing those acted on as its list does."""
|
||||||
|
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||||
|
with store.transaction() as conn:
|
||||||
|
return conn.execute(
|
||||||
|
f"""UPDATE roles SET closed_at = ?
|
||||||
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||||
|
AND last_seen < ?
|
||||||
|
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||||
|
(store.now(), source, subscription_id, stale,
|
||||||
|
json.dumps(list(seen_ids)))).rowcount
|
||||||
|
|
||||||
|
|
||||||
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||||
filtered: bool) -> int:
|
filtered: bool, *, spare_remote: bool = False) -> int:
|
||||||
"""The close of a board whose complete read finished a sweep across
|
"""The close of a board whose complete read finished a sweep across
|
||||||
pulls: its open roles missing from `seen_ids` that also are in
|
pulls: its open roles missing from `seen_ids` that also are in
|
||||||
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||||
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
|||||||
missed may only have shifted back past the cursor when roles closed
|
missed may only have shifted back past the cursor when roles closed
|
||||||
between pulls, so one sweep is not evidence that it is gone. A filtered
|
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||||
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||||
stays open, and stays in `missing`."""
|
stays open, and stays in `missing`, and so does a remote role with
|
||||||
|
`spare_remote`."""
|
||||||
key = f"{source}:missing:{sub.target}"
|
key = f"{source}:missing:{sub.target}"
|
||||||
try:
|
try:
|
||||||
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
earlier = set()
|
earlier = set()
|
||||||
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
||||||
AND uid NOT IN (SELECT role_uid FROM applications
|
|
||||||
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
|
||||||
with store.transaction() as conn:
|
with store.transaction() as conn:
|
||||||
missed = [row[0] for row in conn.execute(
|
missed = [row[0] for row in conn.execute(
|
||||||
"""SELECT source_id FROM roles
|
"""SELECT source_id FROM roles
|
||||||
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
|||||||
|
|
||||||
|
|
||||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
"""`target -> (since, reason)` for every `ats` board and every search
|
||||||
failed: when the failures began, and what the latest one said."""
|
whose last fetch failed: when the failures began, and what the latest
|
||||||
|
one said. No board host is also a search provider, so a board's target
|
||||||
|
(`host:slug`) and a search's (`provider:name`) never collide in one
|
||||||
|
dict."""
|
||||||
out: dict[str, tuple[str, str]] = {}
|
out: dict[str, tuple[str, str]] = {}
|
||||||
for key, value in store.state_items(FAILING).items():
|
for prefix in (FAILING, SEARCH_FAILING):
|
||||||
try:
|
for key, value in store.state_items(prefix).items():
|
||||||
record = json.loads(value)
|
try:
|
||||||
except ValueError:
|
record = json.loads(value)
|
||||||
continue
|
except ValueError:
|
||||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
continue
|
||||||
str(record.get("reason") or ""))
|
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
||||||
|
str(record.get("reason") or ""))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+231
-6
@@ -7,6 +7,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
|
import types
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
|
|||||||
from desk.models import Item # noqa: E402
|
from desk.models import Item # noqa: E402
|
||||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||||
from app.models import Application # noqa: E402
|
from app.models import Application # noqa: E402
|
||||||
from plugins import readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import _runner # noqa: E402
|
from plugins.readers import Filters, Reads, _runner # noqa: E402
|
||||||
from plugins.ats import ATSSource # noqa: E402
|
from plugins.ats import ATSSource # noqa: E402
|
||||||
from app import prep # noqa: E402
|
from app import prep # noqa: E402
|
||||||
from app.jobs import cascade, pull # noqa: E402
|
from app.jobs import cascade, pull # noqa: E402
|
||||||
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
|||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||||
"roles": 1, "closed": 1}
|
"roles": 1, "closed": 1}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||||
|
|
||||||
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
|||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||||
|
|
||||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||||
"roles": 0, "closed": 0}
|
"roles": 0, "closed": 0}
|
||||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||||
|
|
||||||
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
|
|||||||
_answer(monkeypatch, ValueError("gone"))
|
_answer(monkeypatch, ValueError("gone"))
|
||||||
result = _pull(acme)
|
result = _pull(acme)
|
||||||
assert _open(acme) == {"workable:acme:1"}
|
assert _open(acme) == {"workable:acme:1"}
|
||||||
assert (result["failed"], result["closed"]) == (1, 0)
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||||
assert since and reason == "ValueError: gone"
|
assert since and reason == "ValueError: gone"
|
||||||
|
|
||||||
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
|
|||||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||||
|
|
||||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||||
assert (result["failed"], result["closed"]) == (1, 0)
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
assert pull.running(acme) == b["run"]
|
assert pull.running(acme) == b["run"]
|
||||||
|
|
||||||
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
|
|||||||
context = RunContext(waited_for=(failed,))
|
context = RunContext(waited_for=(failed,))
|
||||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------- losses, guarded
|
||||||
|
|
||||||
|
def _days_ago(days: float) -> str:
|
||||||
|
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
|
||||||
|
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
|
||||||
|
[(value, i) for i in source_ids])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def big(store: Store) -> Store:
|
||||||
|
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||||
|
board="greenhouse")
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([], True))
|
||||||
|
result = _pull(big)
|
||||||
|
assert len(_open(big)) == 40
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
|
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_whole_read_of_nothing_after_five_closes_the_board(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([], True))
|
||||||
|
assert _pull(big)["closed"] == 5
|
||||||
|
assert _open(big) == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_suspect_answer_is_accepted_after_stale_days(
|
||||||
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||||
|
_pull(big)
|
||||||
|
_answer(monkeypatch, ([_row("1")], True))
|
||||||
|
_pull(big)
|
||||||
|
assert len(_open(big)) == 40
|
||||||
|
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
|
||||||
|
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
|
||||||
|
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
|
||||||
|
result = _pull(big)
|
||||||
|
assert _open(big) == {"greenhouse:big:1"}
|
||||||
|
assert (result["unanswered"], result["closed"]) == (0, 39)
|
||||||
|
assert pull.failing_boards(big) == {}
|
||||||
|
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
|
||||||
|
"""A whole-listing host `fake` that filters by country and, per `remote`,
|
||||||
|
on remote; its feed lists the ids in the returned list."""
|
||||||
|
listed: list[str] = []
|
||||||
|
module: object = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||||
|
module.READS = Reads( # type: ignore[attr-defined]
|
||||||
|
scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||||
|
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
|
||||||
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||||
|
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
|
||||||
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
|
||||||
|
listed).encode())
|
||||||
|
return listed
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
listed = _remote_host(monkeypatch, remote="none")
|
||||||
|
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
|
||||||
|
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
|
||||||
|
|
||||||
|
def pull_once() -> dict[str, object]:
|
||||||
|
run = pull.claim_run(store)
|
||||||
|
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
|
||||||
|
reach=reach)
|
||||||
|
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
listed[:] = ["1", "2", "3", "4"]
|
||||||
|
pull_once()
|
||||||
|
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
|
||||||
|
listed[:] = ["1"]
|
||||||
|
assert pull_once()["closed"] == 1
|
||||||
|
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
|
||||||
|
assert store.state["ats:remote_unasked:fake:acme"] == "1"
|
||||||
|
|
||||||
|
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
|
||||||
|
assert pull_once()["closed"] == 1
|
||||||
|
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
||||||
|
_pull(acme)
|
||||||
|
|
||||||
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||||
|
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||||
|
assert _pull(acme)["closed"] == 0
|
||||||
|
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
|
||||||
|
assert _pull(acme)["closed"] == 1
|
||||||
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
|
||||||
|
for target in targets:
|
||||||
|
slug = target.partition(":")[2]
|
||||||
|
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
|
||||||
|
|
||||||
|
def fake_fetch(self: ATSSource) -> object:
|
||||||
|
[(target, _)] = self.targets()
|
||||||
|
yield _item("greenhouse", target.partition(":")[2], "1")
|
||||||
|
self.state[f"ats:ok:{target}"] = self.options["run"]
|
||||||
|
|
||||||
|
real = Store.upsert_role
|
||||||
|
|
||||||
|
def upsert(self: Store, item: Item, sub: object = None) -> bool:
|
||||||
|
if item.source_id.startswith("greenhouse:b:"):
|
||||||
|
raise ValueError("a row the store refuses")
|
||||||
|
return real(self, item, sub) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||||
|
monkeypatch.setattr(Store, "upsert_role", upsert)
|
||||||
|
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert result["roles"] == 2
|
||||||
|
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
|
||||||
|
assert "ats:ok:greenhouse:b" not in store.state
|
||||||
|
since, reason = pull.failing_boards(store)["greenhouse:b"]
|
||||||
|
assert since and reason == "ValueError: a row the store refuses"
|
||||||
|
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
since = _days_ago(2)
|
||||||
|
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||||
|
{"since": since, "reason": "404 Not Found"})
|
||||||
|
_answer(monkeypatch, ([_row("1")], True))
|
||||||
|
|
||||||
|
def refuse(self: Store, item: Item, sub: object = None) -> bool:
|
||||||
|
raise ValueError("no")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Store, "upsert_role", refuse)
|
||||||
|
_pull(acme)
|
||||||
|
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
|
||||||
|
assert "ats:ok:workable:acme" not in acme.state
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------- the settle speaks
|
||||||
|
|
||||||
|
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
|
||||||
|
store: Store) -> None:
|
||||||
|
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||||
|
board="greenhouse")
|
||||||
|
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
|
||||||
|
board="greenhouse")
|
||||||
|
for n in range(80):
|
||||||
|
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
|
||||||
|
run = pull.claim_run(store)
|
||||||
|
store.state["ats:ok:greenhouse:big"] = run
|
||||||
|
store.state["ats:complete:greenhouse:big"] = "1"
|
||||||
|
store.state["ats:ids:greenhouse:big"] = json.dumps(
|
||||||
|
[f"greenhouse:big:{n}" for n in range(20)])
|
||||||
|
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
|
||||||
|
{"since": _days_ago(1), "reason": "404 Not Found"})
|
||||||
|
|
||||||
|
said: list[object] = []
|
||||||
|
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 60)
|
||||||
|
[notice] = said
|
||||||
|
assert notice.code == "pull.settled" # type: ignore[attr-defined]
|
||||||
|
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
|
||||||
|
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
|
||||||
|
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||||
|
said: list[object] = []
|
||||||
|
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
|
||||||
|
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert said == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_finder_that_raises_releases_the_pull_claim(
|
||||||
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def broken(store: Store, subs: object) -> dict[str, object]:
|
||||||
|
raise RuntimeError("reach broke")
|
||||||
|
|
||||||
|
monkeypatch.setattr(pull, "reaches", broken)
|
||||||
|
# The worker runs the chain's finder once the job is done; here at once.
|
||||||
|
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
|
||||||
|
with pytest.raises(RuntimeError, match="reach broke"):
|
||||||
|
cascade.run_pull(FakeConfig(), acme, context=context)
|
||||||
|
assert pull.running(acme) == ""
|
||||||
|
|
||||||
|
# And the job itself lets go when the chain cannot be built.
|
||||||
|
def no_chain(run: str) -> object:
|
||||||
|
raise RuntimeError("no chain")
|
||||||
|
|
||||||
|
monkeypatch.setattr(cascade, "pull_chain", no_chain)
|
||||||
|
with pytest.raises(RuntimeError, match="no chain"):
|
||||||
|
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
|
||||||
|
assert pull.running(acme) == ""
|
||||||
@@ -371,3 +371,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
|||||||
assert store.roles() == []
|
assert store.roles() == []
|
||||||
# And a batch that starts after the switch reads nothing.
|
# And a batch that starts after the switch reads nothing.
|
||||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------- failures and the guard
|
||||||
|
|
||||||
|
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The provider `fake` of `_fake_provider`, answering with an error."""
|
||||||
|
_fake_provider(monkeypatch, [], complete=True)
|
||||||
|
|
||||||
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||||
|
raise ValueError("hits renamed")
|
||||||
|
|
||||||
|
readers.PROVIDERS["fake"].page = page
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_whose_provider_raises_is_recorded_failing(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_broken_provider(monkeypatch)
|
||||||
|
result = _pull(store)
|
||||||
|
assert result["unanswered"] == 1
|
||||||
|
since, reason = pull.failing_boards(store)["fake:cph"]
|
||||||
|
assert since and reason == "ValueError: hits renamed"
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||||
|
|
||||||
|
# Failing for `STALE_DAYS` closes its roles, as a board's.
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||||
|
timespec="seconds")
|
||||||
|
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
|
||||||
|
_pull(store)
|
||||||
|
assert store.roles() == []
|
||||||
|
|
||||||
|
# Its next answer removes the record and reopens what it lists.
|
||||||
|
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||||
|
_pull(store)
|
||||||
|
assert pull.failing_boards(store) == {}
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_that_lists_nothing_after_forty_is_suspect(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
forty = [f"h{n}" for n in range(40)]
|
||||||
|
_fake_provider(monkeypatch, forty, complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_fake_provider(monkeypatch, [], complete=True)
|
||||||
|
result = _pull(store)
|
||||||
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||||
|
assert len(store.roles()) == 40
|
||||||
|
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
|
||||||
Reference in new issue
Block a user