Guard each target in a batch, close remote-unasked and narrowed roles by age, settle failing searches, and raise one notice per settle naming what needs a look
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
24a165818c
commit
9c8a610787
4 files changed
+576
-115
No files matched your search
+10
-2
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
|
||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||
than report a pull that never happened; `app job pull` claims the run and
|
||||
starts the chain on a queue of its own instead of calling this.
|
||||
starts the chain on a queue of its own instead of calling this. Anything
|
||||
that raises once the run is claimed, building the chain or handing it
|
||||
over, releases the claim first, or the next pull would wait
|
||||
`pull.ABANDONED` for a run that never started.
|
||||
"""
|
||||
run = pull.claim_run(store)
|
||||
if not run:
|
||||
since = pull.running(store)
|
||||
context.notice(already_running(since))
|
||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||
if not context.task(pull_chain(run)):
|
||||
try:
|
||||
started = context.task(pull_chain(run))
|
||||
except Exception:
|
||||
pull.release_run(store, run)
|
||||
raise
|
||||
if not started:
|
||||
pull.release_run(store, run)
|
||||
raise RuntimeError("no queue to start the pull cascade on")
|
||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||
|
||||
+282
-107
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
||||
reach, so its close keeps open the roles a person acted on: those with a vote
|
||||
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
||||
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
||||
says nothing of the remote roles either, so they stay open too and close only
|
||||
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
||||
that lists under a tenth of what the same read listed last time is recorded
|
||||
by the runner as a failure, not as a list to close against (`_runner.read`).
|
||||
|
||||
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||
@@ -33,14 +36,21 @@ not swept clears them.
|
||||
|
||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||
the roles matching a word, which says nothing of the roles it did not list,
|
||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
||||
closes.
|
||||
so it closes by list nothing: a role of such a board closes once the board
|
||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
||||
reopens what it lists. A target that raises in its batch after its read, a
|
||||
row that does not store, is recorded as failing the same way and the batch
|
||||
goes on with the next one (`run_batch`).
|
||||
|
||||
The settle says what needs a look in one notice (`settled`): the targets
|
||||
that did not answer or were read incomplete, and the boards that closed more
|
||||
than `CLOSE_NOTICE` roles or more than half of theirs.
|
||||
|
||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||
set only while the previous run is settled, in one statement), so the
|
||||
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs.reach import reaches
|
||||
@@ -91,6 +101,19 @@ STALE_DAYS = 21
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The same for a search, kept by the `search` plugin.
|
||||
SEARCH_FAILING = "search:failing:"
|
||||
|
||||
#: A board that closes more roles than this in one settle is named in the
|
||||
#: settle's notice, and so is one that closes more than half of its open
|
||||
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
||||
#: one is an ordinary day.
|
||||
CLOSE_NOTICE = 50
|
||||
CLOSE_SHARE_MIN = 10
|
||||
|
||||
#: How many targets a notice names in its text; its detail names them all.
|
||||
NOTICE_NAMES = 10
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
||||
`close_boards` compares against. `run` is a marker the caller claimed
|
||||
already (the `pull` job, `app job pull`); without one the finder claims
|
||||
its own, and queues nothing when another run is in flight. A run with
|
||||
nothing to read is settled here, since no settle will see a batch of it."""
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
nothing to read is settled here, since no settle will see a batch of it,
|
||||
and so is a run whose finder raises: no batch and no settle would follow,
|
||||
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
||||
held = run
|
||||
try:
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True)
|
||||
for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
held = run
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches: list[dict[str, Any]] = [
|
||||
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
except Exception:
|
||||
if held:
|
||||
release_run(store, held)
|
||||
raise
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||
plugin as its `reach` option.
|
||||
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
||||
which the runner's 10% guard accepts the answer it kept refusing.
|
||||
|
||||
Each target is guarded on its own. A read that fails is the plugin's to
|
||||
record; anything raised after it, a row `upsert_role` refuses or a bug
|
||||
past the read, is recorded here under `<source>:failing:<target>` with
|
||||
its reason, and the batch goes on with the next target. Otherwise one
|
||||
bad row would cost the targets after it in the batch their read on every
|
||||
pull, since batches are stable, and leave no trace on any of them. A
|
||||
Kill (`Cancelled`) still ends the batch.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.readers import _runner
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run,
|
||||
"stale_days": STALE_DAYS}
|
||||
items: Any = None
|
||||
try:
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label,
|
||||
"params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
items = plugin.fetch()
|
||||
for item in items:
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
except Cancelled:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("pull: %s failed after its read (%s); on to the next target",
|
||||
target, exc, exc_info=True)
|
||||
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
||||
finally:
|
||||
# The plugin's generator holds its HTTP client until it is closed.
|
||||
if items is not None and hasattr(items, "close"):
|
||||
items.close()
|
||||
return {"roles": stored, "run": run}
|
||||
|
||||
|
||||
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
|
||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""The run's settle: close every board this run answered in full and
|
||||
stopped listing a role from, and tally what happened.
|
||||
stopped listing a role from, tally what happened, and say what needs a
|
||||
look.
|
||||
|
||||
Reads `self.state` rather than the batch jobs' own results, because a
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
||||
its roles (`failing_boards`). How a read that answered closes is
|
||||
`_close_read`'s.
|
||||
|
||||
The stamps are compared with this run's own marker, the one its batches
|
||||
carried, never with the stored `pull:run`, which a later run may have
|
||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
||||
which lets the next pull start.
|
||||
read) closes nothing by list or by age and says nothing. Last, the run is
|
||||
marked settled, which lets the next pull start.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
switched-off one is closed again here; those closes are asked for and not
|
||||
named in the notice.
|
||||
|
||||
The counts go into the result (`unanswered` counts boards and searches
|
||||
whose stamp is not this run's), and one notice (`settled`) names every
|
||||
target unanswered, every board read incomplete other than one mid-sweep,
|
||||
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
||||
half the open roles it had, when there is any. The result alone was
|
||||
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
||||
|
||||
The `prep` stage that follows maps what this run stored and folds the
|
||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||
and the roles this run added join theirs.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
answered = searched = incomplete = closed = 0
|
||||
unanswered: list[str] = []
|
||||
partial: list[str] = []
|
||||
many: list[tuple[str, int, int]] = []
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
run_marker = _run_of(tally, context)
|
||||
open_before = _open_counts(store) if run_marker else {}
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
unanswered.append(sub.target)
|
||||
since = failing.get(sub.target, ("", ""))[0]
|
||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
||||
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
||||
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
||||
continue
|
||||
store.state.pop(f"{source}:missing:{sub.target}", None)
|
||||
if is_filtered:
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
else:
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if source == "search":
|
||||
searched += 1
|
||||
else:
|
||||
answered += 1
|
||||
read = _close_read(store, source, sub, stale)
|
||||
if read is None:
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||
partial.append(sub.target)
|
||||
continue
|
||||
shut = read
|
||||
closed += shut
|
||||
before = open_before.get(sub.id, 0)
|
||||
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
||||
many.append((sub.target, shut, before))
|
||||
if searched:
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
if unanswered or partial or many:
|
||||
context.notice(settled(unanswered, partial, many, failing,
|
||||
boards=answered, searches=searched, closed=closed))
|
||||
release_run(store, run_marker)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||
"""Close what one target's read of this run says has closed, and return
|
||||
how many; None when the read was incomplete, which closes nothing.
|
||||
|
||||
A read narrowed by terms closes only the roles it has not listed for
|
||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||
two sweeps in a row missed (`_close_swept`); any other complete read
|
||||
clears that record and closes what it did not list, all of it unfiltered
|
||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||
age instead."""
|
||||
state, target = store.state, sub.target
|
||||
if not state.get(f"{source}:complete:{target}", ""):
|
||||
return None
|
||||
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
||||
if state.get(f"{source}:narrowed:{target}", ""):
|
||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||
if spare_remote else 0)
|
||||
if state.get(f"{source}:swept:{target}", ""):
|
||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||
spare_remote=spare_remote)
|
||||
state.pop(f"{source}:missing:{target}", None)
|
||||
if is_filtered:
|
||||
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
||||
spare_remote=spare_remote)
|
||||
return store.mark_closed(source, seen_ids, sub.id)
|
||||
|
||||
|
||||
def _open_counts(store: Store) -> dict[int, int]:
|
||||
"""`subscription id -> open roles`, before the settle closes any, for the
|
||||
share a board closed."""
|
||||
with store.transaction() as conn:
|
||||
return {int(row[0]): int(row[1]) for row in conn.execute(
|
||||
"""SELECT subscription_id, count(*) FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id""")}
|
||||
|
||||
|
||||
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
||||
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
||||
closed: int) -> Notice:
|
||||
"""The notice of a settle that has something to look at: the targets
|
||||
that did not answer (with the reason their failing record gives), the
|
||||
boards read incomplete, and the boards that closed many roles, each
|
||||
as `closed of open`, with the totals. The text names the first
|
||||
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
||||
def named(target: str) -> str:
|
||||
reason = failing.get(target, ("", ""))[1]
|
||||
return f"{target} ({reason})" if reason else target
|
||||
|
||||
groups = [("Not answering", [named(t) for t in unanswered]),
|
||||
("Read incomplete", list(incomplete)),
|
||||
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
||||
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
||||
f"{closed:,} roles."]
|
||||
detail: list[str] = []
|
||||
for title, names in groups:
|
||||
if not names:
|
||||
continue
|
||||
shown = ", ".join(names[:NOTICE_NAMES])
|
||||
more = len(names) - NOTICE_NAMES
|
||||
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
||||
else "."))
|
||||
detail += [f"{title}: {name}" for name in names]
|
||||
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
||||
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
||||
|
||||
|
||||
#: The roles a partial read never closes by its list: one with a vote (any,
|
||||
#: neutral included) or an application (which holds the notes).
|
||||
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)"""
|
||||
|
||||
#: The remote roles a filtered read that could not ask for them leaves alone.
|
||||
_NOT_REMOTE = " AND remote != 'remote'"
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
||||
*, spare_remote: bool = False) -> int:
|
||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||
board it did not list, except a role someone acted on, one with a vote
|
||||
(any, neutral included) or an application (which holds the notes). A
|
||||
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
||||
filtered read cannot tell a role gone from a role outside its filter, and
|
||||
a role acted on stays until the board is read without the filter or
|
||||
fails for `STALE_DAYS`."""
|
||||
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
||||
well, since the read could not ask for them."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""",
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
||||
+ (_NOT_REMOTE if spare_remote else ""),
|
||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||
stale: str, *, remote_only: bool = False) -> int:
|
||||
"""Close the open roles of one board that its read did not list and that
|
||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||
writes on every listing): the rule for what a read cannot speak for. A
|
||||
read narrowed by terms closes all such roles, as a search does; with
|
||||
`remote_only`, a filtered read that could not ask for remote closes only
|
||||
its remote roles so, sparing those acted on as its list does."""
|
||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND last_seen < ?
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||
(store.now(), source, subscription_id, stale,
|
||||
json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
filtered: bool) -> int:
|
||||
filtered: bool, *, spare_remote: bool = False) -> int:
|
||||
"""The close of a board whose complete read finished a sweep across
|
||||
pulls: its open roles missing from `seen_ids` that also are in
|
||||
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
missed may only have shifted back past the cursor when roles closed
|
||||
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||
stays open, and stays in `missing`."""
|
||||
stays open, and stays in `missing`, and so does a remote role with
|
||||
`spare_remote`."""
|
||||
key = f"{source}:missing:{sub.target}"
|
||||
try:
|
||||
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||
except (TypeError, ValueError):
|
||||
earlier = set()
|
||||
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
||||
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
||||
with store.transaction() as conn:
|
||||
missed = [row[0] for row in conn.execute(
|
||||
"""SELECT source_id FROM roles
|
||||
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
"""`target -> (since, reason)` for every `ats` board and every search
|
||||
whose last fetch failed: when the failures began, and what the latest
|
||||
one said. No board host is also a search provider, so a board's target
|
||||
(`host:slug`) and a search's (`provider:name`) never collide in one
|
||||
dict."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
for prefix in (FAILING, SEARCH_FAILING):
|
||||
for key, value in store.state_items(prefix).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
+231
-6
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -21,8 +22,8 @@ if str(ROOT) not in sys.path:
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
||||
from app.models import Application # noqa: E402
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import Filters, Reads, _runner # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app import prep # noqa: E402
|
||||
from app.jobs import cascade, pull # noqa: E402
|
||||
@@ -142,7 +143,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||
"roles": 1, "closed": 1}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||
|
||||
@@ -163,7 +164,7 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
|
||||
"roles": 0, "closed": 0}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
|
||||
@@ -256,7 +257,7 @@ def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||
_answer(monkeypatch, ValueError("gone"))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||
assert since and reason == "ValueError: gone"
|
||||
|
||||
@@ -486,7 +487,7 @@ def test_run_a_settle_does_not_close_on_run_b_ids(
|
||||
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
||||
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
assert pull.running(acme) == b["run"]
|
||||
|
||||
@@ -507,3 +508,227 @@ def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> No
|
||||
context = RunContext(waited_for=(failed,))
|
||||
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
||||
assert result["boards"] == 1 and pull.running(acme) == ""
|
||||
|
||||
|
||||
# -------------------------------------------------------- losses, guarded
|
||||
|
||||
def _days_ago(days: float) -> str:
|
||||
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
|
||||
with store.transaction() as conn:
|
||||
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
|
||||
[(value, i) for i in source_ids])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def big(store: Store) -> Store:
|
||||
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||
board="greenhouse")
|
||||
return store
|
||||
|
||||
|
||||
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([], True))
|
||||
result = _pull(big)
|
||||
assert len(_open(big)) == 40
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
|
||||
|
||||
|
||||
def test_a_whole_read_of_nothing_after_five_closes_the_board(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([], True))
|
||||
assert _pull(big)["closed"] == 5
|
||||
assert _open(big) == set()
|
||||
|
||||
|
||||
def test_a_suspect_answer_is_accepted_after_stale_days(
|
||||
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
||||
_pull(big)
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(big)
|
||||
assert len(_open(big)) == 40
|
||||
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
|
||||
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
|
||||
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
|
||||
result = _pull(big)
|
||||
assert _open(big) == {"greenhouse:big:1"}
|
||||
assert (result["unanswered"], result["closed"]) == (0, 39)
|
||||
assert pull.failing_boards(big) == {}
|
||||
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
|
||||
|
||||
|
||||
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
|
||||
"""A whole-listing host `fake` that filters by country and, per `remote`,
|
||||
on remote; its feed lists the ids in the returned list."""
|
||||
listed: list[str] = []
|
||||
module: object = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
|
||||
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||
module.READS = Reads( # type: ignore[attr-defined]
|
||||
scope="board", listing="whole", text="listing", complete="whole", places="one",
|
||||
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
|
||||
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
||||
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
|
||||
listed).encode())
|
||||
return listed
|
||||
|
||||
|
||||
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
listed = _remote_host(monkeypatch, remote="none")
|
||||
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
|
||||
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
|
||||
|
||||
def pull_once() -> dict[str, object]:
|
||||
run = pull.claim_run(store)
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
|
||||
reach=reach)
|
||||
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
||||
|
||||
listed[:] = ["1", "2", "3", "4"]
|
||||
pull_once()
|
||||
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
|
||||
listed[:] = ["1"]
|
||||
assert pull_once()["closed"] == 1
|
||||
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
|
||||
assert store.state["ats:remote_unasked:fake:acme"] == "1"
|
||||
|
||||
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
|
||||
assert pull_once()["closed"] == 1
|
||||
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
|
||||
|
||||
|
||||
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
||||
_pull(acme)
|
||||
|
||||
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
||||
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
|
||||
|
||||
monkeypatch.setattr(_runner, "read_listing", fake)
|
||||
assert _pull(acme)["closed"] == 0
|
||||
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
|
||||
assert _pull(acme)["closed"] == 1
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
||||
|
||||
|
||||
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
|
||||
for target in targets:
|
||||
slug = target.partition(":")[2]
|
||||
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
|
||||
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
[(target, _)] = self.targets()
|
||||
yield _item("greenhouse", target.partition(":")[2], "1")
|
||||
self.state[f"ats:ok:{target}"] = self.options["run"]
|
||||
|
||||
real = Store.upsert_role
|
||||
|
||||
def upsert(self: Store, item: Item, sub: object = None) -> bool:
|
||||
if item.source_id.startswith("greenhouse:b:"):
|
||||
raise ValueError("a row the store refuses")
|
||||
return real(self, item, sub) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
monkeypatch.setattr(Store, "upsert_role", upsert)
|
||||
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
|
||||
|
||||
assert result["roles"] == 2
|
||||
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
|
||||
assert "ats:ok:greenhouse:b" not in store.state
|
||||
since, reason = pull.failing_boards(store)["greenhouse:b"]
|
||||
assert since and reason == "ValueError: a row the store refuses"
|
||||
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
|
||||
|
||||
|
||||
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
since = _days_ago(2)
|
||||
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||
{"since": since, "reason": "404 Not Found"})
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
|
||||
def refuse(self: Store, item: Item, sub: object = None) -> bool:
|
||||
raise ValueError("no")
|
||||
|
||||
monkeypatch.setattr(Store, "upsert_role", refuse)
|
||||
_pull(acme)
|
||||
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
|
||||
assert "ats:ok:workable:acme" not in acme.state
|
||||
|
||||
|
||||
# ------------------------------------------------------- the settle speaks
|
||||
|
||||
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
|
||||
store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
||||
board="greenhouse")
|
||||
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
|
||||
board="greenhouse")
|
||||
for n in range(80):
|
||||
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
|
||||
run = pull.claim_run(store)
|
||||
store.state["ats:ok:greenhouse:big"] = run
|
||||
store.state["ats:complete:greenhouse:big"] = "1"
|
||||
store.state["ats:ids:greenhouse:big"] = json.dumps(
|
||||
[f"greenhouse:big:{n}" for n in range(20)])
|
||||
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
|
||||
{"since": _days_ago(1), "reason": "404 Not Found"})
|
||||
|
||||
said: list[object] = []
|
||||
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
|
||||
context=RunContext(on_notice=said.append))
|
||||
assert (result["unanswered"], result["closed"]) == (1, 60)
|
||||
[notice] = said
|
||||
assert notice.code == "pull.settled" # type: ignore[attr-defined]
|
||||
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
|
||||
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
|
||||
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
||||
said: list[object] = []
|
||||
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
||||
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
|
||||
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||
context=RunContext(on_notice=said.append))
|
||||
assert said == []
|
||||
|
||||
|
||||
def test_a_finder_that_raises_releases_the_pull_claim(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def broken(store: Store, subs: object) -> dict[str, object]:
|
||||
raise RuntimeError("reach broke")
|
||||
|
||||
monkeypatch.setattr(pull, "reaches", broken)
|
||||
# The worker runs the chain's finder once the job is done; here at once.
|
||||
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
|
||||
with pytest.raises(RuntimeError, match="reach broke"):
|
||||
cascade.run_pull(FakeConfig(), acme, context=context)
|
||||
assert pull.running(acme) == ""
|
||||
|
||||
# And the job itself lets go when the chain cannot be built.
|
||||
def no_chain(run: str) -> object:
|
||||
raise RuntimeError("no chain")
|
||||
|
||||
monkeypatch.setattr(cascade, "pull_chain", no_chain)
|
||||
with pytest.raises(RuntimeError, match="no chain"):
|
||||
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
|
||||
assert pull.running(acme) == ""
|
||||
@@ -371,3 +371,56 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
|
||||
assert store.roles() == []
|
||||
# And a batch that starts after the switch reads nothing.
|
||||
assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ------------------------------------------------- failures and the guard
|
||||
|
||||
def _broken_provider(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The provider `fake` of `_fake_provider`, answering with an error."""
|
||||
_fake_provider(monkeypatch, [], complete=True)
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||
raise ValueError("hits renamed")
|
||||
|
||||
readers.PROVIDERS["fake"].page = page
|
||||
|
||||
|
||||
def test_a_search_whose_provider_raises_is_recorded_failing(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
_pull(store)
|
||||
_broken_provider(monkeypatch)
|
||||
result = _pull(store)
|
||||
assert result["unanswered"] == 1
|
||||
since, reason = pull.failing_boards(store)["fake:cph"]
|
||||
assert since and reason == "ValueError: hits renamed"
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||
|
||||
# Failing for `STALE_DAYS` closes its roles, as a board's.
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||
timespec="seconds")
|
||||
store.state["search:failing:fake:cph"] = json.dumps({"since": old, "reason": reason})
|
||||
_pull(store)
|
||||
assert store.roles() == []
|
||||
|
||||
# Its next answer removes the record and reopens what it lists.
|
||||
_fake_provider(monkeypatch, ["h1"], complete=True)
|
||||
_pull(store)
|
||||
assert pull.failing_boards(store) == {}
|
||||
assert {r.source_id for r in store.roles()} == {"fake:h1"}
|
||||
|
||||
|
||||
def test_a_search_that_lists_nothing_after_forty_is_suspect(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||
params={"q": "backend"})
|
||||
forty = [f"h{n}" for n in range(40)]
|
||||
_fake_provider(monkeypatch, forty, complete=True)
|
||||
_pull(store)
|
||||
_fake_provider(monkeypatch, [], complete=True)
|
||||
result = _pull(store)
|
||||
assert (result["unanswered"], result["closed"]) == (1, 0)
|
||||
assert len(store.roles()) == 40
|
||||
assert "under 10% of the 40" in pull.failing_boards(store)["fake:cph"][1]
|
||||
Reference in new issue
Block a user