Guard each target in a batch, close remote-unasked and narrowed roles by age, settle failing searches, and raise one notice per settle naming what needs a look
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
24a165818c
commit
9c8a610787
4 files changed
+576
-115
No files matched your search
+10
-2
@@ -180,14 +180,22 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
|
||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||
than report a pull that never happened; `app job pull` claims the run and
|
||||
starts the chain on a queue of its own instead of calling this.
|
||||
starts the chain on a queue of its own instead of calling this. Anything
|
||||
that raises once the run is claimed, building the chain or handing it
|
||||
over, releases the claim first, or the next pull would wait
|
||||
`pull.ABANDONED` for a run that never started.
|
||||
"""
|
||||
run = pull.claim_run(store)
|
||||
if not run:
|
||||
since = pull.running(store)
|
||||
context.notice(already_running(since))
|
||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||
if not context.task(pull_chain(run)):
|
||||
try:
|
||||
started = context.task(pull_chain(run))
|
||||
except Exception:
|
||||
pull.release_run(store, run)
|
||||
raise
|
||||
if not started:
|
||||
pull.release_run(store, run)
|
||||
raise RuntimeError("no queue to start the pull cascade on")
|
||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||
|
||||
+282
-107
@@ -18,11 +18,14 @@ is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
runner as a failure, not as a list to close against.
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
||||
reach, so its close keeps open the roles a person acted on: those with a vote
|
||||
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
||||
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
||||
says nothing of the remote roles either, so they stay open too and close only
|
||||
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
||||
that lists under a tenth of what the same read listed last time is recorded
|
||||
by the runner as a failure, not as a list to close against (`_runner.read`).
|
||||
|
||||
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
||||
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
||||
@@ -33,14 +36,21 @@ not swept clears them.
|
||||
|
||||
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
||||
the roles matching a word, which says nothing of the roles it did not list,
|
||||
so it closes nothing; only an unfiltered or a place-filtered complete read
|
||||
closes.
|
||||
so it closes by list nothing: a role of such a board closes once the board
|
||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
||||
reopens what it lists. A target that raises in its batch after its read, a
|
||||
row that does not store, is recorded as failing the same way and the batch
|
||||
goes on with the next one (`run_batch`).
|
||||
|
||||
The settle says what needs a look in one notice (`settled`): the targets
|
||||
that did not answer or were read incomplete, and the boards that closed more
|
||||
than `CLOSE_NOTICE` roles or more than half of theirs.
|
||||
|
||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||
set only while the previous run is settled, in one statement), so the
|
||||
@@ -60,7 +70,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs.reach import reaches
|
||||
@@ -91,6 +101,19 @@ STALE_DAYS = 21
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The same for a search, kept by the `search` plugin.
|
||||
SEARCH_FAILING = "search:failing:"
|
||||
|
||||
#: A board that closes more roles than this in one settle is named in the
|
||||
#: settle's notice, and so is one that closes more than half of its open
|
||||
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
||||
#: one is an ordinary day.
|
||||
CLOSE_NOTICE = 50
|
||||
CLOSE_SHARE_MIN = 10
|
||||
|
||||
#: How many targets a notice names in its text; its detail names them all.
|
||||
NOTICE_NAMES = 10
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
@@ -142,27 +165,38 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
||||
`close_boards` compares against. `run` is a marker the caller claimed
|
||||
already (the `pull` job, `app job pull`); without one the finder claims
|
||||
its own, and queues nothing when another run is in flight. A run with
|
||||
nothing to read is settled here, since no settle will see a batch of it."""
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
nothing to read is settled here, since no settle will see a batch of it,
|
||||
and so is a run whose finder raises: no batch and no settle would follow,
|
||||
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
||||
held = run
|
||||
try:
|
||||
seed_searches(config, store)
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True)
|
||||
for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
held = run
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches: list[dict[str, Any]] = [
|
||||
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
except Exception:
|
||||
if held:
|
||||
release_run(store, held)
|
||||
raise
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
@@ -185,9 +219,19 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||
plugin as its `reach` option.
|
||||
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
||||
which the runner's 10% guard accepts the answer it kept refusing.
|
||||
|
||||
Each target is guarded on its own. A read that fails is the plugin's to
|
||||
record; anything raised after it, a row `upsert_role` refuses or a bug
|
||||
past the read, is recorded here under `<source>:failing:<target>` with
|
||||
its reason, and the batch goes on with the next target. Otherwise one
|
||||
bad row would cost the targets after it in the batch their read on every
|
||||
pull, since batches are stable, and leave no trace on any of them. A
|
||||
Kill (`Cancelled`) still ends the batch.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.readers import _runner
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
@@ -200,24 +244,39 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run,
|
||||
"stale_days": STALE_DAYS}
|
||||
items: Any = None
|
||||
try:
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label,
|
||||
"params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
# A board's params are its filter; a reader's key is kept with
|
||||
# the searches' (`sources.search.<name>`).
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
items = plugin.fetch()
|
||||
for item in items:
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
except Cancelled:
|
||||
raise
|
||||
except Exception as exc:
|
||||
log.warning("pull: %s failed after its read (%s); on to the next target",
|
||||
target, exc, exc_info=True)
|
||||
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
||||
finally:
|
||||
# The plugin's generator holds its HTTP client until it is closed.
|
||||
if items is not None and hasattr(items, "close"):
|
||||
items.close()
|
||||
return {"roles": stored, "run": run}
|
||||
|
||||
|
||||
@@ -235,7 +294,8 @@ def _run_of(tally: Tally, context: RunContext) -> str:
|
||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""The run's settle: close every board this run answered in full and
|
||||
stopped listing a role from, and tally what happened.
|
||||
stopped listing a role from, tally what happened, and say what needs a
|
||||
look.
|
||||
|
||||
Reads `self.state` rather than the batch jobs' own results, because a
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
@@ -243,27 +303,41 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
||||
its roles (`failing_boards`). How a read that answered closes is
|
||||
`_close_read`'s.
|
||||
|
||||
The stamps are compared with this run's own marker, the one its batches
|
||||
carried, never with the stored `pull:run`, which a later run may have
|
||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
||||
which lets the next pull start.
|
||||
read) closes nothing by list or by age and says nothing. Last, the run is
|
||||
marked settled, which lets the next pull start.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
switched-off one is closed again here; those closes are asked for and not
|
||||
named in the notice.
|
||||
|
||||
The counts go into the result (`unanswered` counts boards and searches
|
||||
whose stamp is not this run's), and one notice (`settled`) names every
|
||||
target unanswered, every board read incomplete other than one mid-sweep,
|
||||
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
||||
half the open roles it had, when there is any. The result alone was
|
||||
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
||||
|
||||
The `prep` stage that follows maps what this run stored and folds the
|
||||
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
||||
and the roles this run added join theirs.
|
||||
"""
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
answered = searched = incomplete = closed = 0
|
||||
unanswered: list[str] = []
|
||||
partial: list[str] = []
|
||||
many: list[tuple[str, int, int]] = []
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
run_marker = _run_of(tally, context)
|
||||
open_before = _open_counts(store) if run_marker else {}
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
@@ -273,60 +347,158 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
unanswered.append(sub.target)
|
||||
since = failing.get(sub.target, ("", ""))[0]
|
||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
is_filtered = bool(store.state.get(f"{source}:filtered:{sub.target}", ""))
|
||||
if store.state.get(f"{source}:swept:{sub.target}", ""):
|
||||
closed += _close_swept(store, source, sub, seen_ids, is_filtered)
|
||||
continue
|
||||
store.state.pop(f"{source}:missing:{sub.target}", None)
|
||||
if is_filtered:
|
||||
closed += _close_unlisted(store, source, seen_ids, sub.id)
|
||||
else:
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if source == "search":
|
||||
searched += 1
|
||||
else:
|
||||
answered += 1
|
||||
read = _close_read(store, source, sub, stale)
|
||||
if read is None:
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||
partial.append(sub.target)
|
||||
continue
|
||||
shut = read
|
||||
closed += shut
|
||||
before = open_before.get(sub.id, 0)
|
||||
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
||||
many.append((sub.target, shut, before))
|
||||
if searched:
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
if unanswered or partial or many:
|
||||
context.notice(settled(unanswered, partial, many, failing,
|
||||
boards=answered, searches=searched, closed=closed))
|
||||
release_run(store, run_marker)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int) -> int:
|
||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||
"""Close what one target's read of this run says has closed, and return
|
||||
how many; None when the read was incomplete, which closes nothing.
|
||||
|
||||
A read narrowed by terms closes only the roles it has not listed for
|
||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||
two sweeps in a row missed (`_close_swept`); any other complete read
|
||||
clears that record and closes what it did not list, all of it unfiltered
|
||||
(`Store.mark_closed`), filtered sparing the roles acted on
|
||||
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
||||
(`remote_unasked`) spares the remote ones as well, and closes those by
|
||||
age instead."""
|
||||
state, target = store.state, sub.target
|
||||
if not state.get(f"{source}:complete:{target}", ""):
|
||||
return None
|
||||
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
||||
if state.get(f"{source}:narrowed:{target}", ""):
|
||||
return _close_aged(store, source, sub.id, seen_ids, stale)
|
||||
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
||||
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
||||
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
||||
if spare_remote else 0)
|
||||
if state.get(f"{source}:swept:{target}", ""):
|
||||
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
||||
spare_remote=spare_remote)
|
||||
state.pop(f"{source}:missing:{target}", None)
|
||||
if is_filtered:
|
||||
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
||||
spare_remote=spare_remote)
|
||||
return store.mark_closed(source, seen_ids, sub.id)
|
||||
|
||||
|
||||
def _open_counts(store: Store) -> dict[int, int]:
|
||||
"""`subscription id -> open roles`, before the settle closes any, for the
|
||||
share a board closed."""
|
||||
with store.transaction() as conn:
|
||||
return {int(row[0]): int(row[1]) for row in conn.execute(
|
||||
"""SELECT subscription_id, count(*) FROM roles
|
||||
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
||||
GROUP BY subscription_id""")}
|
||||
|
||||
|
||||
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
||||
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
||||
closed: int) -> Notice:
|
||||
"""The notice of a settle that has something to look at: the targets
|
||||
that did not answer (with the reason their failing record gives), the
|
||||
boards read incomplete, and the boards that closed many roles, each
|
||||
as `closed of open`, with the totals. The text names the first
|
||||
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
||||
def named(target: str) -> str:
|
||||
reason = failing.get(target, ("", ""))[1]
|
||||
return f"{target} ({reason})" if reason else target
|
||||
|
||||
groups = [("Not answering", [named(t) for t in unanswered]),
|
||||
("Read incomplete", list(incomplete)),
|
||||
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
||||
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
||||
f"{closed:,} roles."]
|
||||
detail: list[str] = []
|
||||
for title, names in groups:
|
||||
if not names:
|
||||
continue
|
||||
shown = ", ".join(names[:NOTICE_NAMES])
|
||||
more = len(names) - NOTICE_NAMES
|
||||
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
||||
else "."))
|
||||
detail += [f"{title}: {name}" for name in names]
|
||||
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
||||
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
||||
|
||||
|
||||
#: The roles a partial read never closes by its list: one with a vote (any,
|
||||
#: neutral included) or an application (which holds the notes).
|
||||
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)"""
|
||||
|
||||
#: The remote roles a filtered read that could not ask for them leaves alone.
|
||||
_NOT_REMOTE = " AND remote != 'remote'"
|
||||
|
||||
|
||||
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
||||
*, spare_remote: bool = False) -> int:
|
||||
"""`Store.mark_closed` for a filtered read: close every open role of the
|
||||
board it did not list, except a role someone acted on, one with a vote
|
||||
(any, neutral included) or an application (which holds the notes). A
|
||||
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
||||
filtered read cannot tell a role gone from a role outside its filter, and
|
||||
a role acted on stays until the board is read without the filter or
|
||||
fails for `STALE_DAYS`."""
|
||||
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
||||
well, since the read could not ask for them."""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?))
|
||||
AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""",
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
||||
+ (_NOT_REMOTE if spare_remote else ""),
|
||||
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
||||
stale: str, *, remote_only: bool = False) -> int:
|
||||
"""Close the open roles of one board that its read did not list and that
|
||||
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
||||
writes on every listing): the rule for what a read cannot speak for. A
|
||||
read narrowed by terms closes all such roles, as a search does; with
|
||||
`remote_only`, a filtered read that could not ask for remote closes only
|
||||
its remote roles so, sparing those acted on as its list does."""
|
||||
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
||||
with store.transaction() as conn:
|
||||
return conn.execute(
|
||||
f"""UPDATE roles SET closed_at = ?
|
||||
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
||||
AND last_seen < ?
|
||||
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
||||
(store.now(), source, subscription_id, stale,
|
||||
json.dumps(list(seen_ids)))).rowcount
|
||||
|
||||
|
||||
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
filtered: bool) -> int:
|
||||
filtered: bool, *, spare_remote: bool = False) -> int:
|
||||
"""The close of a board whose complete read finished a sweep across
|
||||
pulls: its open roles missing from `seen_ids` that also are in
|
||||
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
||||
@@ -334,15 +506,14 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
missed may only have shifted back past the cursor when roles closed
|
||||
between pulls, so one sweep is not evidence that it is gone. A filtered
|
||||
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
||||
stays open, and stays in `missing`."""
|
||||
stays open, and stays in `missing`, and so does a remote role with
|
||||
`spare_remote`."""
|
||||
key = f"{source}:missing:{sub.target}"
|
||||
try:
|
||||
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
||||
except (TypeError, ValueError):
|
||||
earlier = set()
|
||||
acted_on = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
||||
AND uid NOT IN (SELECT role_uid FROM applications
|
||||
WHERE role_uid IS NOT NULL)""" if filtered else ""
|
||||
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
||||
with store.transaction() as conn:
|
||||
missed = [row[0] for row in conn.execute(
|
||||
"""SELECT source_id FROM roles
|
||||
@@ -365,16 +536,20 @@ def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
"""`target -> (since, reason)` for every `ats` board and every search
|
||||
whose last fetch failed: when the failures began, and what the latest
|
||||
one said. No board host is also a search provider, so a board's target
|
||||
(`host:slug`) and a search's (`provider:name`) never collide in one
|
||||
dict."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
for prefix in (FAILING, SEARCH_FAILING):
|
||||
for key, value in store.state_items(prefix).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user