Close a search's roles by age only: a search read complete no longer closes every role its query stopped returning, since a search is never a company's whole list

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:32:06 +02:00
1 parent 1127d06f03
commit 19fb73caa0
2 files changed
+63 -25

No files matched your search

+28 -23
View File
@@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list,
so it closes by list nothing: a role of such a board closes once the board
has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board or a
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
`search:failing:<target>`) closes all of its roles, and its next answer
reopens what it lists. A target that raises in its batch after its read, a
row that does not store, is recorded as failing the same way and the batch
goes on with the next one (`run_batch`).
A search never closes by its list, even one stamped complete because it
read as many results as its provider stated: a search is never a company's
whole list, so a role that drops out of its results may still be open. Its
roles close once no search has listed them for `STALE_DAYS`
(`Store.close_unseen`). A board or a search that has failed on every fetch
for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
all of its roles, and its next answer reopens what it lists. A target that
raises in its batch after its read, a row that does not store, is recorded
as failing the same way and the batch goes on with the next one
(`run_batch`).
The settle says what needs a look in one notice (`settled`): the targets
that did not answer or were read incomplete, and the boards that closed more
@@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
search that has failed on every fetch since `STALE_DAYS` ago closes all
its roles (`failing_boards`). How a read that answered closes is
`_close_read`'s.
that answered incompletely is not either. A search never closes by its
list, complete or not: its roles close once no search has listed them
for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
board or a search that has failed on every fetch since `STALE_DAYS` ago
closes all its roles (`failing_boards`). How a board's read that
answered closes is `_close_read`'s.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
@@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
unanswered.append(sub.target)
since = failing.get(sub.target, ("", ""))[0]
shut = store.close_subscription(sub.id) if since and since < stale else 0
elif source == "search":
# Never a company's whole list, complete or not: a search's
# roles close by age alone (`close_unseen`, after the loop).
searched += 1
continue
else:
if source == "search":
searched += 1
else:
answered += 1
answered += 1
read = _close_read(store, source, sub, stale)
if read is None:
if source == "ats":
incomplete += 1
if not store.state.get(f"ats:sweep:{sub.target}"):
partial.append(sub.target)
incomplete += 1
if not store.state.get(f"ats:sweep:{sub.target}"):
partial.append(sub.target)
continue
shut = read
closed += shut
@@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
"""Close what one target's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing.
"""Close what one board's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing. A
search's read never comes here (see `close_boards`).
A read narrowed by terms closes only the roles it has not listed for
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
+35 -2
View File
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from pipeline import RunContext, Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
return closed
def test_a_complete_search_closes_what_it_stopped_listing(
def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A search is never a company's whole list: one that read as many
results as its provider stated is stamped complete, and a role it
stopped listing still stays open until no search has listed it for
`STALE_DAYS`."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
_fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert store.state["search:complete:fake:cph"] == "1"
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
assert (result["searches"], result["closed"]) == (1, 0)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Fifteen of twenty roles dropping out of a complete search closed
them, and named the search as closing many; now they stay open and the
settle has nothing to say."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
twenty = [f"h{n}" for n in range(20)]
_fake_provider(monkeypatch, twenty, complete=True)
_pull(store)
_fake_provider(monkeypatch, twenty[:5], complete=True)
said: list[object] = []
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
assert len(store.roles()) == 20
assert said == []
def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",