Close a search's roles by age only: a search read complete no longer closes every role its query stopped returning, since a search is never a company's whole list

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:32:06 +02:00
1 parent 1127d06f03
commit 19fb73caa0
2 files changed
+63 -25

No files matched your search

+28 -23
View File
@@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list,
so it closes by list nothing: a role of such a board closes once the board so it closes by list nothing: a role of such a board closes once the board
has not listed it for `STALE_DAYS`, the rule searches have. has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see A search never closes by its list, even one stamped complete because it
`plugins/search.py` for how each provider knows). One that did not closes its read as many results as its provider stated: a search is never a company's
roles only once no search has seen them for `STALE_DAYS`. A board or a whole list, so a role that drops out of its results may still be open. Its
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`, roles close once no search has listed them for `STALE_DAYS`
`search:failing:<target>`) closes all of its roles, and its next answer (`Store.close_unseen`). A board or a search that has failed on every fetch
reopens what it lists. A target that raises in its batch after its read, a for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
row that does not store, is recorded as failing the same way and the batch all of its roles, and its next answer reopens what it lists. A target that
goes on with the next one (`run_batch`). raises in its batch after its read, a row that does not store, is recorded
as failing the same way and the batch goes on with the next one
(`run_batch`).
The settle says what needs a look in one notice (`settled`): the targets The settle says what needs a look in one notice (`settled`): the targets
that did not answer or were read incomplete, and the boards that closed more that did not answer or were read incomplete, and the boards that closed more
@@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
had already parsed stamped — see the module docstring. A board whose had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its `ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only that answered incompletely is not either. A search never closes by its
when that list was complete, and otherwise by `STALE_DAYS`. A board or a list, complete or not: its roles close once no search has listed them
search that has failed on every fetch since `STALE_DAYS` ago closes all for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
its roles (`failing_boards`). How a read that answered closes is board or a search that has failed on every fetch since `STALE_DAYS` ago
`_close_read`'s. closes all its roles (`failing_boards`). How a board's read that
answered closes is `_close_read`'s.
The stamps are compared with this run's own marker, the one its batches The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have carried, never with the stored `pull:run`, which a later run may have
@@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
unanswered.append(sub.target) unanswered.append(sub.target)
since = failing.get(sub.target, ("", ""))[0] since = failing.get(sub.target, ("", ""))[0]
shut = store.close_subscription(sub.id) if since and since < stale else 0 shut = store.close_subscription(sub.id) if since and since < stale else 0
elif source == "search":
# Never a company's whole list, complete or not: a search's
# roles close by age alone (`close_unseen`, after the loop).
searched += 1
continue
else: else:
if source == "search": answered += 1
searched += 1
else:
answered += 1
read = _close_read(store, source, sub, stale) read = _close_read(store, source, sub, stale)
if read is None: if read is None:
if source == "ats": incomplete += 1
incomplete += 1 if not store.state.get(f"ats:sweep:{sub.target}"):
if not store.state.get(f"ats:sweep:{sub.target}"): partial.append(sub.target)
partial.append(sub.target)
continue continue
shut = read shut = read
closed += shut closed += shut
@@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None: def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
"""Close what one target's read of this run says has closed, and return """Close what one board's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing. how many; None when the read was incomplete, which closes nothing. A
search's read never comes here (see `close_boards`).
A read narrowed by terms closes only the roles it has not listed for A read narrowed by terms closes only the roles it has not listed for
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
+35 -2
View File
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402 from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402 from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402 from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402 from pipeline import RunContext, Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402 from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402 from prepped import upsert # noqa: E402
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
return closed return closed
def test_a_complete_search_closes_what_it_stopped_listing( def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A search is never a company's whole list: one that read as many
results as its provider stated is stamped complete, and a role it
stopped listing still stays open until no search has listed it for
`STALE_DAYS`."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"}) params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True) _fake_provider(monkeypatch, ["h1", "h2"], complete=True)
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
_fake_provider(monkeypatch, ["h2"], complete=True) _fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store) result = _pull(store)
assert store.state["search:complete:fake:cph"] == "1"
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
assert (result["searches"], result["closed"]) == (1, 0)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"} assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1) assert (result["searches"], result["closed"]) == (1, 1)
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Fifteen of twenty roles dropping out of a complete search closed
them, and named the search as closing many; now they stay open and the
settle has nothing to say."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
twenty = [f"h{n}" for n in range(20)]
_fake_provider(monkeypatch, twenty, complete=True)
_pull(store)
_fake_provider(monkeypatch, twenty[:5], complete=True)
said: list[object] = []
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
assert len(store.roles()) == 20
assert said == []
def test_an_incomplete_search_closes_nothing_until_stale( def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",