Close a search's roles by age only: a search read complete no longer closes every role its query stopped returning, since a search is never a company's whole list
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
1127d06f03
commit
19fb73caa0
2 files changed
+63
-25
No files matched your search
+28
-23
@@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list,
|
|||||||
so it closes by list nothing: a role of such a board closes once the board
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||||
|
|
||||||
A search closes the same way when it listed everything that matches (see
|
A search never closes by its list, even one stamped complete because it
|
||||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
read as many results as its provider stated: a search is never a company's
|
||||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
whole list, so a role that drops out of its results may still be open. Its
|
||||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
roles close once no search has listed them for `STALE_DAYS`
|
||||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
(`Store.close_unseen`). A board or a search that has failed on every fetch
|
||||||
reopens what it lists. A target that raises in its batch after its read, a
|
for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
|
||||||
row that does not store, is recorded as failing the same way and the batch
|
all of its roles, and its next answer reopens what it lists. A target that
|
||||||
goes on with the next one (`run_batch`).
|
raises in its batch after its read, a row that does not store, is recorded
|
||||||
|
as failing the same way and the batch goes on with the next one
|
||||||
|
(`run_batch`).
|
||||||
|
|
||||||
The settle says what needs a look in one notice (`settled`): the targets
|
The settle says what needs a look in one notice (`settled`): the targets
|
||||||
that did not answer or were read incomplete, and the boards that closed more
|
that did not answer or were read incomplete, and the boards that closed more
|
||||||
@@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
had already parsed stamped — see the module docstring. A board whose
|
had already parsed stamped — see the module docstring. A board whose
|
||||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||||
whole batch job never got to it — is never closed by its list, and a board
|
whole batch job never got to it — is never closed by its list, and a board
|
||||||
that answered incompletely is not either. A search closes by its list only
|
that answered incompletely is not either. A search never closes by its
|
||||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
list, complete or not: its roles close once no search has listed them
|
||||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
|
||||||
its roles (`failing_boards`). How a read that answered closes is
|
board or a search that has failed on every fetch since `STALE_DAYS` ago
|
||||||
`_close_read`'s.
|
closes all its roles (`failing_boards`). How a board's read that
|
||||||
|
answered closes is `_close_read`'s.
|
||||||
|
|
||||||
The stamps are compared with this run's own marker, the one its batches
|
The stamps are compared with this run's own marker, the one its batches
|
||||||
carried, never with the stored `pull:run`, which a later run may have
|
carried, never with the stored `pull:run`, which a later run may have
|
||||||
@@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
unanswered.append(sub.target)
|
unanswered.append(sub.target)
|
||||||
since = failing.get(sub.target, ("", ""))[0]
|
since = failing.get(sub.target, ("", ""))[0]
|
||||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||||
|
elif source == "search":
|
||||||
|
# Never a company's whole list, complete or not: a search's
|
||||||
|
# roles close by age alone (`close_unseen`, after the loop).
|
||||||
|
searched += 1
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
if source == "search":
|
answered += 1
|
||||||
searched += 1
|
|
||||||
else:
|
|
||||||
answered += 1
|
|
||||||
read = _close_read(store, source, sub, stale)
|
read = _close_read(store, source, sub, stale)
|
||||||
if read is None:
|
if read is None:
|
||||||
if source == "ats":
|
incomplete += 1
|
||||||
incomplete += 1
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
partial.append(sub.target)
|
||||||
partial.append(sub.target)
|
|
||||||
continue
|
continue
|
||||||
shut = read
|
shut = read
|
||||||
closed += shut
|
closed += shut
|
||||||
@@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
|
|
||||||
|
|
||||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||||
"""Close what one target's read of this run says has closed, and return
|
"""Close what one board's read of this run says has closed, and return
|
||||||
how many; None when the read was incomplete, which closes nothing.
|
how many; None when the read was incomplete, which closes nothing. A
|
||||||
|
search's read never comes here (see `close_boards`).
|
||||||
|
|
||||||
A read narrowed by terms closes only the roles it has not listed for
|
A read narrowed by terms closes only the roles it has not listed for
|
||||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||||
|
|||||||
+35
-2
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
|||||||
from plugins.search import SearchSource # noqa: E402
|
from plugins.search import SearchSource # noqa: E402
|
||||||
from app.config import SearchSeed # noqa: E402
|
from app.config import SearchSeed # noqa: E402
|
||||||
from app.jobs import pull # noqa: E402
|
from app.jobs import pull # noqa: E402
|
||||||
from pipeline import Tally # noqa: E402
|
from pipeline import RunContext, Tally # noqa: E402
|
||||||
from app.models import country_of, remote_of # noqa: E402
|
from app.models import country_of, remote_of # noqa: E402
|
||||||
from app.storage import Store # noqa: E402
|
from app.storage import Store # noqa: E402
|
||||||
from prepped import upsert # noqa: E402
|
from prepped import upsert # noqa: E402
|
||||||
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
|
|||||||
return closed
|
return closed
|
||||||
|
|
||||||
|
|
||||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
|
||||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A search is never a company's whole list: one that read as many
|
||||||
|
results as its provider stated is stamped complete, and a role it
|
||||||
|
stopped listing still stays open until no search has listed it for
|
||||||
|
`STALE_DAYS`."""
|
||||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
params={"q": "backend"})
|
params={"q": "backend"})
|
||||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||||
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
|
|||||||
|
|
||||||
_fake_provider(monkeypatch, ["h2"], complete=True)
|
_fake_provider(monkeypatch, ["h2"], complete=True)
|
||||||
result = _pull(store)
|
result = _pull(store)
|
||||||
|
assert store.state["search:complete:fake:cph"] == "1"
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
||||||
|
assert (result["searches"], result["closed"]) == (1, 0)
|
||||||
|
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
||||||
|
result = _pull(store)
|
||||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||||
assert (result["searches"], result["closed"]) == (1, 1)
|
assert (result["searches"], result["closed"]) == (1, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Fifteen of twenty roles dropping out of a complete search closed
|
||||||
|
them, and named the search as closing many; now they stay open and the
|
||||||
|
settle has nothing to say."""
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
twenty = [f"h{n}" for n in range(20)]
|
||||||
|
_fake_provider(monkeypatch, twenty, complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_fake_provider(monkeypatch, twenty[:5], complete=True)
|
||||||
|
said: list[object] = []
|
||||||
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||||
|
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
|
||||||
|
assert len(store.roles()) == 20
|
||||||
|
assert said == []
|
||||||
|
|
||||||
|
|
||||||
def test_an_incomplete_search_closes_nothing_until_stale(
|
def test_an_incomplete_search_closes_nothing_until_stale(
|
||||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
|||||||
Reference in new issue
Block a user