diff --git a/app/jobs/pull.py b/app/jobs/pull.py index cec1eea..44542f6 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list, so it closes by list nothing: a role of such a board closes once the board has not listed it for `STALE_DAYS`, the rule searches have. -A search closes the same way when it listed everything that matches (see -`plugins/search.py` for how each provider knows). One that did not closes its -roles only once no search has seen them for `STALE_DAYS`. A board or a -search that has failed on every fetch for `STALE_DAYS` (`ats:failing:`, -`search:failing:`) closes all of its roles, and its next answer -reopens what it lists. A target that raises in its batch after its read, a -row that does not store, is recorded as failing the same way and the batch -goes on with the next one (`run_batch`). +A search never closes by its list, even one stamped complete because it +read as many results as its provider stated: a search is never a company's +whole list, so a role that drops out of its results may still be open. Its +roles close once no search has listed them for `STALE_DAYS` +(`Store.close_unseen`). A board or a search that has failed on every fetch +for `STALE_DAYS` (`ats:failing:`, `search:failing:`) closes +all of its roles, and its next answer reopens what it lists. A target that +raises in its batch after its read, a row that does not store, is recorded +as failing the same way and the batch goes on with the next one +(`run_batch`). The settle says what needs a look in one notice (`settled`): the targets that did not answer or were read incomplete, and the boards that closed more @@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *, had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed by its list, and a board - that answered incompletely is not either. A search closes by its list only - when that list was complete, and otherwise by `STALE_DAYS`. A board or a - search that has failed on every fetch since `STALE_DAYS` ago closes all - its roles (`failing_boards`). How a read that answered closes is - `_close_read`'s. + that answered incompletely is not either. A search never closes by its + list, complete or not: its roles close once no search has listed them + for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A + board or a search that has failed on every fetch since `STALE_DAYS` ago + closes all its roles (`failing_boards`). How a board's read that + answered closes is `_close_read`'s. The stamps are compared with this run's own marker, the one its batches carried, never with the stored `pull:run`, which a later run may have @@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *, unanswered.append(sub.target) since = failing.get(sub.target, ("", ""))[0] shut = store.close_subscription(sub.id) if since and since < stale else 0 + elif source == "search": + # Never a company's whole list, complete or not: a search's + # roles close by age alone (`close_unseen`, after the loop). + searched += 1 + continue else: - if source == "search": - searched += 1 - else: - answered += 1 + answered += 1 read = _close_read(store, source, sub, stale) if read is None: - if source == "ats": - incomplete += 1 - if not store.state.get(f"ats:sweep:{sub.target}"): - partial.append(sub.target) + incomplete += 1 + if not store.state.get(f"ats:sweep:{sub.target}"): + partial.append(sub.target) continue shut = read closed += shut @@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None: - """Close what one target's read of this run says has closed, and return - how many; None when the read was incomplete, which closes nothing. + """Close what one board's read of this run says has closed, and return + how many; None when the read was incomplete, which closes nothing. A + search's read never comes here (see `close_boards`). A read narrowed by terms closes only the roles it has not listed for `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what diff --git a/tests/test_search.py b/tests/test_search.py index d9c13c0..a19699a 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 -from pipeline import Tally # noqa: E402 +from pipeline import RunContext, Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 from prepped import upsert # noqa: E402 @@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]: return closed -def test_a_complete_search_closes_what_it_stopped_listing( +def test_a_complete_search_closes_nothing_by_its_list_only_by_age( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """A search is never a company's whole list: one that read as many + results as its provider stated is stamped complete, and a role it + stopped listing still stays open until no search has listed it for + `STALE_DAYS`.""" store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=True) @@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing( _fake_provider(monkeypatch, ["h2"], complete=True) result = _pull(store) + assert store.state["search:complete:fake:cph"] == "1" + assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} + assert (result["searches"], result["closed"]) == (1, 0) + + old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() + with store.transaction() as conn: + conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) + result = _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} assert (result["searches"], result["closed"]) == (1, 1) +def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """Fifteen of twenty roles dropping out of a complete search closed + them, and named the search as closing many; now they stay open and the + settle has nothing to say.""" + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", + params={"q": "backend"}) + twenty = [f"h{n}" for n in range(20)] + _fake_provider(monkeypatch, twenty, complete=True) + _pull(store) + _fake_provider(monkeypatch, twenty[:5], complete=True) + said: list[object] = [] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type] + context=RunContext(on_notice=said.append)) + assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0) + assert len(store.roles()) == 20 + assert said == [] + + def test_an_incomplete_search_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",