diff --git a/app/jobs/pull.py b/app/jobs/pull.py index cec1eea..44542f6 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list, so it closes by list nothing: a role of such a board closes once the board has not listed it for `STALE_DAYS`, the rule searches have. -A search closes the same way when it listed everything that matches (see -`plugins/search.py` for how each provider knows). One that did not closes its -roles only once no search has seen them for `STALE_DAYS`. A board or a -search that has failed on every fetch for `STALE_DAYS` (`ats:failing:`, -`search:failing:`) closes all of its roles, and its next answer -reopens what it lists. A target that raises in its batch after its read, a -row that does not store, is recorded as failing the same way and the batch -goes on with the next one (`run_batch`). +A search never closes by its list, even one stamped complete because it +read as many results as its provider stated: a search is never a company's +whole list, so a role that drops out of its results may still be open. Its +roles close once no search has listed them for `STALE_DAYS` +(`Store.close_unseen`). A board or a search that has failed on every fetch +for `STALE_DAYS` (`ats:failing:`, `search:failing:`) closes +all of its roles, and its next answer reopens what it lists. A target that +raises in its batch after its read, a row that does not store, is recorded +as failing the same way and the batch goes on with the next one +(`run_batch`). The settle says what needs a look in one notice (`settled`): the targets that did not answer or were read incomplete, and the boards that closed more @@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *, had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed by its list, and a board - that answered incompletely is not either. A search closes by its list only - when that list was complete, and otherwise by `STALE_DAYS`. A board or a - search that has failed on every fetch since `STALE_DAYS` ago closes all - its roles (`failing_boards`). How a read that answered closes is - `_close_read`'s. + that answered incompletely is not either. A search never closes by its + list, complete or not: its roles close once no search has listed them + for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A + board or a search that has failed on every fetch since `STALE_DAYS` ago + closes all its roles (`failing_boards`). How a board's read that + answered closes is `_close_read`'s. The stamps are compared with this run's own marker, the one its batches carried, never with the stored `pull:run`, which a later run may have @@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *, unanswered.append(sub.target) since = failing.get(sub.target, ("", ""))[0] shut = store.close_subscription(sub.id) if since and since < stale else 0 + elif source == "search": + # Never a company's whole list, complete or not: a search's + # roles close by age alone (`close_unseen`, after the loop). + searched += 1 + continue else: - if source == "search": - searched += 1 - else: - answered += 1 + answered += 1 read = _close_read(store, source, sub, stale) if read is None: - if source == "ats": - incomplete += 1 - if not store.state.get(f"ats:sweep:{sub.target}"): - partial.append(sub.target) + incomplete += 1 + if not store.state.get(f"ats:sweep:{sub.target}"): + partial.append(sub.target) continue shut = read closed += shut @@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *, def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None: - """Close what one target's read of this run says has closed, and return - how many; None when the read was incomplete, which closes nothing. + """Close what one board's read of this run says has closed, and return + how many; None when the read was incomplete, which closes nothing. A + search's read never comes here (see `close_boards`). A read narrowed by terms closes only the roles it has not listed for `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index cb13887..7f9c698 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -556,17 +556,21 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None A whole listing is complete whenever it parses. A paged one reads each series' pages, `limits.page_delay` apart, until the host has no next - page, the stated total is reached, a page brings nothing new (a host - that ignores the cursor, incomplete) or `limits.max_pages` pages in all. - A series is complete when it read as many distinct result ids as its - stated total, or, without a total, when it ended on the host's last - page; never for `complete="never"`. The read is complete when every - series is. A 429 after the first page ends the read, incomplete. + page, the stated total is reached, the read stalls or `limits.max_pages` + pages in all. A stall is a page that brings no new id while the reader + still hands a cursor: any such page on a host that states no total (it + ignores the cursor), and with a total one the reader hands over empty + (it saw the page come again). A series is complete when it read as many distinct result + ids as its stated total, or, without a total, when it ended on the + host's last page; never after a stall, and never for + `complete="never"`. The read is complete when every series is. A 429 + after the first page ends the read, incomplete. With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns the `sweep` to keep, and the next pull, handed it back, continues from its series and cursor; the sweep is complete when it reaches the end, - and `ids` then hold every id it listed, and the listing is `swept`. A + and `ids` then hold every id it listed, and the listing is `swept`. The + cursor comes back as JSON gives it, a tuple as a list (`_storable`). A read at the reader's `ceiling` is not swept: the host answers nothing past it. @@ -633,7 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None if cursor is None or (isinstance(total, int) and len(seen) >= total): ended = True break - if total is None and len(seen) == before: + # Nothing new while the reader says more follow. Without a total + # the host ignores the cursor, and the read would never end. With + # one, only a page the reader hands over empty is a stall (it saw + # the page come again, as SuccessFactors does): a page of ids + # already read is what a resumed sweep starts on when the list + # grew at the top since, and the read goes on from there. + if len(seen) == before and (total is None or not results): stalled = True break if stop: @@ -663,13 +673,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None def _storable(cursor: Any) -> bool: - """Whether a cursor survives a trip through JSON in state.""" + """Whether a cursor survives a trip through JSON in state. A tuple comes + back as a list (Workday's `(offset, total)`, SuccessFactors' `(start + row, first id)`), which counts as surviving: the next pull hands the + reader its cursor as JSON gives it back, and every paged reader takes + the list as it took the tuple.""" try: - return json.loads(json.dumps(cursor)) == cursor + return json.loads(json.dumps(cursor)) == _as_json(cursor) except (TypeError, ValueError): return False +def _as_json(value: Any) -> Any: + """`value` with every tuple in it a list, as JSON hands it back.""" + if isinstance(value, tuple | list): + return [_as_json(item) for item in value] + if isinstance(value, dict): + return {key: _as_json(item) for key, item in value.items()} + return value + + # ------------------------------------------------------------ with state def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: diff --git a/plugins/readers/lever.py b/plugins/readers/lever.py index 7ee003b..b161296 100644 --- a/plugins/readers/lever.py +++ b/plugins/readers/lever.py @@ -112,9 +112,13 @@ def _places(categories: dict[str, Any]) -> list[Place]: def parse(body: bytes, slug: str) -> list[Row]: + """The board's postings, a JSON list. Any other answer (an error + document sent with a 200) raises, so the read fails and closes nothing + rather than reading as a board with no roles.""" data = json.loads(body) if not isinstance(data, list): - return [] + raise ValueError(f"Lever's answer is a JSON {type(data).__name__}, " + "not a list of postings") rows = [] for j in data: title = (j.get("text") or "").strip() diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index 4762c82..43d4f5d 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -110,20 +110,31 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: `items[0]`. Without the `expand` only the facets come back; asking for `requisitionList.secondaryLocations` gives the roles and each one's other locations, which the plain `requisitionList` leaves out. Past the last role - the list is empty.""" + the list is empty; an answer without the wrapper raises (`_search`).""" offset = int(cursor or 0) site = _parts(slug)[1] data = json.loads(get( f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" f"&expand=requisitionList.secondaryLocations" f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}")) - search = (data.get("items") or [{}])[0] + search = _search(data) roles = search.get("requisitionList") or [] # `_paged` counts `id`, which Oracle spells `Id`. results = [{"id": str(role.get("Id") or "")} for role in roles] return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None +def _search(data: Any) -> dict[str, Any]: + """The search wrapper of a listing's answer, `items[0]`. Every answer has + one, past the last role too; one without (an error document sent with a + 200) raises, so the read fails and closes nothing rather than reading as + an empty last page with no total, which is a complete read.""" + items = data.get("items") if isinstance(data, dict) else None + if not isinstance(items, list) or not items or not isinstance(items[0], dict): + raise ValueError("Oracle's answer has no search wrapper in items") + return items[0] + + #: How many countries `locationsFacet` lists at most; a list this long may #: leave some out. FACET_LIMIT = 10 @@ -162,7 +173,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: def parse(body: bytes, slug: str) -> list[Row]: - return rows((json.loads(body).get("items") or [{}])[0], slug) + return rows(_search(json.loads(body)), slug) def _text(value: Any) -> str: diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index 03dfee2..3ad376d 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: - """One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page - before)`: the next start row is what this page returned, and a page that - starts with the same role as the last one is the site ignoring `startrow`, - which ends the read (incomplete) instead of looping.""" + """One search page, in the one country `ask` names, matching its `query`. + The cursor is `(start row, first role of the page before)`, a list when + a sweep hands it back from state: the next start row is what this page + returned. A page that starts with the same role as the last one is the + site ignoring `startrow`: it comes back with no results and the cursor + it was asked with, which the runner reads as a stall that ends the read + incomplete. A cursor of None there would read as the last page, and, + on a site that states no total, as a complete read.""" start, previous = cursor or (0, None) country = ask.countries[0] if ask.countries else "" text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace") @@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: found = _entries(text) total = _total(text) if found and found[0]["id"] == previous: - return text, [], total, None + return text, [], total, (start, previous) more = bool(found) and (total is None or start + len(found) < total) return (text, [{"id": entry["id"]} for entry in found], total, (start + len(found), found[0]["id"]) if more else None) diff --git a/tests/test_ats_lever.py b/tests/test_ats_lever.py index 8377197..6629ea9 100644 --- a/tests/test_ats_lever.py +++ b/tests/test_ats_lever.py @@ -1,15 +1,20 @@ """The Lever host module's places, country and employment, against a small -saved board. No network. The pay and the plain row are in `test_ats.py`.""" +saved board, and an answer that is no board. No network. The pay and the +plain row are in `test_ats.py`.""" from __future__ import annotations +import json import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from plugins import ats # noqa: E402 from plugins.readers import Ask, lever # noqa: E402 BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes() @@ -33,6 +38,26 @@ def test_the_commitment_is_the_hosts_word() -> None: assert several["employment"] == "Full-time" and one["employment"] == "Permanent" +def test_an_answer_that_is_not_a_list_raises_and_an_empty_list_is_no_roles() -> None: + with pytest.raises(ValueError, match="JSON dict, not a list"): + lever.parse(b'{"ok": false, "error": "Document not found"}', "acme") + assert lever.parse(b"[]", "acme") == [] + + +def test_an_error_document_with_a_200_fails_the_read_and_stamps_nothing( + monkeypatch: pytest.MonkeyPatch) -> None: + """It used to read as a complete board with no roles, which the settle + closed whole under ten roles, where the 10% guard does not hold.""" + monkeypatch.setattr(ats, "_get", lambda *a, **kw: b'{"ok": false}') + source = ats.ATSSource({"boards": [{"url": "lever:acme", "label": "Acme", "params": {}}], + "run": "r"}) + source.state = {} + assert list(source.fetch()) == [] + assert "ats:ok:lever:acme" not in source.state + assert json.loads(source.state["ats:failing:lever:acme"])["reason"].startswith( + "ValueError: Lever's answer is a JSON dict") + + def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None: reads = lever.READS assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many") diff --git a/tests/test_ats_oracle.py b/tests/test_ats_oracle.py index f12d433..011cd3e 100644 --- a/tests/test_ats_oracle.py +++ b/tests/test_ats_oracle.py @@ -165,6 +165,28 @@ def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: N assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type] +@pytest.mark.parametrize("answer", [ + {}, {"items": []}, {"items": {}}, {"items": [None]}, [], {"title": "Unauthorized"}]) +def test_an_answer_without_the_search_wrapper_fails_the_read( + monkeypatch: pytest.MonkeyPatch, no_delay: None, answer: Any) -> None: + """It used to read as an empty last page with no total: a complete + read of a board with no roles.""" + monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(answer).encode()) + with pytest.raises(ValueError, match="no search wrapper in items"): + ats._fetch_rows(object(), f"oracle:{SLUG}") # type: ignore[arg-type] + + +def test_a_board_answering_without_the_wrapper_is_recorded_failing( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"title": "Unauthorized"}') + source = ats.ATSSource({"boards": [{"url": f"oracle:{SLUG}", "label": "Nokia", + "params": {}}], "run": "r"}) + source.state = {} + assert list(source.fetch()) == [] + assert f"ats:ok:oracle:{SLUG}" not in source.state + assert "no search wrapper" in json.loads(source.state[f"ats:failing:oracle:{SLUG}"])["reason"] + + # ----------------------------------------------------------------- detail def _detail_get(body: bytes, asked: list[str]) -> Any: diff --git a/tests/test_ats_successfactors.py b/tests/test_ats_successfactors.py index c3b4e4b..ea41762 100644 --- a/tests/test_ats_successfactors.py +++ b/tests/test_ats_successfactors.py @@ -1,15 +1,17 @@ """The SuccessFactors host: `plugins/readers/successfactors.py` against saved -career-site pages. No network. The fixtures are trimmed pages of Vestas and +career-site pages, and against written ones through a sweep and a pull with +its settle. No network. The fixtures are trimmed pages of Vestas and SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site with a path prefix), and the job pages of Vestas (`addressCountry`), Systematic (`streetAddress`) and Novo Nordisk (no microdata).""" from __future__ import annotations +import json import sys from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, ClassVar import httpx import pytest @@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from pipeline import Tally # noqa: E402 +from app.jobs import pull # noqa: E402 from app.jobs.discover import boards_in # noqa: E402 +from app.storage import Store # noqa: E402 from plugins import ats, readers # noqa: E402 from plugins.readers import successfactors # noqa: E402 FIXTURES = ROOT / "tests" / "fixtures" / "ats" +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +def _pull(store: Store) -> dict[str, Any]: + """One pull of every board, batches and settle, as `test_sweep_strikes`.""" + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + + def _read(name: str) -> str: return (FIXTURES / name).read_text(encoding="utf-8") @@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa assert len(asked) == 2 +def _untotalled(ids: list[int]) -> str: + """A table page that states no total.""" + return _table_page(ids, 0).split("", 1)[1] + + +def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None: + text = _untotalled([1_000_000_001, 1_000_000_002]) + _decoded, results, total, cursor = successfactors.page( + lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"]) + assert (results, total, cursor) == ([], None, (2, "1000000001")) + + +def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """The repeated first page used to end the read with no cursor, which a + site that states no total made complete, and the settle closed every + role past the first page.""" + store = Store(tmp_path / "roledesk.db") + store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme", + company="Acme", board="successfactors") + ids = [1_000_000_000 + n for n in range(4)] + site = {"ignores_startrow": False} + + def fake_get(client, url, host, slug, **kw): + start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"]) + return _untotalled(ids[start:start + 2]).encode() + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + _pull(store) + assert store.state["ats:complete:successfactors:careers.acme.com"] == "1" + assert len(store.roles()) == 4 + + site["ignores_startrow"] = True + result = _pull(store) + assert store.state["ats:complete:successfactors:careers.acme.com"] == "" + assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4) + + +def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back( + monkeypatch: pytest.MonkeyPatch) -> None: + """The cursor `(start row, first id)` is a tuple, kept in state as a + list. The sweep is kept, and the next pull reads on from it.""" + asked = _site(monkeypatch, 5, 2) + monkeypatch.setattr(ats, "MAX_PAGES", 2) + target = "successfactors:careers.acme.com" + state: dict[str, str] = {} + + def pull_once() -> list[Any]: + source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}], + "run": "r"}) + source.state = state + return list(source.fetch()) + + assert len(pull_once()) == 4 + assert state[f"ats:complete:{target}"] == "" + assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"] + assert len(pull_once()) == 1 + assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1") + assert json.loads(state[f"ats:ids:{target}"]) == [ + f"{target}:{1_000_000_000 + n}" for n in range(5)] + assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4] + + def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None: _site(monkeypatch, 0, 10) assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type] diff --git a/tests/test_ats_workday.py b/tests/test_ats_workday.py index a089e9b..9598b75 100644 --- a/tests/test_ats_workday.py +++ b/tests/test_ats_workday.py @@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing( assert len(rows) == 30 and not complete +def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + """The cursor `(offset, total)` is a tuple, kept in state as a list, and + the total later pages state as 0 is carried in it. The sweep is kept, + and the next pull reads on from it to the first page's total.""" + tenant = Tenant(45) + monkeypatch.setattr(ats, "_get", tenant) + monkeypatch.setattr(ats, "MAX_PAGES", 2) + target = f"workday:{SLUG}" + state: dict[str, str] = {} + + def pull_once() -> list[Any]: + source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}], + "run": "r"}) + source.state = state + return list(source.fetch()) + + assert len(pull_once()) == 40 + assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45] + assert len(pull_once()) == 5 + assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1") + assert len(json.loads(state[f"ats:ids:{target}"])) == 45 + assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40] + + def test_a_page_that_is_not_json_fails_the_read( monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: monkeypatch.setattr(ats, "_get", lambda *a, **k: b"maintenance") diff --git a/tests/test_runner.py b/tests/test_runner.py index 124cf4f..41380c2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -215,6 +215,46 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None: assert [index for index, _ in module.asked] == [0, 1] +def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None: + """A reader that sees a page come again hands it over empty with a + cursor, as SuccessFactors does. That ends the read incomplete, also when + the host states a total, where it used to run on to the page cap.""" + module = _paged([["1", "2"]] * 5, total=5) + seen: list[Any] = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + seen.append(cursor) + return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \ + else ([], [], 5, cursor) + + module.page = page + listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], False) + assert seen == [None, 1] + + +def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None: + """What a resumed sweep starts on when the list grew at the top since: + the page at the saved cursor holds only roles the sweep has. The read + goes on to the roles behind them.""" + pages = [["1", "2"], ["1", "2"], ["3"]] + module = _paged(pages, total=3) + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2", "3"], True) + assert [index for index, _ in module.asked] == [0, 1, 2] + + +def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None: + """`complete="end"`: the page past the last role is empty and hands no + cursor. That is the end, not a stall.""" + pages = [["1"], ["2"], []] + module = _paged(pages) + assert module.READS.complete == "end" + listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY) + assert (listing.ids, listing.complete) == (["1", "2"], True) + assert [index for index, _ in module.asked] == [0, 1, 2] + + def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: pages = [["1"], ["2"]] web = _pages_web(pages) @@ -260,6 +300,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None: assert state["ats:swept:fake:acme"] == "1" +def _tupled(pages: list[list[str]], total: int) -> Any: + """`_paged` with a cursor of Workday's kind, `(next page, total)`, which a + sweep hands back from state as a list. `cursors` are the ones it was + handed.""" + module = _paged(pages, total=total) + module.cursors = [] + + def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any: + module.cursors.append(cursor) + index, stated = cursor or (0, total) + ids = pages[index] + more = index + 1 < len(pages) + return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None + + module.page = page + return module + + +def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None: + """A tuple comes back from JSON as a list. The sweep is kept all the + same, the reader gets the list, and the second pull finishes it.""" + module = _tupled([["1"], ["2"], ["3"]], total=3) + state: dict[str, str] = {} + limits = _runner.Limits(max_pages=2, page_delay=0) + first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", first, first.ids) + assert (first.ids, first.complete) == (["1", "2"], False) + assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3] + + second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits) + _runner.stamp(state, "ats", "fake:acme", "r", second, second.ids) + assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True) + assert module.cursors == [None, (1, 3), [2, 3]] + assert "ats:sweep:fake:acme" not in state + + +def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None: + assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None) + assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533")) + assert _runner._storable({"after": (1, [2, (3,)])}) + assert not _runner._storable({1: "a"}) # the key comes back as "1" + assert not _runner._storable(object()) + + def test_a_board_read_in_one_pull_is_not_swept() -> None: pages = [["1"], ["2"]] state: dict[str, str] = {} diff --git a/tests/test_search.py b/tests/test_search.py index d9c13c0..a19699a 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402 from plugins.search import SearchSource # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 -from pipeline import Tally # noqa: E402 +from pipeline import RunContext, Tally # noqa: E402 from app.models import country_of, remote_of # noqa: E402 from app.storage import Store # noqa: E402 from prepped import upsert # noqa: E402 @@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]: return closed -def test_a_complete_search_closes_what_it_stopped_listing( +def test_a_complete_search_closes_nothing_by_its_list_only_by_age( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """A search is never a company's whole list: one that read as many + results as its provider stated is stamped complete, and a role it + stopped listing still stays open until no search has listed it for + `STALE_DAYS`.""" store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", params={"q": "backend"}) _fake_provider(monkeypatch, ["h1", "h2"], complete=True) @@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing( _fake_provider(monkeypatch, ["h2"], complete=True) result = _pull(store) + assert store.state["search:complete:fake:cph"] == "1" + assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"} + assert (result["searches"], result["closed"]) == (1, 0) + + old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat() + with store.transaction() as conn: + conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,)) + result = _pull(store) assert {r.source_id for r in store.roles()} == {"fake:h2"} assert (result["searches"], result["closed"]) == (1, 1) +def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + """Fifteen of twenty roles dropping out of a complete search closed + them, and named the search as closing many; now they stay open and the + settle has nothing to say.""" + store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", + params={"q": "backend"}) + twenty = [f"h{n}" for n in range(20)] + _fake_provider(monkeypatch, twenty, complete=True) + _pull(store) + _fake_provider(monkeypatch, twenty[:5], complete=True) + said: list[object] = [] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type] + result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type] + context=RunContext(on_notice=said.append)) + assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0) + assert len(store.roles()) == 20 + assert said == [] + + def test_an_incomplete_search_closes_nothing_until_stale( store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",