Merge sweeps that resume on a tuple cursor, a stalled read counted incomplete, searches closing by age only, and Lever and Oracle failing on a wrong-shaped answer (wp-rev-runner)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-10-05 09:56:07 +02:00
commit 62cf1c738c
11 files changed
+364 -47

No files matched your search

+28 -23
View File
@@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list,
so it closes by list nothing: a role of such a board closes once the board so it closes by list nothing: a role of such a board closes once the board
has not listed it for `STALE_DAYS`, the rule searches have. has not listed it for `STALE_DAYS`, the rule searches have.
A search closes the same way when it listed everything that matches (see A search never closes by its list, even one stamped complete because it
`plugins/search.py` for how each provider knows). One that did not closes its read as many results as its provider stated: a search is never a company's
roles only once no search has seen them for `STALE_DAYS`. A board or a whole list, so a role that drops out of its results may still be open. Its
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`, roles close once no search has listed them for `STALE_DAYS`
`search:failing:<target>`) closes all of its roles, and its next answer (`Store.close_unseen`). A board or a search that has failed on every fetch
reopens what it lists. A target that raises in its batch after its read, a for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
row that does not store, is recorded as failing the same way and the batch all of its roles, and its next answer reopens what it lists. A target that
goes on with the next one (`run_batch`). raises in its batch after its read, a row that does not store, is recorded
as failing the same way and the batch goes on with the next one
(`run_batch`).
The settle says what needs a look in one notice (`settled`): the targets The settle says what needs a look in one notice (`settled`): the targets
that did not answer or were read incomplete, and the boards that closed more that did not answer or were read incomplete, and the boards that closed more
@@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
had already parsed stamped — see the module docstring. A board whose had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its `ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed by its list, and a board whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only that answered incompletely is not either. A search never closes by its
when that list was complete, and otherwise by `STALE_DAYS`. A board or a list, complete or not: its roles close once no search has listed them
search that has failed on every fetch since `STALE_DAYS` ago closes all for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
its roles (`failing_boards`). How a read that answered closes is board or a search that has failed on every fetch since `STALE_DAYS` ago
`_close_read`'s. closes all its roles (`failing_boards`). How a board's read that
answered closes is `_close_read`'s.
The stamps are compared with this run's own marker, the one its batches The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have carried, never with the stored `pull:run`, which a later run may have
@@ -350,17 +353,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
unanswered.append(sub.target) unanswered.append(sub.target)
since = failing.get(sub.target, ("", ""))[0] since = failing.get(sub.target, ("", ""))[0]
shut = store.close_subscription(sub.id) if since and since < stale else 0 shut = store.close_subscription(sub.id) if since and since < stale else 0
elif source == "search":
# Never a company's whole list, complete or not: a search's
# roles close by age alone (`close_unseen`, after the loop).
searched += 1
continue
else: else:
if source == "search": answered += 1
searched += 1
else:
answered += 1
read = _close_read(store, source, sub, stale) read = _close_read(store, source, sub, stale)
if read is None: if read is None:
if source == "ats": incomplete += 1
incomplete += 1 if not store.state.get(f"ats:sweep:{sub.target}"):
if not store.state.get(f"ats:sweep:{sub.target}"): partial.append(sub.target)
partial.append(sub.target)
continue continue
shut = read shut = read
closed += shut closed += shut
@@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None: def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
"""Close what one target's read of this run says has closed, and return """Close what one board's read of this run says has closed, and return
how many; None when the read was incomplete, which closes nothing. how many; None when the read was incomplete, which closes nothing. A
search's read never comes here (see `close_boards`).
A read narrowed by terms closes only the roles it has not listed for A read narrowed by terms closes only the roles it has not listed for
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what `STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
+33 -10
View File
@@ -556,17 +556,21 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
A whole listing is complete whenever it parses. A paged one reads each A whole listing is complete whenever it parses. A paged one reads each
series' pages, `limits.page_delay` apart, until the host has no next series' pages, `limits.page_delay` apart, until the host has no next
page, the stated total is reached, a page brings nothing new (a host page, the stated total is reached, the read stalls or `limits.max_pages`
that ignores the cursor, incomplete) or `limits.max_pages` pages in all. pages in all. A stall is a page that brings no new id while the reader
A series is complete when it read as many distinct result ids as its still hands a cursor: any such page on a host that states no total (it
stated total, or, without a total, when it ended on the host's last ignores the cursor), and with a total one the reader hands over empty
page; never for `complete="never"`. The read is complete when every (it saw the page come again). A series is complete when it read as many distinct result
series is. A 429 after the first page ends the read, incomplete. ids as its stated total, or, without a total, when it ended on the
host's last page; never after a stall, and never for
`complete="never"`. The read is complete when every series is. A 429
after the first page ends the read, incomplete.
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
the `sweep` to keep, and the next pull, handed it back, continues from the `sweep` to keep, and the next pull, handed it back, continues from
its series and cursor; the sweep is complete when it reaches the end, its series and cursor; the sweep is complete when it reaches the end,
and `ids` then hold every id it listed, and the listing is `swept`. A and `ids` then hold every id it listed, and the listing is `swept`. The
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
read at the reader's `ceiling` is not swept: the host answers nothing read at the reader's `ceiling` is not swept: the host answers nothing
past it. past it.
@@ -633,7 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
if cursor is None or (isinstance(total, int) and len(seen) >= total): if cursor is None or (isinstance(total, int) and len(seen) >= total):
ended = True ended = True
break break
if total is None and len(seen) == before: # Nothing new while the reader says more follow. Without a total
# the host ignores the cursor, and the read would never end. With
# one, only a page the reader hands over empty is a stall (it saw
# the page come again, as SuccessFactors does): a page of ids
# already read is what a resumed sweep starts on when the list
# grew at the top since, and the read goes on from there.
if len(seen) == before and (total is None or not results):
stalled = True stalled = True
break break
if stop: if stop:
@@ -663,13 +673,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
def _storable(cursor: Any) -> bool: def _storable(cursor: Any) -> bool:
"""Whether a cursor survives a trip through JSON in state.""" """Whether a cursor survives a trip through JSON in state. A tuple comes
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
row, first id)`), which counts as surviving: the next pull hands the
reader its cursor as JSON gives it back, and every paged reader takes
the list as it took the tuple."""
try: try:
return json.loads(json.dumps(cursor)) == cursor return json.loads(json.dumps(cursor)) == _as_json(cursor)
except (TypeError, ValueError): except (TypeError, ValueError):
return False return False
def _as_json(value: Any) -> Any:
"""`value` with every tuple in it a list, as JSON hands it back."""
if isinstance(value, tuple | list):
return [_as_json(item) for item in value]
if isinstance(value, dict):
return {key: _as_json(item) for key, item in value.items()}
return value
# ------------------------------------------------------------ with state # ------------------------------------------------------------ with state
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
+5 -1
View File
@@ -112,9 +112,13 @@ def _places(categories: dict[str, Any]) -> list[Place]:
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
"""The board's postings, a JSON list. Any other answer (an error
document sent with a 200) raises, so the read fails and closes nothing
rather than reading as a board with no roles."""
data = json.loads(body) data = json.loads(body)
if not isinstance(data, list): if not isinstance(data, list):
return [] raise ValueError(f"Lever's answer is a JSON {type(data).__name__}, "
"not a list of postings")
rows = [] rows = []
for j in data: for j in data:
title = (j.get("text") or "").strip() title = (j.get("text") or "").strip()
+14 -3
View File
@@ -110,20 +110,31 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
`items[0]`. Without the `expand` only the facets come back; asking for `items[0]`. Without the `expand` only the facets come back; asking for
`requisitionList.secondaryLocations` gives the roles and each one's other `requisitionList.secondaryLocations` gives the roles and each one's other
locations, which the plain `requisitionList` leaves out. Past the last role locations, which the plain `requisitionList` leaves out. Past the last role
the list is empty.""" the list is empty; an answer without the wrapper raises (`_search`)."""
offset = int(cursor or 0) offset = int(cursor or 0)
site = _parts(slug)[1] site = _parts(slug)[1]
data = json.loads(get( data = json.loads(get(
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
f"&expand=requisitionList.secondaryLocations" f"&expand=requisitionList.secondaryLocations"
f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}")) f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}"))
search = (data.get("items") or [{}])[0] search = _search(data)
roles = search.get("requisitionList") or [] roles = search.get("requisitionList") or []
# `_paged` counts `id`, which Oracle spells `Id`. # `_paged` counts `id`, which Oracle spells `Id`.
results = [{"id": str(role.get("Id") or "")} for role in roles] results = [{"id": str(role.get("Id") or "")} for role in roles]
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
def _search(data: Any) -> dict[str, Any]:
"""The search wrapper of a listing's answer, `items[0]`. Every answer has
one, past the last role too; one without (an error document sent with a
200) raises, so the read fails and closes nothing rather than reading as
an empty last page with no total, which is a complete read."""
items = data.get("items") if isinstance(data, dict) else None
if not isinstance(items, list) or not items or not isinstance(items[0], dict):
raise ValueError("Oracle's answer has no search wrapper in items")
return items[0]
#: How many countries `locationsFacet` lists at most; a list this long may #: How many countries `locationsFacet` lists at most; a list this long may
#: leave some out. #: leave some out.
FACET_LIMIT = 10 FACET_LIMIT = 10
@@ -162,7 +173,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
def parse(body: bytes, slug: str) -> list[Row]: def parse(body: bytes, slug: str) -> list[Row]:
return rows((json.loads(body).get("items") or [{}])[0], slug) return rows(_search(json.loads(body)), slug)
def _text(value: Any) -> str: def _text(value: Any) -> str:
+9 -5
View File
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page """One search page, in the one country `ask` names, matching its `query`.
before)`: the next start row is what this page returned, and a page that The cursor is `(start row, first role of the page before)`, a list when
starts with the same role as the last one is the site ignoring `startrow`, a sweep hands it back from state: the next start row is what this page
which ends the read (incomplete) instead of looping.""" returned. A page that starts with the same role as the last one is the
site ignoring `startrow`: it comes back with no results and the cursor
it was asked with, which the runner reads as a stall that ends the read
incomplete. A cursor of None there would read as the last page, and,
on a site that states no total, as a complete read."""
start, previous = cursor or (0, None) start, previous = cursor or (0, None)
country = ask.countries[0] if ask.countries else "" country = ask.countries[0] if ask.countries else ""
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace") text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
found = _entries(text) found = _entries(text)
total = _total(text) total = _total(text)
if found and found[0]["id"] == previous: if found and found[0]["id"] == previous:
return text, [], total, None return text, [], total, (start, previous)
more = bool(found) and (total is None or start + len(found) < total) more = bool(found) and (total is None or start + len(found) < total)
return (text, [{"id": entry["id"]} for entry in found], total, return (text, [{"id": entry["id"]} for entry in found], total,
(start + len(found), found[0]["id"]) if more else None) (start + len(found), found[0]["id"]) if more else None)
+26 -1
View File
@@ -1,15 +1,20 @@
"""The Lever host module's places, country and employment, against a small """The Lever host module's places, country and employment, against a small
saved board. No network. The pay and the plain row are in `test_ats.py`.""" saved board, and an answer that is no board. No network. The pay and the
plain row are in `test_ats.py`."""
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from pathlib import Path from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from plugins import ats # noqa: E402
from plugins.readers import Ask, lever # noqa: E402 from plugins.readers import Ask, lever # noqa: E402
BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes() BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes()
@@ -33,6 +38,26 @@ def test_the_commitment_is_the_hosts_word() -> None:
assert several["employment"] == "Full-time" and one["employment"] == "Permanent" assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
def test_an_answer_that_is_not_a_list_raises_and_an_empty_list_is_no_roles() -> None:
with pytest.raises(ValueError, match="JSON dict, not a list"):
lever.parse(b'{"ok": false, "error": "Document not found"}', "acme")
assert lever.parse(b"[]", "acme") == []
def test_an_error_document_with_a_200_fails_the_read_and_stamps_nothing(
monkeypatch: pytest.MonkeyPatch) -> None:
"""It used to read as a complete board with no roles, which the settle
closed whole under ten roles, where the 10% guard does not hold."""
monkeypatch.setattr(ats, "_get", lambda *a, **kw: b'{"ok": false}')
source = ats.ATSSource({"boards": [{"url": "lever:acme", "label": "Acme", "params": {}}],
"run": "r"})
source.state = {}
assert list(source.fetch()) == []
assert "ats:ok:lever:acme" not in source.state
assert json.loads(source.state["ats:failing:lever:acme"])["reason"].startswith(
"ValueError: Lever's answer is a JSON dict")
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None: def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
reads = lever.READS reads = lever.READS
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many") assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
+22
View File
@@ -165,6 +165,28 @@ def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: N
assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type] assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type]
@pytest.mark.parametrize("answer", [
{}, {"items": []}, {"items": {}}, {"items": [None]}, [], {"title": "Unauthorized"}])
def test_an_answer_without_the_search_wrapper_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None, answer: Any) -> None:
"""It used to read as an empty last page with no total: a complete
read of a board with no roles."""
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(answer).encode())
with pytest.raises(ValueError, match="no search wrapper in items"):
ats._fetch_rows(object(), f"oracle:{SLUG}") # type: ignore[arg-type]
def test_a_board_answering_without_the_wrapper_is_recorded_failing(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"title": "Unauthorized"}')
source = ats.ATSSource({"boards": [{"url": f"oracle:{SLUG}", "label": "Nokia",
"params": {}}], "run": "r"})
source.state = {}
assert list(source.fetch()) == []
assert f"ats:ok:oracle:{SLUG}" not in source.state
assert "no search wrapper" in json.loads(source.state[f"ats:failing:oracle:{SLUG}"])["reason"]
# ----------------------------------------------------------------- detail # ----------------------------------------------------------------- detail
def _detail_get(body: bytes, asked: list[str]) -> Any: def _detail_get(body: bytes, asked: list[str]) -> Any:
+83 -2
View File
@@ -1,15 +1,17 @@
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved """The SuccessFactors host: `plugins/readers/successfactors.py` against saved
career-site pages. No network. The fixtures are trimmed pages of Vestas and career-site pages, and against written ones through a sweep and a pull with
its settle. No network. The fixtures are trimmed pages of Vestas and
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
with a path prefix), and the job pages of Vestas (`addressCountry`), with a path prefix), and the job pages of Vestas (`addressCountry`),
Systematic (`streetAddress`) and Novo Nordisk (no microdata).""" Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any, ClassVar
import httpx import httpx
import pytest import pytest
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.jobs import pull # noqa: E402
from app.jobs.discover import boards_in # noqa: E402 from app.jobs.discover import boards_in # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats, readers # noqa: E402 from plugins import ats, readers # noqa: E402
from plugins.readers import successfactors # noqa: E402 from plugins.readers import successfactors # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats" FIXTURES = ROOT / "tests" / "fixtures" / "ats"
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
def _pull(store: Store) -> dict[str, Any]:
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _read(name: str) -> str: def _read(name: str) -> str:
return (FIXTURES / name).read_text(encoding="utf-8") return (FIXTURES / name).read_text(encoding="utf-8")
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
assert len(asked) == 2 assert len(asked) == 2
def _untotalled(ids: list[int]) -> str:
"""A table page that states no total."""
return _table_page(ids, 0).split("</span>", 1)[1]
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
text = _untotalled([1_000_000_001, 1_000_000_002])
_decoded, results, total, cursor = successfactors.page(
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
assert (results, total, cursor) == ([], None, (2, "1000000001"))
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
"""The repeated first page used to end the read with no cursor, which a
site that states no total made complete, and the settle closed every
role past the first page."""
store = Store(tmp_path / "roledesk.db")
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
company="Acme", board="successfactors")
ids = [1_000_000_000 + n for n in range(4)]
site = {"ignores_startrow": False}
def fake_get(client, url, host, slug, **kw):
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
return _untotalled(ids[start:start + 2]).encode()
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
_pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
assert len(store.roles()) == 4
site["ignores_startrow"] = True
result = _pull(store)
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch) -> None:
"""The cursor `(start row, first id)` is a tuple, kept in state as a
list. The sweep is kept, and the next pull reads on from it."""
asked = _site(monkeypatch, 5, 2)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = "successfactors:careers.acme.com"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 4
assert state[f"ats:complete:{target}"] == ""
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
assert len(pull_once()) == 1
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert json.loads(state[f"ats:ids:{target}"]) == [
f"{target}:{1_000_000_000 + n}" for n in range(5)]
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None: def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
_site(monkeypatch, 0, 10) _site(monkeypatch, 0, 10)
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type] assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
+25
View File
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
assert len(rows) == 30 and not complete assert len(rows) == 30 and not complete
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
the total later pages state as 0 is carried in it. The sweep is kept,
and the next pull reads on from it to the first page's total."""
tenant = Tenant(45)
monkeypatch.setattr(ats, "_get", tenant)
monkeypatch.setattr(ats, "MAX_PAGES", 2)
target = f"workday:{SLUG}"
state: dict[str, str] = {}
def pull_once() -> list[Any]:
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
"run": "r"})
source.state = state
return list(source.fetch())
assert len(pull_once()) == 40
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
assert len(pull_once()) == 5
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
def test_a_page_that_is_not_json_fails_the_read( def test_a_page_that_is_not_json_fails_the_read(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>") monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
+84
View File
@@ -215,6 +215,46 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
assert [index for index, _ in module.asked] == [0, 1] assert [index for index, _ in module.asked] == [0, 1]
def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None:
"""A reader that sees a page come again hands it over empty with a
cursor, as SuccessFactors does. That ends the read incomplete, also when
the host states a total, where it used to run on to the page cap."""
module = _paged([["1", "2"]] * 5, total=5)
seen: list[Any] = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
seen.append(cursor)
return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \
else ([], [], 5, cursor)
module.page = page
listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], False)
assert seen == [None, 1]
def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None:
"""What a resumed sweep starts on when the list grew at the top since:
the page at the saved cursor holds only roles the sweep has. The read
goes on to the roles behind them."""
pages = [["1", "2"], ["1", "2"], ["3"]]
module = _paged(pages, total=3)
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
"""`complete="end"`: the page past the last role is empty and hands no
cursor. That is the end, not a stall."""
pages = [["1"], ["2"], []]
module = _paged(pages)
assert module.READS.complete == "end"
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
assert (listing.ids, listing.complete) == (["1", "2"], True)
assert [index for index, _ in module.asked] == [0, 1, 2]
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None: def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
pages = [["1"], ["2"]] pages = [["1"], ["2"]]
web = _pages_web(pages) web = _pages_web(pages)
@@ -260,6 +300,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
assert state["ats:swept:fake:acme"] == "1" assert state["ats:swept:fake:acme"] == "1"
def _tupled(pages: list[list[str]], total: int) -> Any:
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
sweep hands back from state as a list. `cursors` are the ones it was
handed."""
module = _paged(pages, total=total)
module.cursors = []
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
module.cursors.append(cursor)
index, stated = cursor or (0, total)
ids = pages[index]
more = index + 1 < len(pages)
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
module.page = page
return module
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
"""A tuple comes back from JSON as a list. The sweep is kept all the
same, the reader gets the list, and the second pull finishes it."""
module = _tupled([["1"], ["2"], ["3"]], total=3)
state: dict[str, str] = {}
limits = _runner.Limits(max_pages=2, page_delay=0)
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
assert (first.ids, first.complete) == (["1", "2"], False)
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
assert module.cursors == [None, (1, 3), [2, 3]]
assert "ats:sweep:fake:acme" not in state
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
assert _runner._storable({"after": (1, [2, (3,)])})
assert not _runner._storable({1: "a"}) # the key comes back as "1"
assert not _runner._storable(object())
def test_a_board_read_in_one_pull_is_not_swept() -> None: def test_a_board_read_in_one_pull_is_not_swept() -> None:
pages = [["1"], ["2"]] pages = [["1"], ["2"]]
state: dict[str, str] = {} state: dict[str, str] = {}
+35 -2
View File
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
from plugins.search import SearchSource # noqa: E402 from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402 from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402 from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402 from pipeline import RunContext, Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402 from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
from prepped import upsert # noqa: E402 from prepped import upsert # noqa: E402
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
return closed return closed
def test_a_complete_search_closes_what_it_stopped_listing( def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""A search is never a company's whole list: one that read as many
results as its provider stated is stamped complete, and a role it
stopped listing still stays open until no search has listed it for
`STALE_DAYS`."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"}) params={"q": "backend"})
_fake_provider(monkeypatch, ["h1", "h2"], complete=True) _fake_provider(monkeypatch, ["h1", "h2"], complete=True)
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
_fake_provider(monkeypatch, ["h2"], complete=True) _fake_provider(monkeypatch, ["h2"], complete=True)
result = _pull(store) result = _pull(store)
assert store.state["search:complete:fake:cph"] == "1"
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
assert (result["searches"], result["closed"]) == (1, 0)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"fake:h2"} assert {r.source_id for r in store.roles()} == {"fake:h2"}
assert (result["searches"], result["closed"]) == (1, 1) assert (result["searches"], result["closed"]) == (1, 1)
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Fifteen of twenty roles dropping out of a complete search closed
them, and named the search as closing many; now they stay open and the
settle has nothing to say."""
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
params={"q": "backend"})
twenty = [f"h{n}" for n in range(20)]
_fake_provider(monkeypatch, twenty, complete=True)
_pull(store)
_fake_provider(monkeypatch, twenty[:5], complete=True)
said: list[object] = []
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
context=RunContext(on_notice=said.append))
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
assert len(store.roles()) == 20
assert said == []
def test_an_incomplete_search_closes_nothing_until_stale( def test_an_incomplete_search_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None: store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake", store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",