Merge sweeps that resume on a tuple cursor, a stalled read counted incomplete, searches closing by age only, and Lever and Oracle failing on a wrong-shaped answer (wp-rev-runner)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
62cf1c738c
11 files changed
+359
-42
No files matched your search
+23
-18
@@ -39,14 +39,16 @@ the roles matching a word, which says nothing of the roles it did not list,
|
|||||||
so it closes by list nothing: a role of such a board closes once the board
|
so it closes by list nothing: a role of such a board closes once the board
|
||||||
has not listed it for `STALE_DAYS`, the rule searches have.
|
has not listed it for `STALE_DAYS`, the rule searches have.
|
||||||
|
|
||||||
A search closes the same way when it listed everything that matches (see
|
A search never closes by its list, even one stamped complete because it
|
||||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
read as many results as its provider stated: a search is never a company's
|
||||||
roles only once no search has seen them for `STALE_DAYS`. A board or a
|
whole list, so a role that drops out of its results may still be open. Its
|
||||||
search that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`,
|
roles close once no search has listed them for `STALE_DAYS`
|
||||||
`search:failing:<target>`) closes all of its roles, and its next answer
|
(`Store.close_unseen`). A board or a search that has failed on every fetch
|
||||||
reopens what it lists. A target that raises in its batch after its read, a
|
for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
|
||||||
row that does not store, is recorded as failing the same way and the batch
|
all of its roles, and its next answer reopens what it lists. A target that
|
||||||
goes on with the next one (`run_batch`).
|
raises in its batch after its read, a row that does not store, is recorded
|
||||||
|
as failing the same way and the batch goes on with the next one
|
||||||
|
(`run_batch`).
|
||||||
|
|
||||||
The settle says what needs a look in one notice (`settled`): the targets
|
The settle says what needs a look in one notice (`settled`): the targets
|
||||||
that did not answer or were read incomplete, and the boards that closed more
|
that did not answer or were read incomplete, and the boards that closed more
|
||||||
@@ -302,11 +304,12 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
had already parsed stamped — see the module docstring. A board whose
|
had already parsed stamped — see the module docstring. A board whose
|
||||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||||
whole batch job never got to it — is never closed by its list, and a board
|
whole batch job never got to it — is never closed by its list, and a board
|
||||||
that answered incompletely is not either. A search closes by its list only
|
that answered incompletely is not either. A search never closes by its
|
||||||
when that list was complete, and otherwise by `STALE_DAYS`. A board or a
|
list, complete or not: its roles close once no search has listed them
|
||||||
search that has failed on every fetch since `STALE_DAYS` ago closes all
|
for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
|
||||||
its roles (`failing_boards`). How a read that answered closes is
|
board or a search that has failed on every fetch since `STALE_DAYS` ago
|
||||||
`_close_read`'s.
|
closes all its roles (`failing_boards`). How a board's read that
|
||||||
|
answered closes is `_close_read`'s.
|
||||||
|
|
||||||
The stamps are compared with this run's own marker, the one its batches
|
The stamps are compared with this run's own marker, the one its batches
|
||||||
carried, never with the stored `pull:run`, which a later run may have
|
carried, never with the stored `pull:run`, which a later run may have
|
||||||
@@ -350,14 +353,15 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
unanswered.append(sub.target)
|
unanswered.append(sub.target)
|
||||||
since = failing.get(sub.target, ("", ""))[0]
|
since = failing.get(sub.target, ("", ""))[0]
|
||||||
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
||||||
else:
|
elif source == "search":
|
||||||
if source == "search":
|
# Never a company's whole list, complete or not: a search's
|
||||||
|
# roles close by age alone (`close_unseen`, after the loop).
|
||||||
searched += 1
|
searched += 1
|
||||||
|
continue
|
||||||
else:
|
else:
|
||||||
answered += 1
|
answered += 1
|
||||||
read = _close_read(store, source, sub, stale)
|
read = _close_read(store, source, sub, stale)
|
||||||
if read is None:
|
if read is None:
|
||||||
if source == "ats":
|
|
||||||
incomplete += 1
|
incomplete += 1
|
||||||
if not store.state.get(f"ats:sweep:{sub.target}"):
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
||||||
partial.append(sub.target)
|
partial.append(sub.target)
|
||||||
@@ -380,8 +384,9 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
|||||||
|
|
||||||
|
|
||||||
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
||||||
"""Close what one target's read of this run says has closed, and return
|
"""Close what one board's read of this run says has closed, and return
|
||||||
how many; None when the read was incomplete, which closes nothing.
|
how many; None when the read was incomplete, which closes nothing. A
|
||||||
|
search's read never comes here (see `close_boards`).
|
||||||
|
|
||||||
A read narrowed by terms closes only the roles it has not listed for
|
A read narrowed by terms closes only the roles it has not listed for
|
||||||
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
||||||
|
|||||||
+33
-10
@@ -556,17 +556,21 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
|
|
||||||
A whole listing is complete whenever it parses. A paged one reads each
|
A whole listing is complete whenever it parses. A paged one reads each
|
||||||
series' pages, `limits.page_delay` apart, until the host has no next
|
series' pages, `limits.page_delay` apart, until the host has no next
|
||||||
page, the stated total is reached, a page brings nothing new (a host
|
page, the stated total is reached, the read stalls or `limits.max_pages`
|
||||||
that ignores the cursor, incomplete) or `limits.max_pages` pages in all.
|
pages in all. A stall is a page that brings no new id while the reader
|
||||||
A series is complete when it read as many distinct result ids as its
|
still hands a cursor: any such page on a host that states no total (it
|
||||||
stated total, or, without a total, when it ended on the host's last
|
ignores the cursor), and with a total one the reader hands over empty
|
||||||
page; never for `complete="never"`. The read is complete when every
|
(it saw the page come again). A series is complete when it read as many distinct result
|
||||||
series is. A 429 after the first page ends the read, incomplete.
|
ids as its stated total, or, without a total, when it ended on the
|
||||||
|
host's last page; never after a stall, and never for
|
||||||
|
`complete="never"`. The read is complete when every series is. A 429
|
||||||
|
after the first page ends the read, incomplete.
|
||||||
|
|
||||||
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
With `can_sweep` (boards), a read cut at `max_pages` or by a 429 returns
|
||||||
the `sweep` to keep, and the next pull, handed it back, continues from
|
the `sweep` to keep, and the next pull, handed it back, continues from
|
||||||
its series and cursor; the sweep is complete when it reaches the end,
|
its series and cursor; the sweep is complete when it reaches the end,
|
||||||
and `ids` then hold every id it listed, and the listing is `swept`. A
|
and `ids` then hold every id it listed, and the listing is `swept`. The
|
||||||
|
cursor comes back as JSON gives it, a tuple as a list (`_storable`). A
|
||||||
read at the reader's `ceiling` is not swept: the host answers nothing
|
read at the reader's `ceiling` is not swept: the host answers nothing
|
||||||
past it.
|
past it.
|
||||||
|
|
||||||
@@ -633,7 +637,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||||
ended = True
|
ended = True
|
||||||
break
|
break
|
||||||
if total is None and len(seen) == before:
|
# Nothing new while the reader says more follow. Without a total
|
||||||
|
# the host ignores the cursor, and the read would never end. With
|
||||||
|
# one, only a page the reader hands over empty is a stall (it saw
|
||||||
|
# the page come again, as SuccessFactors does): a page of ids
|
||||||
|
# already read is what a resumed sweep starts on when the list
|
||||||
|
# grew at the top since, and the read goes on from there.
|
||||||
|
if len(seen) == before and (total is None or not results):
|
||||||
stalled = True
|
stalled = True
|
||||||
break
|
break
|
||||||
if stop:
|
if stop:
|
||||||
@@ -663,13 +673,26 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
|||||||
|
|
||||||
|
|
||||||
def _storable(cursor: Any) -> bool:
|
def _storable(cursor: Any) -> bool:
|
||||||
"""Whether a cursor survives a trip through JSON in state."""
|
"""Whether a cursor survives a trip through JSON in state. A tuple comes
|
||||||
|
back as a list (Workday's `(offset, total)`, SuccessFactors' `(start
|
||||||
|
row, first id)`), which counts as surviving: the next pull hands the
|
||||||
|
reader its cursor as JSON gives it back, and every paged reader takes
|
||||||
|
the list as it took the tuple."""
|
||||||
try:
|
try:
|
||||||
return json.loads(json.dumps(cursor)) == cursor
|
return json.loads(json.dumps(cursor)) == _as_json(cursor)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _as_json(value: Any) -> Any:
|
||||||
|
"""`value` with every tuple in it a list, as JSON hands it back."""
|
||||||
|
if isinstance(value, tuple | list):
|
||||||
|
return [_as_json(item) for item in value]
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {key: _as_json(item) for key, item in value.items()}
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------ with state
|
# ------------------------------------------------------------ with state
|
||||||
|
|
||||||
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
|
||||||
|
|||||||
@@ -112,9 +112,13 @@ def _places(categories: dict[str, Any]) -> list[Place]:
|
|||||||
|
|
||||||
|
|
||||||
def parse(body: bytes, slug: str) -> list[Row]:
|
def parse(body: bytes, slug: str) -> list[Row]:
|
||||||
|
"""The board's postings, a JSON list. Any other answer (an error
|
||||||
|
document sent with a 200) raises, so the read fails and closes nothing
|
||||||
|
rather than reading as a board with no roles."""
|
||||||
data = json.loads(body)
|
data = json.loads(body)
|
||||||
if not isinstance(data, list):
|
if not isinstance(data, list):
|
||||||
return []
|
raise ValueError(f"Lever's answer is a JSON {type(data).__name__}, "
|
||||||
|
"not a list of postings")
|
||||||
rows = []
|
rows = []
|
||||||
for j in data:
|
for j in data:
|
||||||
title = (j.get("text") or "").strip()
|
title = (j.get("text") or "").strip()
|
||||||
|
|||||||
@@ -110,20 +110,31 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
`items[0]`. Without the `expand` only the facets come back; asking for
|
`items[0]`. Without the `expand` only the facets come back; asking for
|
||||||
`requisitionList.secondaryLocations` gives the roles and each one's other
|
`requisitionList.secondaryLocations` gives the roles and each one's other
|
||||||
locations, which the plain `requisitionList` leaves out. Past the last role
|
locations, which the plain `requisitionList` leaves out. Past the last role
|
||||||
the list is empty."""
|
the list is empty; an answer without the wrapper raises (`_search`)."""
|
||||||
offset = int(cursor or 0)
|
offset = int(cursor or 0)
|
||||||
site = _parts(slug)[1]
|
site = _parts(slug)[1]
|
||||||
data = json.loads(get(
|
data = json.loads(get(
|
||||||
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
|
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
|
||||||
f"&expand=requisitionList.secondaryLocations"
|
f"&expand=requisitionList.secondaryLocations"
|
||||||
f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}"))
|
f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}"))
|
||||||
search = (data.get("items") or [{}])[0]
|
search = _search(data)
|
||||||
roles = search.get("requisitionList") or []
|
roles = search.get("requisitionList") or []
|
||||||
# `_paged` counts `id`, which Oracle spells `Id`.
|
# `_paged` counts `id`, which Oracle spells `Id`.
|
||||||
results = [{"id": str(role.get("Id") or "")} for role in roles]
|
results = [{"id": str(role.get("Id") or "")} for role in roles]
|
||||||
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
|
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
|
||||||
|
|
||||||
|
|
||||||
|
def _search(data: Any) -> dict[str, Any]:
|
||||||
|
"""The search wrapper of a listing's answer, `items[0]`. Every answer has
|
||||||
|
one, past the last role too; one without (an error document sent with a
|
||||||
|
200) raises, so the read fails and closes nothing rather than reading as
|
||||||
|
an empty last page with no total, which is a complete read."""
|
||||||
|
items = data.get("items") if isinstance(data, dict) else None
|
||||||
|
if not isinstance(items, list) or not items or not isinstance(items[0], dict):
|
||||||
|
raise ValueError("Oracle's answer has no search wrapper in items")
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
#: How many countries `locationsFacet` lists at most; a list this long may
|
#: How many countries `locationsFacet` lists at most; a list this long may
|
||||||
#: leave some out.
|
#: leave some out.
|
||||||
FACET_LIMIT = 10
|
FACET_LIMIT = 10
|
||||||
@@ -162,7 +173,7 @@ def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
|
|||||||
|
|
||||||
|
|
||||||
def parse(body: bytes, slug: str) -> list[Row]:
|
def parse(body: bytes, slug: str) -> list[Row]:
|
||||||
return rows((json.loads(body).get("items") or [{}])[0], slug)
|
return rows(_search(json.loads(body)), slug)
|
||||||
|
|
||||||
|
|
||||||
def _text(value: Any) -> str:
|
def _text(value: Any) -> str:
|
||||||
|
|||||||
@@ -269,10 +269,14 @@ def _entries(text: str) -> list[dict[str, Any]]:
|
|||||||
|
|
||||||
|
|
||||||
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
||||||
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
|
"""One search page, in the one country `ask` names, matching its `query`.
|
||||||
before)`: the next start row is what this page returned, and a page that
|
The cursor is `(start row, first role of the page before)`, a list when
|
||||||
starts with the same role as the last one is the site ignoring `startrow`,
|
a sweep hands it back from state: the next start row is what this page
|
||||||
which ends the read (incomplete) instead of looping."""
|
returned. A page that starts with the same role as the last one is the
|
||||||
|
site ignoring `startrow`: it comes back with no results and the cursor
|
||||||
|
it was asked with, which the runner reads as a stall that ends the read
|
||||||
|
incomplete. A cursor of None there would read as the last page, and,
|
||||||
|
on a site that states no total, as a complete read."""
|
||||||
start, previous = cursor or (0, None)
|
start, previous = cursor or (0, None)
|
||||||
country = ask.countries[0] if ask.countries else ""
|
country = ask.countries[0] if ask.countries else ""
|
||||||
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
|
||||||
@@ -281,7 +285,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|||||||
found = _entries(text)
|
found = _entries(text)
|
||||||
total = _total(text)
|
total = _total(text)
|
||||||
if found and found[0]["id"] == previous:
|
if found and found[0]["id"] == previous:
|
||||||
return text, [], total, None
|
return text, [], total, (start, previous)
|
||||||
more = bool(found) and (total is None or start + len(found) < total)
|
more = bool(found) and (total is None or start + len(found) < total)
|
||||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||||
(start + len(found), found[0]["id"]) if more else None)
|
(start + len(found), found[0]["id"]) if more else None)
|
||||||
|
|||||||
+26
-1
@@ -1,15 +1,20 @@
|
|||||||
"""The Lever host module's places, country and employment, against a small
|
"""The Lever host module's places, country and employment, against a small
|
||||||
saved board. No network. The pay and the plain row are in `test_ats.py`."""
|
saved board, and an answer that is no board. No network. The pay and the
|
||||||
|
plain row are in `test_ats.py`."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from plugins import ats # noqa: E402
|
||||||
from plugins.readers import Ask, lever # noqa: E402
|
from plugins.readers import Ask, lever # noqa: E402
|
||||||
|
|
||||||
BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes()
|
BOARD = (ROOT / "tests" / "fixtures" / "ats" / "lever-places.json").read_bytes()
|
||||||
@@ -33,6 +38,26 @@ def test_the_commitment_is_the_hosts_word() -> None:
|
|||||||
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
assert several["employment"] == "Full-time" and one["employment"] == "Permanent"
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_answer_that_is_not_a_list_raises_and_an_empty_list_is_no_roles() -> None:
|
||||||
|
with pytest.raises(ValueError, match="JSON dict, not a list"):
|
||||||
|
lever.parse(b'{"ok": false, "error": "Document not found"}', "acme")
|
||||||
|
assert lever.parse(b"[]", "acme") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_error_document_with_a_200_fails_the_read_and_stamps_nothing(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""It used to read as a complete board with no roles, which the settle
|
||||||
|
closed whole under ten roles, where the 10% guard does not hold."""
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda *a, **kw: b'{"ok": false}')
|
||||||
|
source = ats.ATSSource({"boards": [{"url": "lever:acme", "label": "Acme", "params": {}}],
|
||||||
|
"run": "r"})
|
||||||
|
source.state = {}
|
||||||
|
assert list(source.fetch()) == []
|
||||||
|
assert "ats:ok:lever:acme" not in source.state
|
||||||
|
assert json.loads(source.state["ats:failing:lever:acme"])["reason"].startswith(
|
||||||
|
"ValueError: Lever's answer is a JSON dict")
|
||||||
|
|
||||||
|
|
||||||
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
def test_it_declares_a_whole_listing_of_trusted_dates_and_its_robots_record() -> None:
|
||||||
reads = lever.READS
|
reads = lever.READS
|
||||||
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
assert (reads.listing, reads.text, reads.places) == ("whole", "listing", "many")
|
||||||
|
|||||||
@@ -165,6 +165,28 @@ def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: N
|
|||||||
assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type]
|
assert ats._fetch_rows(object(), f"oracle:{SLUG}") == ([], True) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("answer", [
|
||||||
|
{}, {"items": []}, {"items": {}}, {"items": [None]}, [], {"title": "Unauthorized"}])
|
||||||
|
def test_an_answer_without_the_search_wrapper_fails_the_read(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, no_delay: None, answer: Any) -> None:
|
||||||
|
"""It used to read as an empty last page with no total: a complete
|
||||||
|
read of a board with no roles."""
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(answer).encode())
|
||||||
|
with pytest.raises(ValueError, match="no search wrapper in items"):
|
||||||
|
ats._fetch_rows(object(), f"oracle:{SLUG}") # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_answering_without_the_wrapper_is_recorded_failing(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||||
|
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"title": "Unauthorized"}')
|
||||||
|
source = ats.ATSSource({"boards": [{"url": f"oracle:{SLUG}", "label": "Nokia",
|
||||||
|
"params": {}}], "run": "r"})
|
||||||
|
source.state = {}
|
||||||
|
assert list(source.fetch()) == []
|
||||||
|
assert f"ats:ok:oracle:{SLUG}" not in source.state
|
||||||
|
assert "no search wrapper" in json.loads(source.state[f"ats:failing:oracle:{SLUG}"])["reason"]
|
||||||
|
|
||||||
|
|
||||||
# ----------------------------------------------------------------- detail
|
# ----------------------------------------------------------------- detail
|
||||||
|
|
||||||
def _detail_get(body: bytes, asked: list[str]) -> Any:
|
def _detail_get(body: bytes, asked: list[str]) -> Any:
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
|
"""The SuccessFactors host: `plugins/readers/successfactors.py` against saved
|
||||||
career-site pages. No network. The fixtures are trimmed pages of Vestas and
|
career-site pages, and against written ones through a sweep and a pull with
|
||||||
|
its settle. No network. The fixtures are trimmed pages of Vestas and
|
||||||
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
SAP (table layout), Demant and Scania (tile layout), Teradyne (a shared site
|
||||||
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
with a path prefix), and the job pages of Vestas (`addressCountry`),
|
||||||
Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
|
Systematic (`streetAddress`) and Novo Nordisk (no microdata)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
import sys
|
import sys
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
@@ -18,13 +20,28 @@ ROOT = Path(__file__).resolve().parents[1]
|
|||||||
if str(ROOT) not in sys.path:
|
if str(ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(ROOT))
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from pipeline import Tally # noqa: E402
|
||||||
|
from app.jobs import pull # noqa: E402
|
||||||
from app.jobs.discover import boards_in # noqa: E402
|
from app.jobs.discover import boards_in # noqa: E402
|
||||||
|
from app.storage import Store # noqa: E402
|
||||||
from plugins import ats, readers # noqa: E402
|
from plugins import ats, readers # noqa: E402
|
||||||
from plugins.readers import successfactors # noqa: E402
|
from plugins.readers import successfactors # noqa: E402
|
||||||
|
|
||||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConfig:
|
||||||
|
sources: ClassVar[dict[str, object]] = {}
|
||||||
|
searches: ClassVar[list[object]] = []
|
||||||
|
|
||||||
|
|
||||||
|
def _pull(store: Store) -> dict[str, Any]:
|
||||||
|
"""One pull of every board, batches and settle, as `test_sweep_strikes`."""
|
||||||
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||||
|
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
def _read(name: str) -> str:
|
def _read(name: str) -> str:
|
||||||
return (FIXTURES / name).read_text(encoding="utf-8")
|
return (FIXTURES / name).read_text(encoding="utf-8")
|
||||||
|
|
||||||
@@ -264,6 +281,70 @@ def test_a_site_that_ignores_startrow_does_not_loop(monkeypatch: pytest.MonkeyPa
|
|||||||
assert len(asked) == 2
|
assert len(asked) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def _untotalled(ids: list[int]) -> str:
|
||||||
|
"""A table page that states no total."""
|
||||||
|
return _table_page(ids, 0).split("</span>", 1)[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_repeated_page_hands_back_a_cursor_with_no_results() -> None:
|
||||||
|
text = _untotalled([1_000_000_001, 1_000_000_002])
|
||||||
|
_decoded, results, total, cursor = successfactors.page(
|
||||||
|
lambda url, **kw: text.encode(), "careers.acme.com", [2, "1000000001"])
|
||||||
|
assert (results, total, cursor) == ([], None, (2, "1000000001"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_site_that_ignores_startrow_and_states_no_total_closes_nothing(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
"""The repeated first page used to end the read with no cursor, which a
|
||||||
|
site that states no total made complete, and the settle closed every
|
||||||
|
role past the first page."""
|
||||||
|
store = Store(tmp_path / "roledesk.db")
|
||||||
|
store.add_subscription("ats", "successfactors:careers.acme.com", label="Acme",
|
||||||
|
company="Acme", board="successfactors")
|
||||||
|
ids = [1_000_000_000 + n for n in range(4)]
|
||||||
|
site = {"ignores_startrow": False}
|
||||||
|
|
||||||
|
def fake_get(client, url, host, slug, **kw):
|
||||||
|
start = 0 if site["ignores_startrow"] else int(dict(httpx.URL(url).params)["startrow"])
|
||||||
|
return _untotalled(ids[start:start + 2]).encode()
|
||||||
|
|
||||||
|
monkeypatch.setattr(ats, "_get", fake_get)
|
||||||
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||||
|
_pull(store)
|
||||||
|
assert store.state["ats:complete:successfactors:careers.acme.com"] == "1"
|
||||||
|
assert len(store.roles()) == 4
|
||||||
|
|
||||||
|
site["ignores_startrow"] = True
|
||||||
|
result = _pull(store)
|
||||||
|
assert store.state["ats:complete:successfactors:careers.acme.com"] == ""
|
||||||
|
assert (result["incomplete"], result["closed"], len(store.roles())) == (1, 0, 4)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_site_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||||
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""The cursor `(start row, first id)` is a tuple, kept in state as a
|
||||||
|
list. The sweep is kept, and the next pull reads on from it."""
|
||||||
|
asked = _site(monkeypatch, 5, 2)
|
||||||
|
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||||
|
target = "successfactors:careers.acme.com"
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
|
||||||
|
def pull_once() -> list[Any]:
|
||||||
|
source = ats.ATSSource({"boards": [{"url": target, "label": "Acme", "params": {}}],
|
||||||
|
"run": "r"})
|
||||||
|
source.state = state
|
||||||
|
return list(source.fetch())
|
||||||
|
|
||||||
|
assert len(pull_once()) == 4
|
||||||
|
assert state[f"ats:complete:{target}"] == ""
|
||||||
|
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [4, "1000000002"]
|
||||||
|
assert len(pull_once()) == 1
|
||||||
|
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||||
|
assert json.loads(state[f"ats:ids:{target}"]) == [
|
||||||
|
f"{target}:{1_000_000_000 + n}" for n in range(5)]
|
||||||
|
assert [int(u.rsplit("=", 1)[1]) for u in asked] == [0, 2, 4]
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
def test_a_board_with_no_roles_is_complete_at_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
_site(monkeypatch, 0, 10)
|
_site(monkeypatch, 0, 10)
|
||||||
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
|
assert ats._fetch_rows(object(), "successfactors:careers.acme.com") == ([], True) # type: ignore[arg-type]
|
||||||
|
|||||||
@@ -251,6 +251,31 @@ def test_a_first_page_stating_no_total_is_read_to_a_page_that_adds_nothing(
|
|||||||
assert len(rows) == 30 and not complete
|
assert len(rows) == 30 and not complete
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_over_the_page_cap_is_swept_from_its_cursor_as_json_gives_it_back(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||||
|
"""The cursor `(offset, total)` is a tuple, kept in state as a list, and
|
||||||
|
the total later pages state as 0 is carried in it. The sweep is kept,
|
||||||
|
and the next pull reads on from it to the first page's total."""
|
||||||
|
tenant = Tenant(45)
|
||||||
|
monkeypatch.setattr(ats, "_get", tenant)
|
||||||
|
monkeypatch.setattr(ats, "MAX_PAGES", 2)
|
||||||
|
target = f"workday:{SLUG}"
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
|
||||||
|
def pull_once() -> list[Any]:
|
||||||
|
source = ats.ATSSource({"boards": [{"url": target, "label": "Zendesk", "params": {}}],
|
||||||
|
"run": "r"})
|
||||||
|
source.state = state
|
||||||
|
return list(source.fetch())
|
||||||
|
|
||||||
|
assert len(pull_once()) == 40
|
||||||
|
assert json.loads(state[f"ats:sweep:{target}"])["cursor"] == [40, 45]
|
||||||
|
assert len(pull_once()) == 5
|
||||||
|
assert (state[f"ats:complete:{target}"], state[f"ats:swept:{target}"]) == ("1", "1")
|
||||||
|
assert len(json.loads(state[f"ats:ids:{target}"])) == 45
|
||||||
|
assert [c[2]["offset"] for c in tenant.calls] == [0, 20, 40]
|
||||||
|
|
||||||
|
|
||||||
def test_a_page_that_is_not_json_fails_the_read(
|
def test_a_page_that_is_not_json_fails_the_read(
|
||||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||||
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
|
monkeypatch.setattr(ats, "_get", lambda *a, **k: b"<html>maintenance</html>")
|
||||||
|
|||||||
@@ -215,6 +215,46 @@ def test_a_page_with_nothing_new_ends_the_read_incomplete() -> None:
|
|||||||
assert [index for index, _ in module.asked] == [0, 1]
|
assert [index for index, _ in module.asked] == [0, 1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_page_with_a_cursor_is_a_stall_with_a_stated_total_too() -> None:
|
||||||
|
"""A reader that sees a page come again hands it over empty with a
|
||||||
|
cursor, as SuccessFactors does. That ends the read incomplete, also when
|
||||||
|
the host states a total, where it used to run on to the page cap."""
|
||||||
|
module = _paged([["1", "2"]] * 5, total=5)
|
||||||
|
seen: list[Any] = []
|
||||||
|
|
||||||
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||||
|
seen.append(cursor)
|
||||||
|
return (["1", "2"], [{"id": "1"}, {"id": "2"}], 5, 1) if cursor is None \
|
||||||
|
else ([], [], 5, cursor)
|
||||||
|
|
||||||
|
module.page = page
|
||||||
|
listing = _runner.read_listing(module, "acme", Web(), limits=NO_DELAY)
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], False)
|
||||||
|
assert seen == [None, 1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_page_of_ids_already_read_is_no_stall_with_a_stated_total() -> None:
|
||||||
|
"""What a resumed sweep starts on when the list grew at the top since:
|
||||||
|
the page at the saved cursor holds only roles the sweep has. The read
|
||||||
|
goes on to the roles behind them."""
|
||||||
|
pages = [["1", "2"], ["1", "2"], ["3"]]
|
||||||
|
module = _paged(pages, total=3)
|
||||||
|
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2", "3"], True)
|
||||||
|
assert [index for index, _ in module.asked] == [0, 1, 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_last_page_ends_a_read_without_a_total_complete() -> None:
|
||||||
|
"""`complete="end"`: the page past the last role is empty and hands no
|
||||||
|
cursor. That is the end, not a stall."""
|
||||||
|
pages = [["1"], ["2"], []]
|
||||||
|
module = _paged(pages)
|
||||||
|
assert module.READS.complete == "end"
|
||||||
|
listing = _runner.read_listing(module, "acme", _pages_web(pages), limits=NO_DELAY)
|
||||||
|
assert (listing.ids, listing.complete) == (["1", "2"], True)
|
||||||
|
assert [index for index, _ in module.asked] == [0, 1, 2]
|
||||||
|
|
||||||
|
|
||||||
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
def test_a_429_after_the_first_page_ends_the_read_and_on_the_first_fails() -> None:
|
||||||
pages = [["1"], ["2"]]
|
pages = [["1"], ["2"]]
|
||||||
web = _pages_web(pages)
|
web = _pages_web(pages)
|
||||||
@@ -260,6 +300,50 @@ def test_a_board_over_the_page_cap_is_swept_across_pulls() -> None:
|
|||||||
assert state["ats:swept:fake:acme"] == "1"
|
assert state["ats:swept:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def _tupled(pages: list[list[str]], total: int) -> Any:
|
||||||
|
"""`_paged` with a cursor of Workday's kind, `(next page, total)`, which a
|
||||||
|
sweep hands back from state as a list. `cursors` are the ones it was
|
||||||
|
handed."""
|
||||||
|
module = _paged(pages, total=total)
|
||||||
|
module.cursors = []
|
||||||
|
|
||||||
|
def page(get: Any, slug: str, cursor: Any, ask: Ask) -> Any:
|
||||||
|
module.cursors.append(cursor)
|
||||||
|
index, stated = cursor or (0, total)
|
||||||
|
ids = pages[index]
|
||||||
|
more = index + 1 < len(pages)
|
||||||
|
return ids, [{"id": i} for i in ids], stated, (index + 1, stated) if more else None
|
||||||
|
|
||||||
|
module.page = page
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_board_whose_cursor_is_a_tuple_is_swept_across_pulls() -> None:
|
||||||
|
"""A tuple comes back from JSON as a list. The sweep is kept all the
|
||||||
|
same, the reader gets the list, and the second pull finishes it."""
|
||||||
|
module = _tupled([["1"], ["2"], ["3"]], total=3)
|
||||||
|
state: dict[str, str] = {}
|
||||||
|
limits = _runner.Limits(max_pages=2, page_delay=0)
|
||||||
|
first = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", first, first.ids)
|
||||||
|
assert (first.ids, first.complete) == (["1", "2"], False)
|
||||||
|
assert json.loads(state["ats:sweep:fake:acme"])["cursor"] == [2, 3]
|
||||||
|
|
||||||
|
second = _runner.read(state, "ats", "fake:acme", module, "acme", Web(), limits=limits)
|
||||||
|
_runner.stamp(state, "ats", "fake:acme", "r", second, second.ids)
|
||||||
|
assert (second.ids, second.complete, second.swept) == (["1", "2", "3"], True, True)
|
||||||
|
assert module.cursors == [None, (1, 3), [2, 3]]
|
||||||
|
assert "ats:sweep:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_cursor_is_storable_when_json_gives_it_back_save_tuples_as_lists() -> None:
|
||||||
|
assert _runner._storable(7) and _runner._storable("token") and _runner._storable(None)
|
||||||
|
assert _runner._storable((40, 45)) and _runner._storable((4, "1431772533"))
|
||||||
|
assert _runner._storable({"after": (1, [2, (3,)])})
|
||||||
|
assert not _runner._storable({1: "a"}) # the key comes back as "1"
|
||||||
|
assert not _runner._storable(object())
|
||||||
|
|
||||||
|
|
||||||
def test_a_board_read_in_one_pull_is_not_swept() -> None:
|
def test_a_board_read_in_one_pull_is_not_swept() -> None:
|
||||||
pages = [["1"], ["2"]]
|
pages = [["1"], ["2"]]
|
||||||
state: dict[str, str] = {}
|
state: dict[str, str] = {}
|
||||||
|
|||||||
+35
-2
@@ -26,7 +26,7 @@ from plugins.readers import Ask, Fields, Reads, _runner # noqa: E402
|
|||||||
from plugins.search import SearchSource # noqa: E402
|
from plugins.search import SearchSource # noqa: E402
|
||||||
from app.config import SearchSeed # noqa: E402
|
from app.config import SearchSeed # noqa: E402
|
||||||
from app.jobs import pull # noqa: E402
|
from app.jobs import pull # noqa: E402
|
||||||
from pipeline import Tally # noqa: E402
|
from pipeline import RunContext, Tally # noqa: E402
|
||||||
from app.models import country_of, remote_of # noqa: E402
|
from app.models import country_of, remote_of # noqa: E402
|
||||||
from app.storage import Store # noqa: E402
|
from app.storage import Store # noqa: E402
|
||||||
from prepped import upsert # noqa: E402
|
from prepped import upsert # noqa: E402
|
||||||
@@ -291,8 +291,12 @@ def _pull(store: Store) -> dict[str, Any]:
|
|||||||
return closed
|
return closed
|
||||||
|
|
||||||
|
|
||||||
def test_a_complete_search_closes_what_it_stopped_listing(
|
def test_a_complete_search_closes_nothing_by_its_list_only_by_age(
|
||||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""A search is never a company's whole list: one that read as many
|
||||||
|
results as its provider stated is stamped complete, and a role it
|
||||||
|
stopped listing still stays open until no search has listed it for
|
||||||
|
`STALE_DAYS`."""
|
||||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
params={"q": "backend"})
|
params={"q": "backend"})
|
||||||
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
_fake_provider(monkeypatch, ["h1", "h2"], complete=True)
|
||||||
@@ -304,10 +308,39 @@ def test_a_complete_search_closes_what_it_stopped_listing(
|
|||||||
|
|
||||||
_fake_provider(monkeypatch, ["h2"], complete=True)
|
_fake_provider(monkeypatch, ["h2"], complete=True)
|
||||||
result = _pull(store)
|
result = _pull(store)
|
||||||
|
assert store.state["search:complete:fake:cph"] == "1"
|
||||||
|
assert {r.source_id for r in store.roles()} == {"fake:h1", "fake:h2"}
|
||||||
|
assert (result["searches"], result["closed"]) == (1, 0)
|
||||||
|
|
||||||
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
|
||||||
|
with store.transaction() as conn:
|
||||||
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'fake:h1'", (old,))
|
||||||
|
result = _pull(store)
|
||||||
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
assert {r.source_id for r in store.roles()} == {"fake:h2"}
|
||||||
assert (result["searches"], result["closed"]) == (1, 1)
|
assert (result["searches"], result["closed"]) == (1, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_complete_search_that_lists_far_fewer_is_not_named_in_the_notice(
|
||||||
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Fifteen of twenty roles dropping out of a complete search closed
|
||||||
|
them, and named the search as closing many; now they stay open and the
|
||||||
|
settle has nothing to say."""
|
||||||
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
params={"q": "backend"})
|
||||||
|
twenty = [f"h{n}" for n in range(20)]
|
||||||
|
_fake_provider(monkeypatch, twenty, complete=True)
|
||||||
|
_pull(store)
|
||||||
|
_fake_provider(monkeypatch, twenty[:5], complete=True)
|
||||||
|
said: list[object] = []
|
||||||
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||||
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
||||||
|
result = pull.close_boards(FakeConfig(), store, Tally(results=tuple(results)), # type: ignore[arg-type]
|
||||||
|
context=RunContext(on_notice=said.append))
|
||||||
|
assert (result["searches"], result["unanswered"], result["closed"]) == (1, 0, 0)
|
||||||
|
assert len(store.roles()) == 20
|
||||||
|
assert said == []
|
||||||
|
|
||||||
|
|
||||||
def test_an_incomplete_search_closes_nothing_until_stale(
|
def test_an_incomplete_search_closes_nothing_until_stale(
|
||||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
store.add_subscription("search", "fake:cph", label="Copenhagen", board="fake",
|
||||||
|
|||||||
Reference in new issue
Block a user