Read role pages in an enrich stage after scoring, best roles first
The pull reads listings only. `enrich` (app/jobs/enrich.py) takes the open roles without text that an enabled agent ranks ungated in its best `enrich.top` at `enrich.min_score` or above, best score first, on boards whose reader has `detail`, within `enrich.per_board` and `enrich.per_run`, one batch per board. `plugins.ats.read_board_details` reads them and keeps the per-board state; a read that gave no text is read again after `enrich.retry_days` (finding 18). Filled roles are upserted, which drops their vectors, and embed and score follow; `cascade.step` passes over both when nothing was filled. The pull cascade is pull, embed, score, enrich, embed, score, extract, score, summarise. Saving an agent and changing a company's affinity queue the `enrich` job (enrich, embed, score) behind their rescore; a vote does not. The stage has a switch, `stage:enrich`, over `enrich.enabled`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
77b45a4b0f
10 files changed
+956
-219
No files matched your search
+136
-163
@@ -1,6 +1,7 @@
|
||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, and Lever's EU instance. No network —
|
||||
the per-role detail reads the `enrich` stage makes, and Lever's EU
|
||||
instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +12,7 @@ import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -308,34 +309,26 @@ def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
def test_detail_is_read_for_unread_roles_only(
|
||||
def test_a_pull_reads_no_role_page(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The pages are `enrich`'s: the pull reads the listing only, and sends
|
||||
every row of a host with `detail` without a description."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
||||
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
||||
module = _fake_host(detail=_detail)
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
||||
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
||||
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one in full"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list"]
|
||||
assert [(i.source_id, i.body) for i in items] == [("fake:acme:1", ""), ("fake:acme:2", "")]
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
||||
def test_a_role_read_on_its_page_keeps_the_place_its_page_gave(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The listing states a place on every pull, and a filled value replaces
|
||||
a filled column, so for a role already read it is withheld."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": b"two in full",
|
||||
f"{base}/3": RuntimeError("down")})
|
||||
module = _fake_host(detail=lambda get, slug, row: {
|
||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||
web["answers"].update({f"{base}/list": b'["1", "3"]'})
|
||||
module = _fake_host(detail=_detail)
|
||||
listing = module.parse
|
||||
module.parse = lambda body, slug: [row | {"location": "2 Locations", "remote": "hybrid"}
|
||||
for row in listing(body, slug)]
|
||||
@@ -345,70 +338,25 @@ def test_a_role_read_on_an_earlier_pull_keeps_the_place_its_page_gave(
|
||||
|
||||
by_id = {i.source_id: (i.body, i.raw["place"], i.raw["remote"]) for i in items}
|
||||
assert by_id["fake:acme:1"] == ("", "", None)
|
||||
# Read now, and its page stated neither: the listing's stand.
|
||||
assert by_id["fake:acme:2"] == ("two in full", "2 Locations", "hybrid")
|
||||
# Never read: the listing's stand until its page is.
|
||||
assert by_id["fake:acme:3"] == ("", "2 Locations", "hybrid")
|
||||
|
||||
|
||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
||||
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
items, state = _run(monkeypatch, module)
|
||||
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
||||
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
||||
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ids_no_longer_listed_leave_the_detailed_list(
|
||||
def test_ids_no_longer_listed_leave_the_detail_lists(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""A listing cut short says nothing about the roles it left out, so their
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
ids stay, and they are not read on their pages again."""
|
||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
@@ -417,6 +365,19 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_drops_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
source.state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||
roles: Any = source.fetch() # a generator, which `close` stops
|
||||
assert next(roles).body == ""
|
||||
roles.close()
|
||||
assert json.loads(source.state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
|
||||
|
||||
def test_fetch_board_reads_the_listing_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
@@ -435,63 +396,137 @@ def test_a_host_without_detail_keeps_the_listings_description(
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
# -------------------------------------------------- read_board_details
|
||||
|
||||
NOW = datetime(2026, 9, 30, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def _details(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||
state: dict[str, str], ids: list[str], **kwargs: Any) -> list[dict[str, Any]]:
|
||||
"""`ats.read_board_details` over the fake host's roles `ids`, stored
|
||||
without text, in that order."""
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
||||
rows = [row | {"description": ""}
|
||||
for row in module.parse(json.dumps(ids).encode(), "acme")]
|
||||
return ats.read_board_details(state, "fake:acme", rows, client=object(), # type: ignore[arg-type]
|
||||
now=kwargs.pop("now", NOW), **kwargs)
|
||||
|
||||
|
||||
def test_details_are_read_in_the_order_given_for_unread_roles_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
||||
f"{base}/3": b"three"}) # 1 answers 404
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [i.body for i in items] == ["", "", "three"]
|
||||
web["answers"].update({f"{base}/1": b"one in full", f"{base}/2": b"two in full",
|
||||
f"{base}/3": b"three in full"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailempty:fake:acme": json.dumps({"fake:acme:2": NOW.isoformat()})}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["3", "2", "1"])
|
||||
|
||||
assert web["asked"] == [f"{base}/3", f"{base}/1"]
|
||||
assert [(r["source_id"], r["description"], r["location"]) for r in filled] == [
|
||||
("3", "three in full", "Copenhagen"), ("1", "one in full", "Copenhagen")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_details_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
module = _fake_host(detail=_detail)
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, module, state, ids, cap=2)
|
||||
assert [r["description"] for r in filled] == ["full 0", "full 1"]
|
||||
|
||||
web["asked"].clear()
|
||||
filled = _details(monkeypatch, module, state, ids[2:], cap=2)
|
||||
assert web["asked"] == [f"{base}/2", f"{base}/3"]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_a_429_stops_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/1": b"one", f"{base}/2": ValueError("no such page shape"),
|
||||
f"{base}/3": _status(429), f"{base}/4": b"four"})
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3", "4"])
|
||||
assert web["asked"] == [f"{base}/1", f"{base}/2", f"{base}/3"]
|
||||
assert [r["source_id"] for r in filled] == ["1"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
# A 429 is not the role's fault.
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
||||
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
"""Two roles that always fail and a cap of two: the third is read on the
|
||||
second pull, since roles never tried go before those that failed."""
|
||||
"""Roles never tried go before those that failed, within the cap."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
||||
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
web["answers"].update({f"{base}/1": failure, f"{base}/2": failure,
|
||||
f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
module = _fake_host(detail=_detail)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
state: dict[str, str] = {}
|
||||
_details(monkeypatch, module, state, ["1", "2"])
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
||||
assert [i.body for i in items] == ["", "", "three", "four"]
|
||||
_details(monkeypatch, module, state, ["1", "2", "3", "4"], cap=2)
|
||||
assert web["asked"] == [f"{base}/3", f"{base}/4"]
|
||||
|
||||
# With nothing new left, the failed ones are tried again, within the cap.
|
||||
web["asked"].clear()
|
||||
web["answers"][f"{base}/1"] = b"one at last"
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one at last", "", "", ""]
|
||||
filled = _details(monkeypatch, module, state, ["1", "2"], cap=2)
|
||||
assert [r["description"] for r in filled] == ["one at last"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
web["answers"].update({f"{base}/2": _status(410), f"{base}/3": b"three"}) # 1 answers 404
|
||||
state: dict[str, str] = {}
|
||||
filled = _details(monkeypatch, _fake_host(detail=_detail), state, ["1", "2", "3"])
|
||||
assert [r["source_id"] for r in filled] == ["3"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
||||
def test_a_read_without_text_is_read_again_after_some_days(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""Finding 18: an Oracle requisition not yet published has an empty page
|
||||
today and a full one next week."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
||||
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
web["answers"][f"{base}/1"] = b""
|
||||
module = _fake_host(detail=lambda get, slug, row: {
|
||||
"description": get(f"{base}/{row['source_id']}").decode()})
|
||||
state: dict[str, str] = {}
|
||||
assert _details(monkeypatch, module, state, ["1"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert json.loads(state["ats:detailempty:fake:acme"]) == {
|
||||
"fake:acme:1": NOW.isoformat(timespec="seconds")}
|
||||
|
||||
web["asked"].clear()
|
||||
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS - 1)
|
||||
assert _details(monkeypatch, module, state, ["1"], now=later) == []
|
||||
assert web["asked"] == []
|
||||
|
||||
web["answers"][f"{base}/1"] = b"published now"
|
||||
later = NOW + timedelta(days=ats.EMPTY_RETRY_DAYS)
|
||||
[filled] = _details(monkeypatch, module, state, ["1"], now=later)
|
||||
assert filled["description"] == "published now"
|
||||
assert json.loads(state["ats:detailempty:fake:acme"]) == {}
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
|
||||
|
||||
def test_a_role_read_before_the_dates_were_kept_is_read_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/1"] = b"one"
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1"])}
|
||||
[filled] = _details(monkeypatch, _fake_host(detail=_detail), state, ["1"])
|
||||
assert filled["description"] == "one"
|
||||
|
||||
|
||||
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
@@ -499,9 +534,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""`salary_text`, `department` and `title` would be rewritten from the
|
||||
listing on the next pull, so detail does not fill them."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]'})
|
||||
|
||||
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
||||
"salary_min": 60000, "salary_max": 70000,
|
||||
@@ -510,7 +542,8 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
[row] = _details(monkeypatch, _fake_host(detail=detail), {}, ["1"])
|
||||
item = ats.item_of(row, "fake:acme", "Acme")
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
||||
@@ -522,66 +555,6 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
||||
module: Any = _fake_host(detail=_detail)
|
||||
del module.feed_url, module.parse
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
||||
index = int(cursor or 0)
|
||||
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
||||
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
||||
assert not readers.problems(module, "fake")
|
||||
return module
|
||||
|
||||
|
||||
def test_a_paged_host_reads_every_page_then_each_role_once(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
||||
f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
items, state = _run(monkeypatch, _paged_host())
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
||||
f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "two"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
||||
assert [i.body for i in items] == ["", ""]
|
||||
|
||||
|
||||
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
||||
f"{base}/1": b"one"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
||||
_items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The consumer stops after the first role, so the second, read but not
|
||||
stored, is read again on the next pull."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
roles: Any = source.fetch() # a generator, which `close` stops
|
||||
assert next(roles).body == "one"
|
||||
roles.close()
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
|
||||
Reference in new issue
Block a user