Read never-tried roles before failed ones in detail, count a 404 as read, and take only the fields a later pull keeps
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1e47192aba
commit
0ffd0726e9
2 files changed
+233
-41
No files matched your search
@@ -335,6 +335,152 @@ def test_a_host_without_detail_keeps_the_listings_description(
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_detail_answering_404_or_410_counts_as_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
|
||||
f"{base}/3": b"three"}) # 1 answers 404
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [i.body for i in items] == ["", "", "three"]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
|
||||
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
"""Two roles that always fail and a cap of two: the third is read on the
|
||||
second pull, since roles never tried go before those that failed."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
|
||||
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
|
||||
assert [i.body for i in items] == ["", "", "three", "four"]
|
||||
|
||||
# With nothing new left, the failed ones are tried again, within the cap.
|
||||
web["asked"].clear()
|
||||
web["answers"][f"{base}/1"] = b"one at last"
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one at last", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == []
|
||||
|
||||
|
||||
def test_failed_ids_no_longer_listed_leave_the_failed_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
|
||||
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
|
||||
caplog: pytest.LogCaptureFixture) -> None:
|
||||
"""`salary_text`, `department` and `title` would be rewritten from the
|
||||
listing on the next pull, so detail does not fill them."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]'})
|
||||
|
||||
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
|
||||
"salary_min": 60000, "salary_max": 70000,
|
||||
"posted": datetime(2026, 9, 1, tzinfo=UTC),
|
||||
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=ats.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
|
||||
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
||||
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
||||
assert "department, salary_text, title left out" in caplog.text
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
||||
"salary_min", "salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
|
||||
module = _fake_host(detail=_detail)
|
||||
del module.feed_url, module.parse
|
||||
|
||||
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
|
||||
index = int(cursor or 0)
|
||||
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
|
||||
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
|
||||
|
||||
module.page = page
|
||||
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
|
||||
assert not atshosts.problems(module, "fake")
|
||||
return module
|
||||
|
||||
|
||||
def test_a_paged_host_reads_every_page_then_each_role_once(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
|
||||
f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
items, state = _run(monkeypatch, _paged_host())
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
|
||||
f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "two"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
|
||||
assert [i.body for i in items] == ["", ""]
|
||||
|
||||
|
||||
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
|
||||
f"{base}/1": b"one"})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
|
||||
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
|
||||
_items, state = _run(monkeypatch, _paged_host(), state)
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
|
||||
|
||||
|
||||
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""The consumer stops after the first role, so the second, read but not
|
||||
stored, is read again on the next pull."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
roles = source.fetch()
|
||||
assert next(roles).body == "one"
|
||||
roles.close()
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
Reference in new issue
Block a user