Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
c7da12b93d
18 files changed
+695
-91
No files matched your search
+166
-1
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
||||
return (FIXTURES / "greenhouse.json").read_bytes()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
||||
]
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
||||
assert complete # a host that does not page answers with the whole board
|
||||
|
||||
|
||||
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(ats._TooLarge):
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
return (FIXTURES / name).read_bytes()
|
||||
|
||||
|
||||
def _status(code: int) -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("POST", "https://apply.workable.com/")
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
|
||||
|
||||
def test_workable_follows_next_page_to_the_stated_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable-2.json" if payload else "workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
||||
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
||||
assert complete
|
||||
|
||||
|
||||
def test_smartrecruiters_pages_by_offset_to_total_found(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
urls: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
urls.append(url)
|
||||
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
|
||||
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
||||
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
||||
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
||||
assert complete
|
||||
|
||||
|
||||
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""totalFound says 3; the second page is empty, so two were read."""
|
||||
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""A page that shifted under an offset repeats a role and misses one: three
|
||||
rows read, two distinct, against a total of three."""
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
page = json.loads(_page("smartrecruiters.json"))
|
||||
if "offset=2" in url:
|
||||
page["content"] = page["content"][1:]
|
||||
return json.dumps(page).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_the_page_cap_leaves_a_board_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable.json") # always another page
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert (len(calls), complete) == (3, False)
|
||||
|
||||
|
||||
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
if payload:
|
||||
raise _status(429)
|
||||
return _page("workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
||||
|
||||
|
||||
def test_a_429_on_the_first_page_is_a_failure(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
raise _status(429)
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# -------------------------------------------------------- failing boards
|
||||
|
||||
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]})
|
||||
source.state = {"ats:run": "run-1"}
|
||||
return source
|
||||
|
||||
|
||||
def test_a_failing_board_records_since_and_the_latest_reason(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, _status(404))
|
||||
assert list(source.fetch()) == []
|
||||
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
||||
assert first["reason"] == "404 Not Found" and first["since"]
|
||||
assert "ats:ok:greenhouse:gone" not in source.state
|
||||
|
||||
# A later failure keeps when the failures began and updates the reason.
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
first | {"since": "2026-09-01T00:00:00+00:00"})
|
||||
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
||||
later.state = source.state
|
||||
list(later.fetch())
|
||||
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
||||
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
||||
|
||||
|
||||
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
||||
assert len(list(source.fetch())) == 2
|
||||
assert "ats:failing:greenhouse:gone" not in source.state
|
||||
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
||||
assert source.state["ats:complete:greenhouse:gone"] == ""
|
||||
Reference in new issue
Block a user