Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
c7da12b93d
18 files changed
+695
-91
No files matched your search
+55
-2
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -139,16 +140,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
|
||||
return b"short" if params["page"] == 3 else b"full"
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setattr(search, "parse_jobindex",
|
||||
lambda body: full[:5] if body == b"short" else full)
|
||||
monkeypatch.setattr(search, "_jobindex_page",
|
||||
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
||||
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
||||
|
||||
# Stopped by the page limit with every page full: not complete.
|
||||
pages.clear()
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert (pages, complete) == ([1, 2], False)
|
||||
|
||||
|
||||
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A full page whose parser dropped one item still has 20 items, so the
|
||||
search reads on instead of calling itself complete at 19 rows."""
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
||||
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
||||
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert not complete
|
||||
|
||||
|
||||
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
||||
|
||||
|
||||
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
||||
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
||||
asked: list[int] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
page = int(url.rsplit("/", 1)[1])
|
||||
asked.append(page)
|
||||
start = (page - 1) * search.ADZUNA_PAGE
|
||||
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
||||
body: dict[str, Any] = {"results": results}
|
||||
if count is not None:
|
||||
body["count"] = count
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
||||
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
||||
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
||||
_adzuna(monkeypatch, count)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (60, complete)
|
||||
|
||||
|
||||
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
||||
assert search.provider_of("adzuna:berlin") == "adzuna"
|
||||
|
||||
Reference in new issue
Block a user