Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+48 -17
View File
@@ -24,9 +24,12 @@ browser session of its own site) and Jooble (a key needs a manual
application, and its API host answered a Cloudflare challenge).
A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that ran out of results before its page
limit was complete, and the pull closes what it stopped listing; the rest
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
one run is not closed here. A search that read everything that matches was
complete, and the pull closes what it stopped listing; the rest close once no
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
how many results match, so an Adzuna search is complete when it read that
many; Jobindex states no count, so a Jobindex search is complete when a page
within the page limit came back short.
"""
from __future__ import annotations
@@ -124,20 +127,28 @@ def _text(fragment: str) -> str:
def parse_jobindex(body: bytes) -> list[Row]:
"""One page of a Jobindex RSS search. An item is `<title>` "Role, Company",
return _jobindex_page(body)[0]
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
"""One page of a Jobindex RSS search, and how many items it had before
any was dropped. An item is `<title>` "Role, Company",
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
("robot job") has no logo, so its company is what follows the title's
last comma, or '' when there is none."""
last comma, or '' when there is none. A body with no `<channel>` is not
a search result and raises, since an empty list would read as a search
that ran out and close everything it listed before."""
root = ET.fromstring(body)
channel = root.find("channel")
if channel is None:
return []
raise ValueError("jobindex answered without an RSS channel")
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
search_area = area_match.group(1).strip() if area_match else ""
rows = []
for item in channel.findall("item"):
items = channel.findall("item")
for item in items:
link = (item.findtext("link") or "").strip()
full_title = (item.findtext("title") or "").strip()
if not full_title or not _web_url(link):
@@ -169,21 +180,24 @@ def parse_jobindex(body: bytes) -> list[Row]:
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows
return rows, len(items)
def search_jobindex(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page or the page
limit. Returns the rows and whether the results ran out before the limit."""
limit. Returns the rows and whether the results ran out: a page with
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
a dropped item does not pass for the end. Jobindex states no total, so a
search that read the page limit full is never complete."""
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
if params.get("area"):
query["geoareaid"] = params["area"]
rows: list[Row] = []
for page in range(1, _pages(params) + 1):
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page}))
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
rows += got
if len(got) < JOBINDEX_PAGE:
if items < JOBINDEX_PAGE:
return rows, True
return rows, False
@@ -202,7 +216,10 @@ def parse_adzuna(body: bytes, country: str) -> list[Row]:
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
and snippets carry `<strong>` highlights. A salary Adzuna estimated
(`salary_is_predicted`) is not the posting's, so it is dropped."""
data = json.loads(body)
return _adzuna_rows(json.loads(body), country)
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
rows = []
for job in data.get("results") or []:
title = _text(job.get("title") or "")
@@ -237,6 +254,9 @@ def _parse_iso(value: Any) -> datetime | None:
def search_adzuna(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, until a short page, the stated `count` or
the page limit. Returns the rows and whether they are all `count` says
match, or, on an answer with no `count`, whether a page came back short."""
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
if not (app_id and app_key):
@@ -252,13 +272,24 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
if params.get("where"):
query["where"] = params["where"]
rows: list[Row] = []
seen: set[str] = set()
count: Any = None
ran_out = False
for page in range(1, _pages(params) + 1):
url = ADZUNA_SEARCH.format(country=country, page=page)
got = parse_adzuna(_get(client, url, query), country)
rows += got
if len(got) < ADZUNA_PAGE:
return rows, True
return rows, False
data = json.loads(_get(client, url, query))
results = data.get("results") or []
count = data.get("count", count)
seen |= {str(job.get("id") or "") for job in results}
rows += _adzuna_rows(data, country)
if len(results) < ADZUNA_PAGE:
ran_out = True
break
if isinstance(count, int) and len(seen) >= count:
break
# Adzuna states how many results match; the search is complete when it
# read that many distinct ones. Without a count, a short page is the end.
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],