Read never-tried roles before failed ones in detail, count a 404 as read, and take only the fields a later pull keeps

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:41:39 +02:00
1 parent 1e47192aba
commit 0ffd0726e9
2 files changed
+233 -41

No files matched your search

+87 -41
View File
@@ -571,53 +571,91 @@ def _check_robots(client: httpx.Client, target: str,
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
#: The row keys a host's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value for them, so what a role's
#: own page said is not wiped the pull after it was read. `description`,
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
#: earliest date a role ever had. `title` and `url` are not here, since the
#: listing always states them and would overwrite them; nor are `department`
#: (no column of its own, only `raw` and the classifier's guess) and
#: `salary_text`, both rewritten from the listing on every pull.
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
"salary_max", "posted")
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str]) -> tuple[list[Row], set[str]]:
"""The rows of a board whose host has `detail`, each role not yet in
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
detail so far, for `ats:detailed:<target>`.
detailed: set[str], failed: set[str]
) -> tuple[list[Row], set[str], set[str]]:
"""The rows of a board whose host has `detail`, with the roles not yet
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
`PAGE_DELAY` apart, first the roles never tried and then those whose
detail failed before (`failed`), so a few roles that always fail cannot
take the whole cap pull after pull. Returns the rows and the compound ids
read (for `ats:detailed:<target>`) and failed (for
`ats:detailfailed:<target>`).
Every row not read in detail in this call goes out with an empty
description: a role read on an earlier pull keeps the one it has, since
`upsert_role` never overwrites a filled column with an empty value, and a
teaser from the listing must not replace it. A detail that raises is
logged and the role left for the next pull; a 429 stops the reads for this
board until then. None of this touches completeness, which is the
listing's alone. The ids of roles no longer listed are dropped, but only
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
is left out with a debug line. Every row not read in detail in this call
goes out with an empty description: a role read on an earlier pull keeps
the one it has, since `upsert_role` never overwrites a filled column with
an empty value, and a teaser from the listing must not replace it.
A detail that answers 404 or 410 counts as read: the role is gone or has
no page of its own, and asking again would not change that. One that
raises otherwise is logged and the role kept in `failed`; a 429 stops the
reads for this board until the next pull, and does not mark the role as
failing. None of this touches completeness, which is the listing's alone.
The ids of roles no longer listed are dropped from both sets, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done = set(detailed)
out: list[Row] = []
tried = 0
stopped = False
for row in rows:
compound = f"{host}:{slug}:{row['source_id']}"
if compound in done or stopped or tried >= DETAIL_CAP:
out.append(row | {"description": ""})
continue
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(PAGE_DELAY)
tried += 1
try:
filled = module.detail(get, slug, row)
except Exception as exc:
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
stopped = True
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
out.append(row | {"description": ""})
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
failing.add(key)
continue
out.append(row | filled | {"source_id": row["source_id"]})
done.add(compound)
except Exception as exc:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
target, row["source_id"], ", ".join(ignored))
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
done.add(key)
failing.discard(key)
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
if complete:
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
return out, done
listed = {compound(row) for row in rows}
done &= listed
failing &= listed
return out, done, failing
def fetch_board(target: str) -> tuple[list[Row], bool]:
@@ -671,6 +709,14 @@ class ATSSource(Source):
)},
)
def _id_set(self, key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is
none or it does not parse."""
try:
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
except (ValueError, TypeError):
return set()
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
@@ -702,14 +748,13 @@ class ATSSource(Source):
continue
self.state.pop(failing, None)
detailed_key = f"ats:detailed:{target}"
detailed: set[str] | None = None
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
try:
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
except (ValueError, TypeError):
detailed = set()
rows, detailed = _read_details(client, target, rows, complete, detailed)
detailed, failed = (self._id_set(key) for key in detail_keys)
rows, detailed, failed = _read_details(client, target, rows, complete,
detailed, failed)
detail_sets = (detailed, failed)
seen_ids = []
for row in rows:
@@ -757,5 +802,6 @@ class ATSSource(Source):
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detailed is not None:
self.state[detailed_key] = json.dumps(sorted(detailed))
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
+146
View File
@@ -335,6 +335,152 @@ def test_a_host_without_detail_keeps_the_listings_description(
assert "ats:detailed:fake:acme" not in state
def test_a_detail_answering_404_or_410_counts_as_read(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410),
f"{base}/3": b"three"}) # 1 answers 404
items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert [i.body for i in items] == ["", "", "three"]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")])
def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
"""Two roles that always fail and a cap of two: the third is read on the
second pull, since roles never tried go before those that failed."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure,
f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"})
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
module = _fake_host(detail=_detail)
_items, state = _run(monkeypatch, module)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
web["asked"].clear()
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"]
assert [i.body for i in items] == ["", "", "three", "four"]
# With nothing new left, the failed ones are tried again, within the cap.
web["asked"].clear()
web["answers"][f"{base}/1"] = b"one at last"
items, state = _run(monkeypatch, module, state)
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one at last", "", "", ""]
assert json.loads(state["ats:detailed:fake:acme"]) == [
"fake:acme:1", "fake:acme:3", "fake:acme:4"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_a_429_on_a_detail_does_not_mark_the_role_as_failing(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)})
_items, state = _run(monkeypatch, _fake_host(detail=_detail))
assert json.loads(state["ats:detailfailed:fake:acme"]) == []
assert json.loads(state["ats:detailed:fake:acme"]) == []
def test_failed_ids_no_longer_listed_leave_the_failed_list(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")})
state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"]
def test_detail_fills_only_the_fields_a_later_pull_keeps(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any],
caplog: pytest.LogCaptureFixture) -> None:
"""`salary_text`, `department` and `title` would be rewritten from the
listing on the next pull, so detail does not fill them."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1"]'})
def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
return {"description": "in full", "remote": "hybrid", "currency": "EUR",
"salary_min": 60000, "salary_max": 70000,
"posted": datetime(2026, 9, 1, tzinfo=UTC),
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
"title": "Another title"}
caplog.set_level("DEBUG", logger=ats.log.name)
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
assert (item.body, item.title, item.published_at) == (
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"],
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
assert "department, salary_text, title left out" in caplog.text
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
"salary_min", "salary_max", "posted"}
def _paged_host() -> types.ModuleType:
"""A host `fake` that pages its listing, one id a page, and has `detail`."""
module = _fake_host(detail=_detail)
del module.feed_url, module.parse
def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]:
index = int(cursor or 0)
ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}"))
return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None
module.page = page
module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug)
assert not atshosts.problems(module, "fake")
return module
def test_a_paged_host_reads_every_page_then_each_role_once(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]',
f"{base}/1": b"one", f"{base}/2": b"two"})
items, state = _run(monkeypatch, _paged_host())
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1",
f"{base}/1", f"{base}/2"]
assert [i.body for i in items] == ["one", "two"]
assert state["ats:complete:fake:acme"] == "1"
web["asked"].clear()
items, state = _run(monkeypatch, _paged_host(), state)
assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"]
assert [i.body for i in items] == ["", ""]
def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429),
f"{base}/1": b"one"})
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]),
"ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])}
_items, state = _run(monkeypatch, _paged_host(), state)
assert state["ats:complete:fake:acme"] == ""
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"]
def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
"""The consumer stops after the first role, so the second, read but not
stored, is read again on the next pull."""
base = "https://jobs.fake.example/acme"
web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"})
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
roles = source.fetch()
assert next(roles).body == "one"
roles.close()
assert "ats:detailed:fake:acme" not in source.state
# ------------------------------------------------------------ robots.txt
ROBOTS = "https://jobs.fake.example/robots.txt"