Read never-tried roles before failed ones in detail, count a 404 as read, and take only the fields a later pull keeps

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 19:41:39 +02:00
1 parent 1e47192aba
commit 0ffd0726e9
2 files changed
+233 -41

No files matched your search

+87 -41
View File
@@ -571,53 +571,91 @@ def _check_robots(client: httpx.Client, target: str,
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
#: The row keys a host's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value for them, so what a role's
#: own page said is not wiped the pull after it was read. `description`,
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
#: earliest date a role ever had. `title` and `url` are not here, since the
#: listing always states them and would overwrite them; nor are `department`
#: (no column of its own, only `raw` and the classifier's guess) and
#: `salary_text`, both rewritten from the listing on every pull.
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
"salary_max", "posted")
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str]) -> tuple[list[Row], set[str]]:
"""The rows of a board whose host has `detail`, each role not yet in
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
detail so far, for `ats:detailed:<target>`.
detailed: set[str], failed: set[str]
) -> tuple[list[Row], set[str], set[str]]:
"""The rows of a board whose host has `detail`, with the roles not yet
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
`PAGE_DELAY` apart, first the roles never tried and then those whose
detail failed before (`failed`), so a few roles that always fail cannot
take the whole cap pull after pull. Returns the rows and the compound ids
read (for `ats:detailed:<target>`) and failed (for
`ats:detailfailed:<target>`).
Every row not read in detail in this call goes out with an empty
description: a role read on an earlier pull keeps the one it has, since
`upsert_role` never overwrites a filled column with an empty value, and a
teaser from the listing must not replace it. A detail that raises is
logged and the role left for the next pull; a 429 stops the reads for this
board until then. None of this touches completeness, which is the
listing's alone. The ids of roles no longer listed are dropped, but only
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
is left out with a debug line. Every row not read in detail in this call
goes out with an empty description: a role read on an earlier pull keeps
the one it has, since `upsert_role` never overwrites a filled column with
an empty value, and a teaser from the listing must not replace it.
A detail that answers 404 or 410 counts as read: the role is gone or has
no page of its own, and asking again would not change that. One that
raises otherwise is logged and the role kept in `failed`; a 429 stops the
reads for this board until the next pull, and does not mark the role as
failing. None of this touches completeness, which is the listing's alone.
The ids of roles no longer listed are dropped from both sets, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done = set(detailed)
out: list[Row] = []
tried = 0
stopped = False
for row in rows:
compound = f"{host}:{slug}:{row['source_id']}"
if compound in done or stopped or tried >= DETAIL_CAP:
out.append(row | {"description": ""})
continue
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(PAGE_DELAY)
tried += 1
try:
filled = module.detail(get, slug, row)
except Exception as exc:
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
stopped = True
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
out.append(row | {"description": ""})
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
failing.add(key)
continue
out.append(row | filled | {"source_id": row["source_id"]})
done.add(compound)
except Exception as exc:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
target, row["source_id"], ", ".join(ignored))
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
done.add(key)
failing.discard(key)
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
if complete:
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
return out, done
listed = {compound(row) for row in rows}
done &= listed
failing &= listed
return out, done, failing
def fetch_board(target: str) -> tuple[list[Row], bool]:
@@ -671,6 +709,14 @@ class ATSSource(Source):
)},
)
def _id_set(self, key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is
none or it does not parse."""
try:
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
except (ValueError, TypeError):
return set()
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
@@ -702,14 +748,13 @@ class ATSSource(Source):
continue
self.state.pop(failing, None)
detailed_key = f"ats:detailed:{target}"
detailed: set[str] | None = None
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
try:
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
except (ValueError, TypeError):
detailed = set()
rows, detailed = _read_details(client, target, rows, complete, detailed)
detailed, failed = (self._id_set(key) for key in detail_keys)
rows, detailed, failed = _read_details(client, target, rows, complete,
detailed, failed)
detail_sets = (detailed, failed)
seen_ids = []
for row in rows:
@@ -757,5 +802,6 @@ class ATSSource(Source):
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detailed is not None:
self.state[detailed_key] = json.dumps(sorted(detailed))
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))