Read never-tried roles before failed ones in detail, count a 404 as read, and take only the fields a later pull keeps
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1e47192aba
commit
0ffd0726e9
2 files changed
+233
-41
No files matched your search
+87
-41
@@ -571,53 +571,91 @@ def _check_robots(client: httpx.Client, target: str,
|
||||
#: large new board is read over the next pulls, 200 at a time.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
#: The row keys a host's `detail` may fill: those whose column survives a
|
||||
#: later pull that sends the listing's empty value for them, so what a role's
|
||||
#: own page said is not wiped the pull after it was read. `description`,
|
||||
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
|
||||
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
|
||||
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
|
||||
#: earliest date a role ever had. `title` and `url` are not here, since the
|
||||
#: listing always states them and would overwrite them; nor are `department`
|
||||
#: (no column of its own, only `raw` and the classifier's guess) and
|
||||
#: `salary_text`, both rewritten from the listing on every pull.
|
||||
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
|
||||
"salary_max", "posted")
|
||||
|
||||
|
||||
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||
detailed: set[str]) -> tuple[list[Row], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, each role not yet in
|
||||
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
|
||||
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
|
||||
detail so far, for `ats:detailed:<target>`.
|
||||
detailed: set[str], failed: set[str]
|
||||
) -> tuple[list[Row], set[str], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, with the roles not yet
|
||||
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
|
||||
`PAGE_DELAY` apart, first the roles never tried and then those whose
|
||||
detail failed before (`failed`), so a few roles that always fail cannot
|
||||
take the whole cap pull after pull. Returns the rows and the compound ids
|
||||
read (for `ats:detailed:<target>`) and failed (for
|
||||
`ats:detailfailed:<target>`).
|
||||
|
||||
Every row not read in detail in this call goes out with an empty
|
||||
description: a role read on an earlier pull keeps the one it has, since
|
||||
`upsert_role` never overwrites a filled column with an empty value, and a
|
||||
teaser from the listing must not replace it. A detail that raises is
|
||||
logged and the role left for the next pull; a 429 stops the reads for this
|
||||
board until then. None of this touches completeness, which is the
|
||||
listing's alone. The ids of roles no longer listed are dropped, but only
|
||||
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
|
||||
is left out with a debug line. Every row not read in detail in this call
|
||||
goes out with an empty description: a role read on an earlier pull keeps
|
||||
the one it has, since `upsert_role` never overwrites a filled column with
|
||||
an empty value, and a teaser from the listing must not replace it.
|
||||
|
||||
A detail that answers 404 or 410 counts as read: the role is gone or has
|
||||
no page of its own, and asking again would not change that. One that
|
||||
raises otherwise is logged and the role kept in `failed`; a 429 stops the
|
||||
reads for this board until the next pull, and does not mark the role as
|
||||
failing. None of this touches completeness, which is the listing's alone.
|
||||
The ids of roles no longer listed are dropped from both sets, but only
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done = set(detailed)
|
||||
out: list[Row] = []
|
||||
tried = 0
|
||||
stopped = False
|
||||
for row in rows:
|
||||
compound = f"{host}:{slug}:{row['source_id']}"
|
||||
if compound in done or stopped or tried >= DETAIL_CAP:
|
||||
out.append(row | {"description": ""})
|
||||
continue
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
def compound(row: Row) -> str:
|
||||
return f"{host}:{slug}:{row['source_id']}"
|
||||
|
||||
unread = [row for row in rows if compound(row) not in done]
|
||||
queue = ([row for row in unread if compound(row) not in failing]
|
||||
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
|
||||
filled: dict[str, Row] = {}
|
||||
for row in queue:
|
||||
key = compound(row)
|
||||
time.sleep(PAGE_DELAY)
|
||||
tried += 1
|
||||
try:
|
||||
filled = module.detail(get, slug, row)
|
||||
except Exception as exc:
|
||||
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
|
||||
answer = module.detail(get, slug, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if code == 429:
|
||||
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||
target)
|
||||
stopped = True
|
||||
break
|
||||
if code in (404, 410):
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
else:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
out.append(row | {"description": ""})
|
||||
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
|
||||
failing.add(key)
|
||||
continue
|
||||
out.append(row | filled | {"source_id": row["source_id"]})
|
||||
done.add(compound)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
failing.add(key)
|
||||
continue
|
||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||
target, row["source_id"], ", ".join(ignored))
|
||||
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
|
||||
if complete:
|
||||
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
|
||||
return out, done
|
||||
listed = {compound(row) for row in rows}
|
||||
done &= listed
|
||||
failing &= listed
|
||||
return out, done, failing
|
||||
|
||||
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
@@ -671,6 +709,14 @@ class ATSSource(Source):
|
||||
)},
|
||||
)
|
||||
|
||||
def _id_set(self, key: str) -> set[str]:
|
||||
"""A JSON list of ids kept in state, as a set; empty when there is
|
||||
none or it does not parse."""
|
||||
try:
|
||||
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
|
||||
except (ValueError, TypeError):
|
||||
return set()
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
with httpx.Client() as client:
|
||||
# Handed in by the batch that runs this plugin, as the marker of
|
||||
@@ -702,14 +748,13 @@ class ATSSource(Source):
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
detailed_key = f"ats:detailed:{target}"
|
||||
detailed: set[str] | None = None
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||
try:
|
||||
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
|
||||
except (ValueError, TypeError):
|
||||
detailed = set()
|
||||
rows, detailed = _read_details(client, target, rows, complete, detailed)
|
||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||
detailed, failed)
|
||||
detail_sets = (detailed, failed)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
@@ -757,5 +802,6 @@ class ATSSource(Source):
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
if detailed is not None:
|
||||
self.state[detailed_key] = json.dumps(sorted(detailed))
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
Reference in new issue
Block a user