diff --git a/plugins/ats.py b/plugins/ats.py index ccf8e3e..9b7a64e 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -571,53 +571,91 @@ def _check_robots(client: httpx.Client, target: str, #: large new board is read over the next pulls, 200 at a time. DETAIL_CAP = 200 +#: The row keys a host's `detail` may fill: those whose column survives a +#: later pull that sends the listing's empty value for them, so what a role's +#: own page said is not wiped the pull after it was read. `description`, +#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are +#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only +#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the +#: earliest date a role ever had. `title` and `url` are not here, since the +#: listing always states them and would overwrite them; nor are `department` +#: (no column of its own, only `raw` and the classifier's guess) and +#: `salary_text`, both rewritten from the listing on every pull. +DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min", + "salary_max", "posted") + def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool, - detailed: set[str]) -> tuple[list[Row], set[str]]: - """The rows of a board whose host has `detail`, each role not yet in - `detailed` (compound ids) filled from its own page, at most `DETAIL_CAP` - of them and `PAGE_DELAY` apart. Returns the rows and the ids read in - detail so far, for `ats:detailed:`. + detailed: set[str], failed: set[str] + ) -> tuple[list[Row], set[str], set[str]]: + """The rows of a board whose host has `detail`, with the roles not yet + read in detail filled from their own pages: at most `DETAIL_CAP` of them, + `PAGE_DELAY` apart, first the roles never tried and then those whose + detail failed before (`failed`), so a few roles that always fail cannot + take the whole cap pull after pull. Returns the rows and the compound ids + read (for `ats:detailed:`) and failed (for + `ats:detailfailed:`). - Every row not read in detail in this call goes out with an empty - description: a role read on an earlier pull keeps the one it has, since - `upsert_role` never overwrites a filled column with an empty value, and a - teaser from the listing must not replace it. A detail that raises is - logged and the role left for the next pull; a 429 stops the reads for this - board until then. None of this touches completeness, which is the - listing's alone. The ids of roles no longer listed are dropped, but only + Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key + is left out with a debug line. Every row not read in detail in this call + goes out with an empty description: a role read on an earlier pull keeps + the one it has, since `upsert_role` never overwrites a filled column with + an empty value, and a teaser from the listing must not replace it. + + A detail that answers 404 or 410 counts as read: the role is gone or has + no page of its own, and asking again would not change that. One that + raises otherwise is logged and the role kept in `failed`; a 429 stops the + reads for this board until the next pull, and does not mark the role as + failing. None of this touches completeness, which is the listing's alone. + The ids of roles no longer listed are dropped from both sets, but only after a complete listing, so one cut short does not make the next pull read its missing roles again.""" host, _, slug = target.partition(":") module = atshosts.HOSTS[host] get = _getter(client, host, slug) - done = set(detailed) - out: list[Row] = [] - tried = 0 - stopped = False - for row in rows: - compound = f"{host}:{slug}:{row['source_id']}" - if compound in done or stopped or tried >= DETAIL_CAP: - out.append(row | {"description": ""}) - continue + done, failing = set(detailed), set(failed) - set(detailed) + + def compound(row: Row) -> str: + return f"{host}:{slug}:{row['source_id']}" + + unread = [row for row in rows if compound(row) not in done] + queue = ([row for row in unread if compound(row) not in failing] + + [row for row in unread if compound(row) in failing])[:DETAIL_CAP] + filled: dict[str, Row] = {} + for row in queue: + key = compound(row) time.sleep(PAGE_DELAY) - tried += 1 try: - filled = module.detail(get, slug, row) - except Exception as exc: - if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429: + answer = module.detail(get, slug, row) + except httpx.HTTPStatusError as exc: + code = exc.response.status_code + if code == 429: log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull", target) - stopped = True + break + if code in (404, 410): + done.add(key) + failing.discard(key) else: - log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) - out.append(row | {"description": ""}) + log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code) + failing.add(key) continue - out.append(row | filled | {"source_id": row["source_id"]}) - done.add(compound) + except Exception as exc: + log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) + failing.add(key) + continue + if ignored := sorted(set(answer) - set(DETAIL_FIELDS)): + log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS", + target, row["source_id"], ", ".join(ignored)) + filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS} + done.add(key) + failing.discard(key) + out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows] if complete: - done &= {f"{host}:{slug}:{row['source_id']}" for row in rows} - return out, done + listed = {compound(row) for row in rows} + done &= listed + failing &= listed + return out, done, failing def fetch_board(target: str) -> tuple[list[Row], bool]: @@ -671,6 +709,14 @@ class ATSSource(Source): )}, ) + def _id_set(self, key: str) -> set[str]: + """A JSON list of ids kept in state, as a set; empty when there is + none or it does not parse.""" + try: + return {str(i) for i in json.loads(self.state.get(key) or "[]")} + except (ValueError, TypeError): + return set() + def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # Handed in by the batch that runs this plugin, as the marker of @@ -702,14 +748,13 @@ class ATSSource(Source): continue self.state.pop(failing, None) - detailed_key = f"ats:detailed:{target}" - detailed: set[str] | None = None + detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}") + detail_sets: tuple[set[str], set[str]] | None = None if hasattr(atshosts.HOSTS.get(host), "detail"): - try: - detailed = set(json.loads(self.state.get(detailed_key) or "[]")) - except (ValueError, TypeError): - detailed = set() - rows, detailed = _read_details(client, target, rows, complete, detailed) + detailed, failed = (self._id_set(key) for key in detail_keys) + rows, detailed, failed = _read_details(client, target, rows, complete, + detailed, failed) + detail_sets = (detailed, failed) seen_ids = [] for row in rows: @@ -757,5 +802,6 @@ class ATSSource(Source): self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) - if detailed is not None: - self.state[detailed_key] = json.dumps(sorted(detailed)) + if detail_sets is not None: + for key, ids in zip(detail_keys, detail_sets, strict=True): + self.state[key] = json.dumps(sorted(ids)) diff --git a/tests/test_atshosts.py b/tests/test_atshosts.py index 4b2b1be..0e2248f 100644 --- a/tests/test_atshosts.py +++ b/tests/test_atshosts.py @@ -335,6 +335,152 @@ def test_a_host_without_detail_keeps_the_listings_description( assert "ats:detailed:fake:acme" not in state +def test_a_detail_answering_404_or_410_counts_as_read( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/2": _status(410), + f"{base}/3": b"three"}) # 1 answers 404 + items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert [i.body for i in items] == ["", "", "three"] + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:2", "fake:acme:3"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == [] + + +@pytest.mark.parametrize("failure", [_status(500), ValueError("no such page shape")]) +def test_roles_whose_detail_keeps_failing_do_not_starve_the_rest( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None: + """Two roles that always fail and a cap of two: the third is read on the + second pull, since roles never tried go before those that failed.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2", "3", "4"]', f"{base}/1": failure, + f"{base}/2": failure, f"{base}/3": b"three", f"{base}/4": b"four"}) + monkeypatch.setattr(ats, "DETAIL_CAP", 2) + module = _fake_host(detail=_detail) + _items, state = _run(monkeypatch, module) + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + + web["asked"].clear() + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/3", f"{base}/4"] + assert [i.body for i in items] == ["", "", "three", "four"] + + # With nothing new left, the failed ones are tried again, within the cap. + web["asked"].clear() + web["answers"][f"{base}/1"] = b"one at last" + items, state = _run(monkeypatch, module, state) + assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one at last", "", "", ""] + assert json.loads(state["ats:detailed:fake:acme"]) == [ + "fake:acme:1", "fake:acme:3", "fake:acme:4"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] + + +def test_a_429_on_a_detail_does_not_mark_the_role_as_failing( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": _status(429)}) + _items, state = _run(monkeypatch, _fake_host(detail=_detail)) + assert json.loads(state["ats:detailfailed:fake:acme"]) == [] + assert json.loads(state["ats:detailed:fake:acme"]) == [] + + +def test_failed_ids_no_longer_listed_leave_the_failed_list( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["2"]', f"{base}/2": ValueError("odd page")}) + state = {"ats:detailfailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])} + _items, state = _run(monkeypatch, _fake_host(detail=_detail), state) + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:2"] + + +def test_detail_fills_only_the_fields_a_later_pull_keeps( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], + caplog: pytest.LogCaptureFixture) -> None: + """`salary_text`, `department` and `title` would be rewritten from the + listing on the next pull, so detail does not fill them.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1"]'}) + + def detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]: + return {"description": "in full", "remote": "hybrid", "currency": "EUR", + "salary_min": 60000, "salary_max": 70000, + "posted": datetime(2026, 9, 1, tzinfo=UTC), + "salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales", + "title": "Another title"} + + caplog.set_level("DEBUG", logger=ats.log.name) + [item], _state = _run(monkeypatch, _fake_host(detail=detail)) + assert (item.body, item.title, item.published_at) == ( + "in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC)) + assert (item.raw["remote"], item.raw["currency"], item.raw["salary_min"], + item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000) + assert (item.raw["salary_text"], item.raw["department"]) == ("", "") + assert "department, salary_text, title left out" in caplog.text + assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency", + "salary_min", "salary_max", "posted"} + + +def _paged_host() -> types.ModuleType: + """A host `fake` that pages its listing, one id a page, and has `detail`.""" + module = _fake_host(detail=_detail) + del module.feed_url, module.parse + + def page(get: Any, slug: str, cursor: Any) -> tuple[Any, list[dict[str, Any]], Any, Any]: + index = int(cursor or 0) + ids = json.loads(get(f"https://jobs.fake.example/{slug}/list?page={index}")) + return ids, [{"id": i} for i in ids], 2, index + 1 if ids else None + + module.page = page + module.rows = lambda ids, slug: _fake_host().parse(json.dumps(ids).encode(), slug) + assert not atshosts.problems(module, "fake") + return module + + +def test_a_paged_host_reads_every_page_then_each_role_once( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": b'["2"]', + f"{base}/1": b"one", f"{base}/2": b"two"}) + items, state = _run(monkeypatch, _paged_host()) + assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1", + f"{base}/1", f"{base}/2"] + assert [i.body for i in items] == ["one", "two"] + assert state["ats:complete:fake:acme"] == "1" + + web["asked"].clear() + items, state = _run(monkeypatch, _paged_host(), state) + assert web["asked"] == [f"{base}/list?page=0", f"{base}/list?page=1"] + assert [i.body for i in items] == ["", ""] + + +def test_a_paged_listing_cut_short_keeps_the_ids_it_did_not_reach( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list?page=0": b'["1"]', f"{base}/list?page=1": _status(429), + f"{base}/1": b"one"}) + state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"]), + "ats:detailfailed:fake:acme": json.dumps(["fake:acme:3"])} + _items, state = _run(monkeypatch, _paged_host(), state) + assert state["ats:complete:fake:acme"] == "" + assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"] + assert json.loads(state["ats:detailfailed:fake:acme"]) == ["fake:acme:3"] + + +def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids( + monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None: + """The consumer stops after the first role, so the second, read but not + stored, is read again on the next pull.""" + base = "https://jobs.fake.example/acme" + web["answers"].update({f"{base}/list": b'["1", "2"]', f"{base}/1": b"one", f"{base}/2": b"two"}) + monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail)) + source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"}) + roles = source.fetch() + assert next(roles).body == "one" + roles.close() + assert "ats:detailed:fake:acme" not in source.state + + # ------------------------------------------------------------ robots.txt ROBOTS = "https://jobs.fake.example/robots.txt"