From c7da12b93dd699206462f93ecb4a33a215ddfee6 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 14:02:49 +0200 Subject: [PATCH] Paged boards are read to their stated total, and a board failing for 21 days closes Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus --- app/cli.py | 27 ++-- app/jobs/pull.py | 71 ++++++--- app/jobs/sources.py | 2 +- app/static/app.css | 1 + app/static/companies.js | 10 ++ app/storage/store.py | 7 + app/web/api.py | 23 ++- docs/decisions.md | 8 +- plugins/ats.py | 170 +++++++++++++++++++--- plugins/search.py | 65 ++++++--- tests/fixtures/ats/smartrecruiters-2.json | 16 ++ tests/fixtures/ats/smartrecruiters.json | 4 +- tests/fixtures/ats/workable-2.json | 25 ++++ tests/fixtures/ats/workable.json | 3 +- tests/test_ats.py | 167 ++++++++++++++++++++- tests/test_pull.py | 98 ++++++++++++- tests/test_search.py | 57 +++++++- tests/test_web.py | 32 +++- 18 files changed, 695 insertions(+), 91 deletions(-) create mode 100644 tests/fixtures/ats/smartrecruiters-2.json create mode 100644 tests/fixtures/ats/workable-2.json diff --git a/app/cli.py b/app/cli.py index 109d79e..f73bd40 100644 --- a/app/cli.py +++ b/app/cli.py @@ -78,7 +78,7 @@ def build_parser() -> argparse.ArgumentParser: discover.add_argument("domain", help="its website, e.g. pleo.io") discover.add_argument("--add", action="store_true", help="add the company, with its board when one is readable") - companies_sub.add_parser("check", help="report boards that do not answer") + companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull") searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches_sub = searches.add_subparsers(dest="action", required=True) @@ -322,19 +322,28 @@ def add_company(store: Store, board: str, name: str) -> int: def check_companies(store: Store) -> int: """`app companies check`: every `ats` board, fetched once. Prints a - line per board that did not answer and exits 1 if any did not.""" - from plugins.ats import fetch_board + line per board that did not answer now, answered incompletely, or has + been failing in the pull, with what the pull recorded beside the live + result, and exits 1 if any did not answer now.""" + from app import jobs + from plugins.ats import failure_reason, fetch_board subs = store.subscriptions(source="ats") - dead = [] + failing = jobs.pull.failing_boards(store) + dead = 0 for sub in subs: try: - fetch_board(sub.target) + rows, complete = fetch_board(sub.target) + live = f"{len(rows)} roles" + ("" if complete else ", incomplete") except Exception as exc: - dead.append((sub.target, sub.label, str(exc))) - for target, label, reason in dead: - print(f"{target} ({label}): {reason}") - print(f"checked {len(subs)} boards, {len(dead)} dead") + complete, live = False, f"not answering: {failure_reason(exc)}" + dead += 1 + since, reason = failing.get(sub.target, ("", "")) + if complete and not since: + continue + stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering" + print(f"{sub.target} ({sub.label}): now {live}; {stored}") + print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull") return 1 if dead else 0 diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 74e5cca..75c9091 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -7,15 +7,19 @@ sized for exactly this. The final job's `settle` then closes every board a complete fetch of it did not list again — see `close_boards`. Closure is read from `self.state`, which each plugin itself stamps per target -on a successful parse (`ats:ok:`, `ats:ids:`, and the same -under `search:`), rather than from what each `pull_batch` job happens to -return. A batch job that raises partway through still leaves the boards it had -already finished stamped, so one bad board in a batch of ten does not cost the -other nine their closure. +on a successful parse (`ats:ok:`, `ats:complete:`, +`ats:ids:`, and the same under `search:`), rather than from what each +`pull_batch` job happens to return. A batch job that raises partway through +still leaves the boards it had already finished stamped, so one bad board in a +batch of ten does not cost the other nine their closure. A board that answered +but was not read to its stated total (a paged host that hit a 429 or the page +cap) stores what it read and closes nothing. -A search closes the same way when its results ran out before its page limit, -since it then listed everything that matches. One that hit the limit did not, -so its roles close only once no search has seen them for `STALE_DAYS`. +A search closes the same way when it listed everything that matches (see +`plugins/search.py` for how each provider knows). One that did not closes its +roles only once no search has seen them for `STALE_DAYS`. A board +that has failed on every fetch for `STALE_DAYS` (`ats:failing:`) +closes all of its roles, and its next answer reopens what it lists. """ from __future__ import annotations @@ -43,15 +47,22 @@ SETTLE = "app.jobs.pull:close_boards" #: The plugins a pull reads, by the `source` their subscriptions carry. SOURCES = ("ats", "search") -#: How long a search role stays open once no search lists it any more. +#: How long a search role stays open once no search lists it any more, and +#: how long a board may fail on every fetch before its roles close. STALE_DAYS = 21 +#: The `state` prefix the `ats` plugin records a failing board under: +#: `ats:failing:` -> `{"since", "reason"}`, removed on its next answer. +FAILING = "ats:failing:" + #: The `state` key that records the starter searches were imported. SEARCHES_SEEDED = "searches:seeded" def _now() -> str: - return datetime.now(UTC).isoformat(timespec="seconds") + """A run marker: to the microsecond, so two pulls a second apart are two + runs and a stamp from one never matches the other.""" + return datetime.now(UTC).isoformat(timespec="microseconds") def _boards(config: Config, store: Store) -> list[dict[str, Any]]: @@ -124,14 +135,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *, `pull_batch` job that raised partway through still leaves the boards it had already parsed stamped — see the module docstring. A board whose `ats:ok:` was not written during *this* run — it failed, or its - whole batch job never got to it — is never closed. A search closes by its - list only when that list was complete, and otherwise by `STALE_DAYS`. + whole batch job never got to it — is never closed by its list, and a board + that answered incompletely is not either. A search closes by its list only + when that list was complete, and otherwise by `STALE_DAYS`. A board that + has failed on every fetch since `STALE_DAYS` ago closes all its roles. A board or search switched off while its batch was in flight had its roles reopened by that batch, after the switch closed them, so every switched-off one is closed again here. """ - answered = searched = failed = closed = 0 + answered = searched = failed = incomplete = closed = 0 + stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds") + failing = failing_boards(store) for source in SOURCES: for sub in store.subscriptions(source=source): if not (sub.subscribed and sub.fetch): @@ -141,21 +156,39 @@ def close_boards(config: Config, store: Store, tally: Tally, *, stamp = store.state.get(f"{source}:ok:{sub.target}", "") if not (stamp and stamp == run_marker): failed += 1 + since = failing.get(sub.target, ("", ""))[0] if source == "ats" else "" + if since and since < stale: + closed += store.close_subscription(sub.id) continue if source == "search": searched += 1 - if not store.state.get(f"search:complete:{sub.target}", ""): - continue else: answered += 1 + if not store.state.get(f"{source}:complete:{sub.target}", ""): + if source == "ats": + incomplete += 1 + continue seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) closed += store.mark_closed(source, seen_ids, sub.id) if searched: - stale = datetime.now(UTC) - timedelta(days=STALE_DAYS) - closed += store.close_unseen("search", stale.isoformat(timespec="seconds")) + closed += store.close_unseen("search", stale) roles = sum(int(result.get("roles", 0)) for result in tally.results) - return {"boards": answered, "searches": searched, "failed": failed, "roles": roles, - "closed": closed} + return {"boards": answered, "searches": searched, "failed": failed, + "incomplete": incomplete, "roles": roles, "closed": closed} + + +def failing_boards(store: Store) -> dict[str, tuple[str, str]]: + """`target -> (since, reason)` for every `ats` board whose last fetch + failed: when the failures began, and what the latest one said.""" + out: dict[str, tuple[str, str]] = {} + for key, value in store.state_items(FAILING).items(): + try: + record = json.loads(value) + except ValueError: + continue + out[key[len(FAILING):]] = (str(record.get("since") or ""), + str(record.get("reason") or "")) + return out def search_target(provider: str, name: str) -> str: diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 8637670..16156f4 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -48,7 +48,7 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]: if store.company(key) is not None: raise ValueError(f"{name} is already followed") try: - rows = fetch_board(board) + rows, _complete = fetch_board(board) except Exception as exc: raise ValueError(f"{board} did not answer: {exc}") from exc if not rows: diff --git a/app/static/app.css b/app/static/app.css index eaff7e5..6bfc580 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -1200,6 +1200,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di gap: .4rem; font-size: .8rem; color: var(--dim); } .srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; } +.crow .sub .failing { color: var(--warn); } .linky { padding: 0; border: 0; background: none; font-size: .8rem; diff --git a/app/static/companies.js b/app/static/companies.js index 8c8aab5..4c13f96 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -73,6 +73,16 @@ sub.appendChild(link); } sub.append(` · ${count(company.open_roles, "open role")}`); + if (company.failing_since) { + // The board has failed on every pull since then; after 21 days its + // roles close. + const failing = document.createElement("span"); + failing.className = "failing"; + failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}` + + (company.failing_reason ? `: ${company.failing_reason}` : ""); + sub.append(" · "); + sub.appendChild(failing); + } return sub; } diff --git a/app/storage/store.py b/app/storage/store.py index c463971..a007148 100644 --- a/app/storage/store.py +++ b/app/storage/store.py @@ -1230,6 +1230,13 @@ class Store: `self.state`: it can keep a counter without being able to read roles.""" return _StateMap(self) + def state_items(self, prefix: str) -> dict[str, str]: + """Every `state` key that starts with `prefix`, with its value, in one + query, for a reader that wants one kind of key across every target.""" + return {str(r["key"]): str(r["value"]) for r in self._rows( + "SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key", + (len(prefix), prefix))} + def _check_status(status: str) -> None: if status not in STATUSES: diff --git a/app/web/api.py b/app/web/api.py index 74d81b1..d3ff044 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -264,7 +264,9 @@ class CompanyOut(BaseModel): board's subscription, whether it is on, and how many of its roles are still open. `ats` is the host its careers site runs on whether or not the app reads it, so a followed company with no `board` and an `ats` is one - the app cannot read yet; `careers_url` is where a person can.""" + the app cannot read yet; `careers_url` is where a person can. + `failing_since` and `failing_reason` say since when the board has failed + on every fetch and why it last did, both empty while it answers.""" key: str name: str @@ -276,17 +278,19 @@ class CompanyOut(BaseModel): ats: str = "" careers_url: str = "" country: str = "" + failing_since: str = "" + failing_reason: str = "" -def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]: - """`company_key -> (subscription id, ats host, on)`, read off each +def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: + """`company_key -> (subscription id, ats host, on, target)`, read off each company's own subscription. Same join as `Store.company_boards`, with the id and switch state a company row needs to draw its own on/off control rather than fetching `/api/sources` as well.""" - boards: dict[str, tuple[int, str, bool]] = {} + boards: dict[str, tuple[int, str, bool, str]] = {} for sub in store.subscriptions(source="ats"): - boards.setdefault(company_key(sub.company), (sub.id, sub.board, - bool(sub.subscribed and sub.fetch))) + boards.setdefault(company_key(sub.company), ( + sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target)) return boards @@ -296,13 +300,16 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut absent) has it; 0 throughout when there is no agent.""" boards = _company_boards(store) counts = store.open_role_counts() + failing = jobs.pull.failing_boards(store) out = [] for c in store.companies(_agent_id(store, agent)): - sub_id, host, on = boards.get(c.key, (None, "", None)) + sub_id, host, on, target = boards.get(c.key, (None, "", None, "")) + since, reason = failing.get(target, ("", "")) out.append(CompanyOut(key=c.key, name=c.name, board=host, subscription_id=sub_id, on=on, affinity=c.affinity, open_roles=counts.get(c.key, 0), ats=c.ats, - careers_url=c.careers_url, country=c.country)) + careers_url=c.careers_url, country=c.country, + failing_since=since, failing_reason=reason)) return out diff --git a/docs/decisions.md b/docs/decisions.md index d1a82d8..030e993 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -38,7 +38,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning ## Ingestion -**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker and the ids it saw, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. +**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker, the ids it saw and whether they are the whole board, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran and the read was complete. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. **Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards. @@ -50,7 +50,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API. -**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. A search whose results ran out before its page limit did list everything, and closes like a board; the rest close once no search has listed them for 21 days. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. +**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. **Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later. @@ -64,6 +64,10 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role. +**Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse. + +**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result. + **`pull` is a job that starts the cascade, on a default schedule of every 6h.** A schedule enqueues a job kind and a cascade is started by one, so the `pull` job only asks for the chain; declared beside it in `app/jobs/cascade.py`, the schedule is a starting point that an edit under Runs overrides for good. `pull` is not also a task, which would draw two buttons under Runs. ## Ranking diff --git a/plugins/ats.py b/plugins/ats.py index add18f3..6bf1929 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:acme`. Every board is fetched independently; one that fails or times -out is logged and skipped, and the rest continue. +out is logged, recorded as failing under `ats:failing:` (see +`ATSSource.fetch`), and skipped, and the rest continue. + +Workable and SmartRecruiters page their list and state a total; every other +host answers with the whole board at once. A paged board counts as complete +only when it read as many distinct roles as the total says (see `_paged`). """ from __future__ import annotations @@ -27,6 +32,7 @@ import html import json import logging import re +import time import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator from datetime import UTC, datetime @@ -119,10 +125,11 @@ class _TooLarge(ValueError): def _get(client: httpx.Client, url: str, host: str, slug: str, *, - method: str = "GET") -> bytes: + method: str = "GET", payload: dict[str, Any] | None = None) -> bytes: """Fetch one board feed: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at - `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`.""" + `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST + sends `payload` as JSON, `{}` when there is none.""" allow = _allowed_host(host, slug) current = url for _hop in range(4): @@ -132,7 +139,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *, response = client.request( method, current, timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, - json={} if method == "POST" else None, + json=(payload or {}) if method == "POST" else None, ) if response.is_redirect: location = response.headers.get("location") or "" @@ -258,15 +265,20 @@ def _parse_ashby(body: bytes) -> list[Row]: def _parse_workable(body: bytes, slug: str) -> list[Row]: - """POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` -> - `{"total": int, "results": [...]}`, as Hugging Face's board answered on - 2026-09-28. A result carries no URL and no text: the job page is + return _workable_rows(json.loads(body), slug) + + +def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs + -> `{"total": int, "results": [...], "nextPage": token}`, as Hugging + Face's board answered on 2026-09-28. Ten results a page; the next page is + the same POST with `{"token": nextPage}`, and the last page has no + `nextPage` (see `_paged`). A result carries no URL and no text: the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and the description is only on the per-job endpoint, one request per role, so `description` stays empty here as it does for SmartRecruiters. `location` is an object and `department` a list; `remote` is a flag beside `workplace` (remote, hybrid, on_site).""" - data = json.loads(body) rows = [] for j in data.get("results") or []: title = (j.get("title") or "").strip() @@ -293,11 +305,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]: def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]: - """https://api.smartrecruiters.com/v1/companies/{slug}/postings -> - `{"content": [...]}`. The list omits the job description (only the + return _smartrecruiters_rows(json.loads(body), slug) + + +def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]: + """One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings + -> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a + page, whatever `limit` asks for; the next is `?offset=` further on (see + `_paged`). The list omits the job description (only the per-posting detail endpoint has it, and fetching one per row would be hundreds of extra requests per board), so `description` stays empty here.""" - data = json.loads(body) rows = [] for j in data.get("content") or []: title = (j.get("name") or "").strip() @@ -474,9 +491,91 @@ def careers_url(target: str) -> str: return _BOARDS[host][1].format(slug=slug) -def _fetch_rows(client: httpx.Client, target: str) -> list[Row]: - """One board, fetched and parsed. Raises on any failure — refused host, - timeout, a feed shape none of the parsers understand. +#: Pages read from one paged board at most, and the pause between two of +#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so +#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers +#: a burst with 429, from seeing one. +MAX_PAGES = 100 +PAGE_DELAY = 0.5 + +#: SmartRecruiters' page size, which is also the most it serves per page. +SMARTRECRUITERS_PAGE = 100 + +#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its +#: raw results, the total it states, and the cursor of the next page (None +#: after the last). The first call gets cursor None. +Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any] + + +def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page: + url = _BOARDS["workable"][0].format(slug=slug) + data = json.loads(_get(client, url, "workable", slug, method="POST", + payload={"token": cursor} if cursor else {})) + return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None + + +def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page: + offset = int(cursor or 0) + url = (_BOARDS["smartrecruiters"][0].format(slug=slug) + + f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}") + data = json.loads(_get(client, url, "smartrecruiters", slug)) + content = data.get("content") or [] + return data, content, data.get("totalFound"), offset + len(content) if content else None + + +#: host -> (one page, that page's rows) for the hosts that page. +_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page], + Callable[[dict[str, Any], str], list[Row]]]] = { + "workable": (_workable_page, _workable_rows), + "smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows), +} + + +def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: + """Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart, + until the host has no next page, the stated total is reached, or + `MAX_PAGES`. Returns the rows and whether the board is complete: as many + distinct results read as the host's stated total. Counted on the raw + results, so a row the parser drops does not make a board incomplete, and + on distinct ids, so a page that shifted under an offset and repeated a + role does not make it complete. + + A 429 after the first page stops the read with what it has, incomplete; + one on the first page raises like any other failure.""" + host, _, slug = target.partition(":") + page_of, rows_of = _PAGED[host] + rows: dict[str, Row] = {} + seen: set[str] = set() + total: Any = None + cursor: Any = None + for page in range(MAX_PAGES): + if page: + time.sleep(PAGE_DELAY) + try: + data, results, total, cursor = page_of(client, slug, cursor) + except httpx.HTTPStatusError as exc: + if not page or exc.response.status_code != 429: + raise + log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete", + target, page + 1, len(seen), total) + return list(rows.values()), False + seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} + for row in rows_of(data, slug): + rows.setdefault(row["source_id"], row) + if cursor is None or (isinstance(total, int) and len(seen) >= total): + break + complete = isinstance(total, int) and len(seen) == total + if not complete: + log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total) + return list(rows.values()), complete + + +def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: + """One board, fetched and parsed: its rows and whether they are the whole + board. Raises on any failure — refused host, timeout, a feed shape none + of the parsers understand. A host that does not page answers with the + whole board, so it is complete whenever it parses; a paged one is + complete when `_paged` says so. A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES` gets one retry without it: the description is lost, but the titles land @@ -484,9 +583,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]: if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") + if host in _PAGED: + return _paged(client, target) url = _BOARDS[host][0].format(slug=slug) try: - body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET") + body = _get(client, url, host, slug, method="GET") except _TooLarge: if host != "greenhouse" or "?" not in url: raise @@ -494,18 +595,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]: "retrying without it", target, MAX_BYTES) url = url.split("?", 1)[0] body = _get(client, url, host, slug, method="GET") - return _PARSERS[host](body, slug) + return _PARSERS[host](body, slug), True -def fetch_board(target: str) -> list[Row]: +def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no `self.state`, just - "does this board answer, and what does it list". For `app companies - add|check`, which ask that question about one board at a time outside any - pull. Raises the same way `_fetch_rows` does.""" + "does this board answer, what does it list, and is that all of it". For + `app companies add|check`, which ask that question about one board at a + time outside any pull. Raises the same way `_fetch_rows` does.""" with httpx.Client() as client: return _fetch_rows(client, target) +def failure_reason(exc: BaseException) -> str: + """One short line for why a board did not answer: the status of an HTTP + error, else the exception's class and the first line of its message.""" + if isinstance(exc, httpx.HTTPStatusError): + return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() + lines = str(exc).strip().splitlines() + text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__ + return text[:200] + + # -------------------------------------------------------------- the plugin class ATSSource(Source): @@ -541,11 +652,23 @@ class ATSSource(Source): for target, label in self.targets(): host, _, slug = target.partition(":") + failing = f"ats:failing:{target}" try: - rows = _fetch_rows(client, target) + rows, complete = _fetch_rows(client, target) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) + # The first failure in a row sets `since`, later ones only + # the reason; `app.jobs.pull` closes the board's roles + # once `since` is `STALE_DAYS` old. + try: + since = json.loads(self.state.get(failing, "") or "{}").get("since") + except ValueError: + since = None + self.state[failing] = json.dumps({ + "since": since or datetime.now(UTC).isoformat(timespec="seconds"), + "reason": failure_reason(exc)}) continue + self.state.pop(failing, None) seen_ids = [] for row in rows: @@ -576,6 +699,9 @@ class ATSSource(Source): ) # Stamped only once the board has parsed without raising, so a # board that blew up half way through is neither marked ok nor - # has a partial id list treated as the whole of it. + # has a partial id list treated as the whole of it. `complete` + # says whether the ids are the whole board, which the pull + # needs before it closes what they leave out. self.state[f"ats:ok:{target}"] = run_marker + self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) diff --git a/plugins/search.py b/plugins/search.py index c73057d..26e4608 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -24,9 +24,12 @@ browser session of its own site) and Jooble (a key needs a manual application, and its API host answered a Cloudflare challenge). A search result list is never a board's complete list, so a role missing from -one run is not closed here. A search that ran out of results before its page -limit was complete, and the pull closes what it stopped listing; the rest -close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). +one run is not closed here. A search that read everything that matches was +complete, and the pull closes what it stopped listing; the rest close once no +search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states +how many results match, so an Adzuna search is complete when it read that +many; Jobindex states no count, so a Jobindex search is complete when a page +within the page limit came back short. """ from __future__ import annotations @@ -124,20 +127,28 @@ def _text(fragment: str) -> str: def parse_jobindex(body: bytes) -> list[Row]: - """One page of a Jobindex RSS search. An item is `` "Role, Company", + return _jobindex_page(body)[0] + + +def _jobindex_page(body: bytes) -> tuple[list[Row], int]: + """One page of a Jobindex RSS search, and how many items it had before + any was dropped. An item is `<title>` "Role, Company", a `<link>` to Jobindex's own page for it, and a `<description>` of HTML whose `<h4>` is the title alone, whose logo's `alt` is the company on a paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad ("robot job") has no logo, so its company is what follows the title's - last comma, or '' when there is none.""" + last comma, or '' when there is none. A body with no `<channel>` is not + a search result and raises, since an empty list would read as a search + that ran out and close everything it listed before.""" root = ET.fromstring(body) channel = root.find("channel") if channel is None: - return [] + raise ValueError("jobindex answered without an RSS channel") area_match = _SEARCH_AREA.search(channel.findtext("title") or "") search_area = area_match.group(1).strip() if area_match else "" rows = [] - for item in channel.findall("item"): + items = channel.findall("item") + for item in items: link = (item.findtext("link") or "").strip() full_title = (item.findtext("title") or "").strip() if not full_title or not _web_url(link): @@ -169,21 +180,24 @@ def parse_jobindex(body: bytes) -> list[Row]: "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), "salary_min": None, "salary_max": None, "currency": "", }) - return rows + return rows, len(items) def search_jobindex(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: """Every page of one search, newest first, until a short page or the page - limit. Returns the rows and whether the results ran out before the limit.""" + limit. Returns the rows and whether the results ran out: a page with + fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so + a dropped item does not pass for the end. Jobindex states no total, so a + search that read the page limit full is never complete.""" query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} if params.get("area"): query["geoareaid"] = params["area"] rows: list[Row] = [] for page in range(1, _pages(params) + 1): - got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page})) + got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page})) rows += got - if len(got) < JOBINDEX_PAGE: + if items < JOBINDEX_PAGE: return rows, True return rows, False @@ -202,7 +216,10 @@ def parse_adzuna(body: bytes, country: str) -> list[Row]: """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles and snippets carry `<strong>` highlights. A salary Adzuna estimated (`salary_is_predicted`) is not the posting's, so it is dropped.""" - data = json.loads(body) + return _adzuna_rows(json.loads(body), country) + + +def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]: rows = [] for job in data.get("results") or []: title = _text(job.get("title") or "") @@ -237,6 +254,9 @@ def _parse_iso(value: Any) -> datetime | None: def search_adzuna(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: + """Every page of one search, until a short page, the stated `count` or + the page limit. Returns the rows and whether they are all `count` says + match, or, on an answer with no `count`, whether a page came back short.""" app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") if not (app_id and app_key): @@ -252,13 +272,24 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any], if params.get("where"): query["where"] = params["where"] rows: list[Row] = [] + seen: set[str] = set() + count: Any = None + ran_out = False for page in range(1, _pages(params) + 1): url = ADZUNA_SEARCH.format(country=country, page=page) - got = parse_adzuna(_get(client, url, query), country) - rows += got - if len(got) < ADZUNA_PAGE: - return rows, True - return rows, False + data = json.loads(_get(client, url, query)) + results = data.get("results") or [] + count = data.get("count", count) + seen |= {str(job.get("id") or "") for job in results} + rows += _adzuna_rows(data, country) + if len(results) < ADZUNA_PAGE: + ran_out = True + break + if isinstance(count, int) and len(seen) >= count: + break + # Adzuna states how many results match; the search is complete when it + # read that many distinct ones. Without a count, a short page is the end. + return rows, (len(seen) == count) if isinstance(count, int) else ran_out _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], diff --git a/tests/fixtures/ats/smartrecruiters-2.json b/tests/fixtures/ats/smartrecruiters-2.json new file mode 100644 index 0000000..8e59f6d --- /dev/null +++ b/tests/fixtures/ats/smartrecruiters-2.json @@ -0,0 +1,16 @@ +{ + "offset": 2, + "limit": 2, + "totalFound": 3, + "content": [ + { + "id": "744000152001234", + "name": "Warehouse Operations Lead", + "company": {"identifier": "Acme", "name": "Acme"}, + "releasedDate": "2026-09-25T07:00:00.000Z", + "location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"}, + "department": {"id": "3", "label": "Operations"}, + "function": {"id": "operations", "label": "Operations"} + } + ] +} diff --git a/tests/fixtures/ats/smartrecruiters.json b/tests/fixtures/ats/smartrecruiters.json index 71e2429..46efc4f 100644 --- a/tests/fixtures/ats/smartrecruiters.json +++ b/tests/fixtures/ats/smartrecruiters.json @@ -1,7 +1,7 @@ { "offset": 0, - "limit": 100, - "totalFound": 2, + "limit": 2, + "totalFound": 3, "content": [ { "id": "744000151326819", diff --git a/tests/fixtures/ats/workable-2.json b/tests/fixtures/ats/workable-2.json new file mode 100644 index 0000000..e0d275e --- /dev/null +++ b/tests/fixtures/ats/workable-2.json @@ -0,0 +1,25 @@ +{ + "total": 3, + "results": [ + { + "id": 6125077, + "shortcode": "0A1B2C3D4E", + "title": "Maintenance Site Team Leader", + "remote": false, + "location": { + "country": "Ireland", + "countryCode": "IE", + "city": "Dublin", + "region": "County Dublin", + "display": "Dublin, County Dublin, Ireland" + }, + "state": "published", + "published": "2026-09-28T00:00:00.000Z", + "type": "full", + "department": [ + "Maintenance" + ], + "workplace": "on_site" + } + ] +} diff --git a/tests/fixtures/ats/workable.json b/tests/fixtures/ats/workable.json index 3047023..dd1397c 100644 --- a/tests/fixtures/ats/workable.json +++ b/tests/fixtures/ats/workable.json @@ -1,5 +1,6 @@ { - "total": 2, + "total": 3, + "nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=", "results": [ { "id": 6125026, diff --git a/tests/test_ats.py b/tests/test_ats.py index 12a959e..299ef2c 100644 --- a/tests/test_ats.py +++ b/tests/test_ats.py @@ -4,9 +4,12 @@ under `tests/fixtures/ats/`.""" from __future__ import annotations +import json import sys from pathlib import Path +from typing import Any +import httpx import pytest ROOT = Path(__file__).resolve().parents[1] @@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body( return (FIXTURES / "greenhouse.json").read_bytes() monkeypatch.setattr(ats, "_get", fake_get) - rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type] + rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type] assert calls == [ "https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true", "https://boards-api.greenhouse.io/v1/boards/acme/jobs", ] assert [r["source_id"] for r in rows] == ["1001", "1002"] + assert complete # a host that does not page answers with the whole board def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( @@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( monkeypatch.setattr(ats, "_get", fake_get) with pytest.raises(ats._TooLarge): ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type] + + +# ----------------------------------------------------------------- paging + +def _page(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def _status(code: int) -> httpx.HTTPStatusError: + request = httpx.Request("POST", "https://apply.workable.com/") + return httpx.HTTPStatusError(str(code), request=request, + response=httpx.Response(code, request=request)) + + +@pytest.fixture +def no_delay(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + + +def test_workable_follows_next_page_to_the_stated_total( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + calls: list[dict[str, Any] | None] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None): + calls.append(payload) + return _page("workable-2.json" if payload else "workable.json") + + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type] + + assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}] + assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"] + assert complete + + +def test_smartrecruiters_pages_by_offset_to_total_found( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + urls: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None): + urls.append(url) + return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json") + + monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] + + base = "https://api.smartrecruiters.com/v1/companies/Acme/postings" + assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"] + assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands" + assert complete + + +def test_a_board_that_ends_short_of_its_total_is_incomplete( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + """totalFound says 3; the second page is empty, so two were read.""" + empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode() + + def fake_get(client, url, host, slug, *, method="GET", payload=None): + return empty if "offset=2" in url else _page("smartrecruiters.json") + + monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] + assert (len(rows), complete) == (2, False) + + +def test_a_repeated_row_does_not_count_twice_toward_the_total( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + """A page that shifted under an offset repeats a role and misses one: three + rows read, two distinct, against a total of three.""" + def fake_get(client, url, host, slug, *, method="GET", payload=None): + page = json.loads(_page("smartrecruiters.json")) + if "offset=2" in url: + page["content"] = page["content"][1:] + return json.dumps(page).encode() + + monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2) + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type] + assert (len(rows), complete) == (2, False) + + +def test_the_page_cap_leaves_a_board_incomplete( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + calls: list[dict[str, Any] | None] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None): + calls.append(payload) + return _page("workable.json") # always another page + + monkeypatch.setattr(ats, "MAX_PAGES", 3) + monkeypatch.setattr(ats, "_get", fake_get) + _rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type] + assert (len(calls), complete) == (3, False) + + +def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + def fake_get(client, url, host, slug, *, method="GET", payload=None): + if payload: + raise _status(429) + return _page("workable.json") + + monkeypatch.setattr(ats, "_get", fake_get) + rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type] + assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False) + + +def test_a_429_on_the_first_page_is_a_failure( + monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None: + def fake_get(client, url, host, slug, *, method="GET", payload=None): + raise _status(429) + + monkeypatch.setattr(ats, "_get", fake_get) + with pytest.raises(httpx.HTTPStatusError): + ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type] + + +# -------------------------------------------------------- failing boards + +def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource: + """One board, `greenhouse:gone`, whose fetch answers `answer` or raises it.""" + def fake(client: object, target: str) -> object: + if isinstance(answer, Exception): + raise answer + return answer + + monkeypatch.setattr(ats, "_fetch_rows", fake) + source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]}) + source.state = {"ats:run": "run-1"} + return source + + +def test_a_failing_board_records_since_and_the_latest_reason( + monkeypatch: pytest.MonkeyPatch) -> None: + source = _source(monkeypatch, _status(404)) + assert list(source.fetch()) == [] + first = json.loads(source.state["ats:failing:greenhouse:gone"]) + assert first["reason"] == "404 Not Found" and first["since"] + assert "ats:ok:greenhouse:gone" not in source.state + + # A later failure keeps when the failures began and updates the reason. + source.state["ats:failing:greenhouse:gone"] = json.dumps( + first | {"since": "2026-09-01T00:00:00+00:00"}) + later = _source(monkeypatch, httpx.ConnectTimeout("timed out")) + later.state = source.state + list(later.fetch()) + assert json.loads(source.state["ats:failing:greenhouse:gone"]) == { + "since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"} + + +def test_an_answer_clears_the_failure_and_stamps_completeness( + monkeypatch: pytest.MonkeyPatch) -> None: + source = _source(monkeypatch, (_rows("greenhouse"), False)) + source.state["ats:failing:greenhouse:gone"] = json.dumps( + {"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"}) + assert len(list(source.fetch())) == 2 + assert "ats:failing:greenhouse:gone" not in source.state + assert source.state["ats:ok:greenhouse:gone"] == "run-1" + assert source.state["ats:complete:greenhouse:gone"] == "" diff --git a/tests/test_pull.py b/tests/test_pull.py index 191bfec..2efbdac 100644 --- a/tests/test_pull.py +++ b/tests/test_pull.py @@ -4,8 +4,9 @@ touches the network.""" from __future__ import annotations +import json import sys -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import ClassVar @@ -17,6 +18,7 @@ if str(ROOT) not in sys.path: from desk.models import Item # noqa: E402 from pipeline import Cancelled, RunContext # noqa: E402 +from plugins import ats # noqa: E402 from plugins.ats import ATSSource # noqa: E402 from app.jobs import pull # noqa: E402 from app.storage import Store # noqa: E402 @@ -126,6 +128,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( # (or its batch never got to it) and stamped nothing. store.state["ats:run"] = "run-1" store.state["ats:ok:greenhouse:acme"] = "run-1" + store.state["ats:complete:greenhouse:acme"] = "1" store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' class FakeTally: @@ -133,8 +136,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1, - "closed": 1} + assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0, + "roles": 1, "closed": 1} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} @@ -155,6 +158,91 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] - assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0, - "closed": 0} + assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0, + "roles": 0, "closed": 0} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} + + +# ---------------------------------------------- whole pulls, fetch stubbed + +def _row(job_id: str) -> dict[str, object]: + return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin", + "url": f"https://x.example/{job_id}", "posted": None, "department": "", + "description": "", "remote": None, "salary": ""} + + +def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None: + """Every board's fetch returns `answer`, `(rows, complete)`, or raises it.""" + def fake(client: object, target: str) -> object: + if isinstance(answer, Exception): + raise answer + return answer + + monkeypatch.setattr(ats, "_fetch_rows", fake) + + +def _pull(store: Store) -> dict[str, object]: + for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type] + pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + + class Tally: + results: ClassVar[list[dict[str, object]]] = [] + + return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + + +def _open(store: Store) -> set[str]: + return {r.source_id for r in store.roles()} + + +@pytest.fixture +def acme(store: Store) -> Store: + store.add_subscription("ats", "workable:acme", label="Acme", company="Acme", + board="workable") + return store + + +def test_an_incomplete_board_stores_what_it_read_and_closes_nothing( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row("1"), _row("2")], True)) + _pull(acme) + # A 429 on page two: page one's role is stored again, the other stays open. + _answer(monkeypatch, ([_row("1"), _row("3")], False)) + result = _pull(acme) + assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"} + assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0) + + +def test_a_failing_board_keeps_its_roles_before_stale_days( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row("1")], True)) + _pull(acme) + _answer(monkeypatch, ValueError("gone")) + result = _pull(acme) + assert _open(acme) == {"workable:acme:1"} + assert (result["failed"], result["closed"]) == (1, 0) + since, reason = pull.failing_boards(acme)["workable:acme"] + assert since and reason == "ValueError: gone" + + +def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens( + acme: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _answer(monkeypatch, ([_row("1"), _row("2")], True)) + _pull(acme) + _answer(monkeypatch, ValueError("404 for good")) + _pull(acme) + old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat( + timespec="seconds") + acme.state[f"{pull.FAILING}workable:acme"] = json.dumps( + {"since": old, "reason": "404 Not Found"}) + + result = _pull(acme) + assert _open(acme) == set() + assert result["closed"] == 2 + # Still failing, and still the date the failures began. + assert pull.failing_boards(acme)["workable:acme"][0] == old + + _answer(monkeypatch, ([_row("2")], True)) + _pull(acme) + assert _open(acme) == {"workable:acme:2"} + assert pull.failing_boards(acme) == {} diff --git a/tests/test_search.py b/tests/test_search.py index c1c5c62..8f5e16d 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one from __future__ import annotations +import json import sys from datetime import UTC, datetime, timedelta from pathlib import Path @@ -139,16 +140,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N return b"short" if params["page"] == 3 else b"full" monkeypatch.setattr(search, "_get", fake_get) - monkeypatch.setattr(search, "parse_jobindex", - lambda body: full[:5] if body == b"short" else full) + monkeypatch.setattr(search, "_jobindex_page", + lambda body: (full[:5], 5) if body == b"short" else (full, len(full))) rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) + # Stopped by the page limit with every page full: not complete. pages.clear() rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] assert (pages, complete) == ([1, 2], False) +def test_a_dropped_jobindex_item_does_not_pass_for_the_end( + monkeypatch: pytest.MonkeyPatch) -> None: + """A full page whose parser dropped one item still has 20 items, so the + search reads on instead of calling itself complete at 19 rows.""" + full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)] + monkeypatch.setattr(search, "_get", lambda client, url, params: b"") + monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full))) + _rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] + assert not complete + + +def test_a_jobindex_answer_without_a_channel_raises() -> None: + with pytest.raises(ValueError, match="channel"): + search.parse_jobindex(b"<rss version='2.0'></rss>") + + +def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]: + """Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`.""" + asked: list[int] = [] + + def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes: + page = int(url.rsplit("/", 1)[1]) + asked.append(page) + start = (page - 1) * search.ADZUNA_PAGE + results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"} + for n in range(start, min(start + search.ADZUNA_PAGE, 60))] + body: dict[str, Any] = {"results": results} + if count is not None: + body["count"] = count + return json.dumps(body).encode() + + monkeypatch.setattr(search, "_get", fake_get) + monkeypatch.setenv("ADZUNA_APP_ID", "id") + monkeypatch.setenv("ADZUNA_APP_KEY", "key") + return asked + + +@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)]) +def test_an_adzuna_search_is_complete_when_it_read_its_count( + monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None: + _adzuna(monkeypatch, count) + rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] + assert (len(rows), done) == (60, complete) + + +def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None: + asked = _adzuna(monkeypatch, search.ADZUNA_PAGE) + rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type] + assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True) + + def test_provider_of() -> None: assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("adzuna:berlin") == "adzuna" diff --git a/tests/test_web.py b/tests/test_web.py index 48108b9..3bf63a8 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app.""" from __future__ import annotations +import json from collections.abc import Iterator from datetime import UTC, datetime, timedelta from pathlib import Path @@ -395,7 +396,8 @@ def test_companies_lists_board_and_open_role_count( [row] = [c for c in res.json() if c["key"] == "acme"] assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, - "ats": "greenhouse", "careers_url": "", "country": "DK"} + "ats": "greenhouse", "careers_url": "", "country": "DK", + "failing_since": "", "failing_reason": ""} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") @@ -404,6 +406,32 @@ def test_companies_lists_board_and_open_role_count( "", None, "successfactors", "https://novo.example/careers") +def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch) -> None: + """What the pull recorded, on `/api/companies` and beside `app companies + check`'s live result.""" + import plugins.ats as ats + from app.cli import check_companies + + store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse") + store.add_company("Acme", key="acme", ats="greenhouse") + store.state["ats:failing:greenhouse:acme"] = json.dumps( + {"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"}) + + [row] = client.get("/api/companies").json() + assert (row["failing_since"], row["failing_reason"]) == ( + "2026-09-01T06:00:00+00:00", "404 Not Found") + + monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True)) + assert check_companies(store) == 0 + assert capsys.readouterr().out.splitlines() == [ + "greenhouse:acme (greenhouse:acme): now 1 roles; " + "pull: failing since 2026-09-01: 404 Not Found", + "checked 1 boards, 0 dead, 1 failing in the pull", + ] + + def test_set_affinity_validates_updates_and_enqueues_a_score( client: TestClient, store: Store, queue: Any) -> None: store.add_company("Acme", key="acme") @@ -909,7 +937,7 @@ def test_a_board_is_followed_only_if_it_answers( client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: import plugins.ats as ats - monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}]) + monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True)) res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"}) assert res.status_code == 201 assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"