Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
c7da12b93d
18 files changed
+695
-91
No files matched your search
+18
-9
@@ -78,7 +78,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
discover.add_argument("domain", help="its website, e.g. pleo.io")
|
||||
discover.add_argument("--add", action="store_true",
|
||||
help="add the company, with its board when one is readable")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
|
||||
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
@@ -322,19 +322,28 @@ def add_company(store: Store, board: str, name: str) -> int:
|
||||
|
||||
def check_companies(store: Store) -> int:
|
||||
"""`app companies check`: every `ats` board, fetched once. Prints a
|
||||
line per board that did not answer and exits 1 if any did not."""
|
||||
from plugins.ats import fetch_board
|
||||
line per board that did not answer now, answered incompletely, or has
|
||||
been failing in the pull, with what the pull recorded beside the live
|
||||
result, and exits 1 if any did not answer now."""
|
||||
from app import jobs
|
||||
from plugins.ats import failure_reason, fetch_board
|
||||
|
||||
subs = store.subscriptions(source="ats")
|
||||
dead = []
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
dead = 0
|
||||
for sub in subs:
|
||||
try:
|
||||
fetch_board(sub.target)
|
||||
rows, complete = fetch_board(sub.target)
|
||||
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
|
||||
except Exception as exc:
|
||||
dead.append((sub.target, sub.label, str(exc)))
|
||||
for target, label, reason in dead:
|
||||
print(f"{target} ({label}): {reason}")
|
||||
print(f"checked {len(subs)} boards, {len(dead)} dead")
|
||||
complete, live = False, f"not answering: {failure_reason(exc)}"
|
||||
dead += 1
|
||||
since, reason = failing.get(sub.target, ("", ""))
|
||||
if complete and not since:
|
||||
continue
|
||||
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
|
||||
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
|
||||
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
|
||||
+52
-19
@@ -7,15 +7,19 @@ sized for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which each plugin itself stamps per target
|
||||
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
||||
under `search:`), rather than from what each `pull_batch` job happens to
|
||||
return. A batch job that raises partway through still leaves the boards it had
|
||||
already finished stamped, so one bad board in a batch of ten does not cost the
|
||||
other nine their closure.
|
||||
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
||||
`ats:ids:<target>`, and the same under `search:`), rather than from what each
|
||||
`pull_batch` job happens to return. A batch job that raises partway through
|
||||
still leaves the boards it had already finished stamped, so one bad board in a
|
||||
batch of ten does not cost the other nine their closure. A board that answered
|
||||
but was not read to its stated total (a paged host that hit a 429 or the page
|
||||
cap) stores what it read and closes nothing.
|
||||
|
||||
A search closes the same way when its results ran out before its page limit,
|
||||
since it then listed everything that matches. One that hit the limit did not,
|
||||
so its roles close only once no search has seen them for `STALE_DAYS`.
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,15 +47,22 @@ SETTLE = "app.jobs.pull:close_boards"
|
||||
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
||||
SOURCES = ("ats", "search")
|
||||
|
||||
#: How long a search role stays open once no search lists it any more.
|
||||
#: How long a search role stays open once no search lists it any more, and
|
||||
#: how long a board may fail on every fetch before its roles close.
|
||||
STALE_DAYS = 21
|
||||
|
||||
#: The `state` prefix the `ats` plugin records a failing board under:
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
"""A run marker: to the microsecond, so two pulls a second apart are two
|
||||
runs and a stamp from one never matches the other."""
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
@@ -124,14 +135,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed. A search closes by its
|
||||
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
"""
|
||||
answered = searched = failed = closed = 0
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
@@ -141,21 +156,39 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
if not store.state.get(f"search:complete:{sub.target}", ""):
|
||||
continue
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
||||
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
||||
closed += store.close_unseen("search", stale)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
||||
"closed": closed}
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
def search_target(provider: str, name: str) -> str:
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
if store.company(key) is not None:
|
||||
raise ValueError(f"{name} is already followed")
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
rows, _complete = fetch_board(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
if not rows:
|
||||
|
||||
@@ -1200,6 +1200,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
gap: .4rem; font-size: .8rem; color: var(--dim);
|
||||
}
|
||||
.srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; }
|
||||
.crow .sub .failing { color: var(--warn); }
|
||||
|
||||
.linky {
|
||||
padding: 0; border: 0; background: none; font-size: .8rem;
|
||||
|
||||
@@ -73,6 +73,16 @@
|
||||
sub.appendChild(link);
|
||||
}
|
||||
sub.append(` · ${count(company.open_roles, "open role")}`);
|
||||
if (company.failing_since) {
|
||||
// The board has failed on every pull since then; after 21 days its
|
||||
// roles close.
|
||||
const failing = document.createElement("span");
|
||||
failing.className = "failing";
|
||||
failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}`
|
||||
+ (company.failing_reason ? `: ${company.failing_reason}` : "");
|
||||
sub.append(" · ");
|
||||
sub.appendChild(failing);
|
||||
}
|
||||
return sub;
|
||||
}
|
||||
|
||||
|
||||
@@ -1230,6 +1230,13 @@ class Store:
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
|
||||
def state_items(self, prefix: str) -> dict[str, str]:
|
||||
"""Every `state` key that starts with `prefix`, with its value, in one
|
||||
query, for a reader that wants one kind of key across every target."""
|
||||
return {str(r["key"]): str(r["value"]) for r in self._rows(
|
||||
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
|
||||
(len(prefix), prefix))}
|
||||
|
||||
|
||||
def _check_status(status: str) -> None:
|
||||
if status not in STATUSES:
|
||||
|
||||
+15
-8
@@ -264,7 +264,9 @@ class CompanyOut(BaseModel):
|
||||
board's subscription, whether it is on, and how many of its roles are
|
||||
still open. `ats` is the host its careers site runs on whether or not the
|
||||
app reads it, so a followed company with no `board` and an `ats` is one
|
||||
the app cannot read yet; `careers_url` is where a person can."""
|
||||
the app cannot read yet; `careers_url` is where a person can.
|
||||
`failing_since` and `failing_reason` say since when the board has failed
|
||||
on every fetch and why it last did, both empty while it answers."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
@@ -276,17 +278,19 @@ class CompanyOut(BaseModel):
|
||||
ats: str = ""
|
||||
careers_url: str = ""
|
||||
country: str = ""
|
||||
failing_since: str = ""
|
||||
failing_reason: str = ""
|
||||
|
||||
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]:
|
||||
"""`company_key -> (subscription id, ats host, on)`, read off each
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
|
||||
"""`company_key -> (subscription id, ats host, on, target)`, read off each
|
||||
company's own subscription. Same join as `Store.company_boards`, with the
|
||||
id and switch state a company row needs to draw its own on/off control
|
||||
rather than fetching `/api/sources` as well."""
|
||||
boards: dict[str, tuple[int, str, bool]] = {}
|
||||
boards: dict[str, tuple[int, str, bool, str]] = {}
|
||||
for sub in store.subscriptions(source="ats"):
|
||||
boards.setdefault(company_key(sub.company), (sub.id, sub.board,
|
||||
bool(sub.subscribed and sub.fetch)))
|
||||
boards.setdefault(company_key(sub.company), (
|
||||
sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target))
|
||||
return boards
|
||||
|
||||
|
||||
@@ -296,13 +300,16 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
absent) has it; 0 throughout when there is no agent."""
|
||||
boards = _company_boards(store)
|
||||
counts = store.open_role_counts()
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
out = []
|
||||
for c in store.companies(_agent_id(store, agent)):
|
||||
sub_id, host, on = boards.get(c.key, (None, "", None))
|
||||
sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
|
||||
since, reason = failing.get(target, ("", ""))
|
||||
out.append(CompanyOut(key=c.key, name=c.name, board=host,
|
||||
subscription_id=sub_id, on=on, affinity=c.affinity,
|
||||
open_roles=counts.get(c.key, 0), ats=c.ats,
|
||||
careers_url=c.careers_url, country=c.country))
|
||||
careers_url=c.careers_url, country=c.country,
|
||||
failing_since=since, failing_reason=reason))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
+6
-2
@@ -38,7 +38,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
## Ingestion
|
||||
|
||||
**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker and the ids it saw, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list.
|
||||
**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker, the ids it saw and whether they are the whole board, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran and the read was complete. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list.
|
||||
|
||||
**Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards.
|
||||
|
||||
@@ -50,7 +50,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API.
|
||||
|
||||
**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. A search whose results ran out before its page limit did list everything, and closes like a board; the rest close once no search has listed them for 21 days. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists.
|
||||
**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists.
|
||||
|
||||
**Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later.
|
||||
|
||||
@@ -64,6 +64,10 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role.
|
||||
|
||||
**Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse.
|
||||
|
||||
**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result.
|
||||
|
||||
**`pull` is a job that starts the cascade, on a default schedule of every 6h.** A schedule enqueues a job kind and a cascade is started by one, so the `pull` job only asks for the chain; declared beside it in `app/jobs/cascade.py`, the schedule is a starting point that an edit under Runs overrides for good. `pull` is not also a task, which would draw two buttons under Runs.
|
||||
|
||||
## Ranking
|
||||
|
||||
+148
-22
@@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:acme`. Every board is fetched independently; one that fails or times
|
||||
out is logged and skipped, and the rest continue.
|
||||
out is logged, recorded as failing under `ats:failing:<target>` (see
|
||||
`ATSSource.fetch`), and skipped, and the rest continue.
|
||||
|
||||
Workable and SmartRecruiters page their list and state a total; every other
|
||||
host answers with the whole board at once. A paged board counts as complete
|
||||
only when it read as many distinct roles as the total says (see `_paged`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,6 +32,7 @@ import html
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
@@ -119,10 +125,11 @@ class _TooLarge(ValueError):
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET") -> bytes:
|
||||
method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
|
||||
"""Fetch one board feed: https only, the host's own allow-list, at most
|
||||
three redirect hops each checked the same way, and the body cut off at
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`."""
|
||||
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none."""
|
||||
allow = _allowed_host(host, slug)
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
@@ -132,7 +139,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
response = client.request(
|
||||
method, current, timeout=TIMEOUT, follow_redirects=False,
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
|
||||
json={} if method == "POST" else None,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
)
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
@@ -258,15 +265,20 @@ def _parse_ashby(body: bytes) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` ->
|
||||
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
|
||||
2026-09-28. A result carries no URL and no text: the job page is
|
||||
return _workable_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. Ten results a page; the next page is
|
||||
the same POST with `{"token": nextPage}`, and the last page has no
|
||||
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
|
||||
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
|
||||
is only on the per-job endpoint, one request per role, so `description`
|
||||
stays empty here as it does for SmartRecruiters. `location` is an object
|
||||
and `department` a list; `remote` is a flag beside `workplace`
|
||||
(remote, hybrid, on_site)."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
@@ -293,11 +305,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]:
|
||||
|
||||
|
||||
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
|
||||
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings ->
|
||||
`{"content": [...]}`. The list omits the job description (only the
|
||||
return _smartrecruiters_rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on (see
|
||||
`_paged`). The list omits the job description (only the
|
||||
per-posting detail endpoint has it, and fetching one per row would be
|
||||
hundreds of extra requests per board), so `description` stays empty here."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
@@ -474,9 +491,91 @@ def careers_url(target: str) -> str:
|
||||
return _BOARDS[host][1].format(slug=slug)
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"""One board, fetched and parsed. Raises on any failure — refused host,
|
||||
timeout, a feed shape none of the parsers understand.
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
|
||||
#: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
|
||||
#: a burst with 429, from seeing one.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
#: SmartRecruiters' page size, which is also the most it serves per page.
|
||||
SMARTRECRUITERS_PAGE = 100
|
||||
|
||||
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
|
||||
#: raw results, the total it states, and the cursor of the next page (None
|
||||
#: after the last). The first call gets cursor None.
|
||||
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
url = _BOARDS["workable"][0].format(slug=slug)
|
||||
data = json.loads(_get(client, url, "workable", slug, method="POST",
|
||||
payload={"token": cursor} if cursor else {}))
|
||||
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
||||
|
||||
|
||||
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
|
||||
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
|
||||
data = json.loads(_get(client, url, "smartrecruiters", slug))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
#: host -> (one page, that page's rows) for the hosts that page.
|
||||
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
|
||||
Callable[[dict[str, Any], str], list[Row]]]] = {
|
||||
"workable": (_workable_page, _workable_rows),
|
||||
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
|
||||
}
|
||||
|
||||
|
||||
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
|
||||
until the host has no next page, the stated total is reached, or
|
||||
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
|
||||
distinct results read as the host's stated total. Counted on the raw
|
||||
results, so a row the parser drops does not make a board incomplete, and
|
||||
on distinct ids, so a page that shifted under an offset and repeated a
|
||||
role does not make it complete.
|
||||
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
page_of, rows_of = _PAGED[host]
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
cursor: Any = None
|
||||
for page in range(MAX_PAGES):
|
||||
if page:
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
data, results, total, cursor = page_of(client, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not page or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
||||
target, page + 1, len(seen), total)
|
||||
return list(rows.values()), False
|
||||
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
for row in rows_of(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if cursor is None or (isinstance(total, int) and len(seen) >= total):
|
||||
break
|
||||
complete = isinstance(total, int) and len(seen) == total
|
||||
if not complete:
|
||||
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
|
||||
return list(rows.values()), complete
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""One board, fetched and parsed: its rows and whether they are the whole
|
||||
board. Raises on any failure — refused host, timeout, a feed shape none
|
||||
of the parsers understand. A host that does not page answers with the
|
||||
whole board, so it is complete whenever it parses; a paged one is
|
||||
complete when `_paged` says so.
|
||||
|
||||
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
|
||||
gets one retry without it: the description is lost, but the titles land
|
||||
@@ -484,9 +583,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
if host in _PAGED:
|
||||
return _paged(client, target)
|
||||
url = _BOARDS[host][0].format(slug=slug)
|
||||
try:
|
||||
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET")
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
except _TooLarge:
|
||||
if host != "greenhouse" or "?" not in url:
|
||||
raise
|
||||
@@ -494,18 +595,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
|
||||
"retrying without it", target, MAX_BYTES)
|
||||
url = url.split("?", 1)[0]
|
||||
body = _get(client, url, host, slug, method="GET")
|
||||
return _PARSERS[host](body, slug)
|
||||
return _PARSERS[host](body, slug), True
|
||||
|
||||
|
||||
def fetch_board(target: str) -> list[Row]:
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, and what does it list". For `app companies
|
||||
add|check`, which ask that question about one board at a time outside any
|
||||
pull. Raises the same way `_fetch_rows` does."""
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class ATSSource(Source):
|
||||
@@ -541,11 +652,23 @@ class ATSSource(Source):
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
# The first failure in a row sets `since`, later ones only
|
||||
# the reason; `app.jobs.pull` closes the board's roles
|
||||
# once `since` is `STALE_DAYS` old.
|
||||
try:
|
||||
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
||||
except ValueError:
|
||||
since = None
|
||||
self.state[failing] = json.dumps({
|
||||
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
@@ -576,6 +699,9 @@ class ATSSource(Source):
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
+48
-17
@@ -24,9 +24,12 @@ browser session of its own site) and Jooble (a key needs a manual
|
||||
application, and its API host answered a Cloudflare challenge).
|
||||
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that ran out of results before its page
|
||||
limit was complete, and the pull closes what it stopped listing; the rest
|
||||
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
one run is not closed here. A search that read everything that matches was
|
||||
complete, and the pull closes what it stopped listing; the rest close once no
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
|
||||
how many results match, so an Adzuna search is complete when it read that
|
||||
many; Jobindex states no count, so a Jobindex search is complete when a page
|
||||
within the page limit came back short.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -124,20 +127,28 @@ def _text(fragment: str) -> str:
|
||||
|
||||
|
||||
def parse_jobindex(body: bytes) -> list[Row]:
|
||||
"""One page of a Jobindex RSS search. An item is `<title>` "Role, Company",
|
||||
return _jobindex_page(body)[0]
|
||||
|
||||
|
||||
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
||||
"""One page of a Jobindex RSS search, and how many items it had before
|
||||
any was dropped. An item is `<title>` "Role, Company",
|
||||
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
||||
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
||||
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's
|
||||
last comma, or '' when there is none."""
|
||||
last comma, or '' when there is none. A body with no `<channel>` is not
|
||||
a search result and raises, since an empty list would read as a search
|
||||
that ran out and close everything it listed before."""
|
||||
root = ET.fromstring(body)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
return []
|
||||
raise ValueError("jobindex answered without an RSS channel")
|
||||
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
search_area = area_match.group(1).strip() if area_match else ""
|
||||
rows = []
|
||||
for item in channel.findall("item"):
|
||||
items = channel.findall("item")
|
||||
for item in items:
|
||||
link = (item.findtext("link") or "").strip()
|
||||
full_title = (item.findtext("title") or "").strip()
|
||||
if not full_title or not _web_url(link):
|
||||
@@ -169,21 +180,24 @@ def parse_jobindex(body: bytes) -> list[Row]:
|
||||
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
return rows, len(items)
|
||||
|
||||
|
||||
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page or the page
|
||||
limit. Returns the rows and whether the results ran out before the limit."""
|
||||
limit. Returns the rows and whether the results ran out: a page with
|
||||
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
||||
a dropped item does not pass for the end. Jobindex states no total, so a
|
||||
search that read the page limit full is never complete."""
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
||||
if params.get("area"):
|
||||
query["geoareaid"] = params["area"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
rows += got
|
||||
if len(got) < JOBINDEX_PAGE:
|
||||
if items < JOBINDEX_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
@@ -202,7 +216,10 @@ def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
||||
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
||||
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
||||
data = json.loads(body)
|
||||
return _adzuna_rows(json.loads(body), country)
|
||||
|
||||
|
||||
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
||||
rows = []
|
||||
for job in data.get("results") or []:
|
||||
title = _text(job.get("title") or "")
|
||||
@@ -237,6 +254,9 @@ def _parse_iso(value: Any) -> datetime | None:
|
||||
|
||||
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, until a short page, the stated `count` or
|
||||
the page limit. Returns the rows and whether they are all `count` says
|
||||
match, or, on an answer with no `count`, whether a page came back short."""
|
||||
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
||||
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
||||
if not (app_id and app_key):
|
||||
@@ -252,13 +272,24 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
if params.get("where"):
|
||||
query["where"] = params["where"]
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
count: Any = None
|
||||
ran_out = False
|
||||
for page in range(1, _pages(params) + 1):
|
||||
url = ADZUNA_SEARCH.format(country=country, page=page)
|
||||
got = parse_adzuna(_get(client, url, query), country)
|
||||
rows += got
|
||||
if len(got) < ADZUNA_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
data = json.loads(_get(client, url, query))
|
||||
results = data.get("results") or []
|
||||
count = data.get("count", count)
|
||||
seen |= {str(job.get("id") or "") for job in results}
|
||||
rows += _adzuna_rows(data, country)
|
||||
if len(results) < ADZUNA_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(count, int) and len(seen) >= count:
|
||||
break
|
||||
# Adzuna states how many results match; the search is complete when it
|
||||
# read that many distinct ones. Without a count, a short page is the end.
|
||||
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"offset": 2,
|
||||
"limit": 2,
|
||||
"totalFound": 3,
|
||||
"content": [
|
||||
{
|
||||
"id": "744000152001234",
|
||||
"name": "Warehouse Operations Lead",
|
||||
"company": {"identifier": "Acme", "name": "Acme"},
|
||||
"releasedDate": "2026-09-25T07:00:00.000Z",
|
||||
"location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"},
|
||||
"department": {"id": "3", "label": "Operations"},
|
||||
"function": {"id": "operations", "label": "Operations"}
|
||||
}
|
||||
]
|
||||
}
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"offset": 0,
|
||||
"limit": 100,
|
||||
"totalFound": 2,
|
||||
"limit": 2,
|
||||
"totalFound": 3,
|
||||
"content": [
|
||||
{
|
||||
"id": "744000151326819",
|
||||
|
||||
Vendored
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"total": 3,
|
||||
"results": [
|
||||
{
|
||||
"id": 6125077,
|
||||
"shortcode": "0A1B2C3D4E",
|
||||
"title": "Maintenance Site Team Leader",
|
||||
"remote": false,
|
||||
"location": {
|
||||
"country": "Ireland",
|
||||
"countryCode": "IE",
|
||||
"city": "Dublin",
|
||||
"region": "County Dublin",
|
||||
"display": "Dublin, County Dublin, Ireland"
|
||||
},
|
||||
"state": "published",
|
||||
"published": "2026-09-28T00:00:00.000Z",
|
||||
"type": "full",
|
||||
"department": [
|
||||
"Maintenance"
|
||||
],
|
||||
"workplace": "on_site"
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+2
-1
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"total": 2,
|
||||
"total": 3,
|
||||
"nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=",
|
||||
"results": [
|
||||
{
|
||||
"id": 6125026,
|
||||
|
||||
+166
-1
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
|
||||
return (FIXTURES / "greenhouse.json").read_bytes()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
|
||||
"https://boards-api.greenhouse.io/v1/boards/acme/jobs",
|
||||
]
|
||||
assert [r["source_id"] for r in rows] == ["1001", "1002"]
|
||||
assert complete # a host that does not page answers with the whole board
|
||||
|
||||
|
||||
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(ats._TooLarge):
|
||||
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- paging
|
||||
|
||||
def _page(name: str) -> bytes:
|
||||
return (FIXTURES / name).read_bytes()
|
||||
|
||||
|
||||
def _status(code: int) -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("POST", "https://apply.workable.com/")
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
|
||||
|
||||
def test_workable_follows_next_page_to_the_stated_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable-2.json" if payload else "workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
|
||||
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
|
||||
assert complete
|
||||
|
||||
|
||||
def test_smartrecruiters_pages_by_offset_to_total_found(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
urls: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
urls.append(url)
|
||||
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
|
||||
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
|
||||
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
|
||||
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
|
||||
assert complete
|
||||
|
||||
|
||||
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""totalFound says 3; the second page is empty, so two were read."""
|
||||
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
return empty if "offset=2" in url else _page("smartrecruiters.json")
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_a_repeated_row_does_not_count_twice_toward_the_total(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
"""A page that shifted under an offset repeats a role and misses one: three
|
||||
rows read, two distinct, against a total of three."""
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
page = json.loads(_page("smartrecruiters.json"))
|
||||
if "offset=2" in url:
|
||||
page["content"] = page["content"][1:]
|
||||
return json.dumps(page).encode()
|
||||
|
||||
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
|
||||
assert (len(rows), complete) == (2, False)
|
||||
|
||||
|
||||
def test_the_page_cap_leaves_a_board_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
calls: list[dict[str, Any] | None] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
calls.append(payload)
|
||||
return _page("workable.json") # always another page
|
||||
|
||||
monkeypatch.setattr(ats, "MAX_PAGES", 3)
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert (len(calls), complete) == (3, False)
|
||||
|
||||
|
||||
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
if payload:
|
||||
raise _status(429)
|
||||
return _page("workable.json")
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
|
||||
|
||||
|
||||
def test_a_429_on_the_first_page_is_a_failure(
|
||||
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None):
|
||||
raise _status(429)
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# -------------------------------------------------------- failing boards
|
||||
|
||||
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
|
||||
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}]})
|
||||
source.state = {"ats:run": "run-1"}
|
||||
return source
|
||||
|
||||
|
||||
def test_a_failing_board_records_since_and_the_latest_reason(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, _status(404))
|
||||
assert list(source.fetch()) == []
|
||||
first = json.loads(source.state["ats:failing:greenhouse:gone"])
|
||||
assert first["reason"] == "404 Not Found" and first["since"]
|
||||
assert "ats:ok:greenhouse:gone" not in source.state
|
||||
|
||||
# A later failure keeps when the failures began and updates the reason.
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
first | {"since": "2026-09-01T00:00:00+00:00"})
|
||||
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
|
||||
later.state = source.state
|
||||
list(later.fetch())
|
||||
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
|
||||
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
|
||||
|
||||
|
||||
def test_an_answer_clears_the_failure_and_stamps_completeness(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
source = _source(monkeypatch, (_rows("greenhouse"), False))
|
||||
source.state["ats:failing:greenhouse:gone"] = json.dumps(
|
||||
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
|
||||
assert len(list(source.fetch())) == 2
|
||||
assert "ats:failing:greenhouse:gone" not in source.state
|
||||
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
|
||||
assert source.state["ats:complete:greenhouse:gone"] == ""
|
||||
+93
-5
@@ -4,8 +4,9 @@ touches the network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -17,6 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext # noqa: E402
|
||||
from plugins import ats # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
@@ -126,6 +128,7 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
# (or its batch never got to it) and stamped nothing.
|
||||
store.state["ats:run"] = "run-1"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:complete:greenhouse:acme"] = "1"
|
||||
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
class FakeTally:
|
||||
@@ -133,8 +136,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
|
||||
"closed": 1}
|
||||
assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
"roles": 1, "closed": 1}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||
|
||||
|
||||
@@ -155,6 +158,91 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
|
||||
"closed": 0}
|
||||
assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
|
||||
"roles": 0, "closed": 0}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
|
||||
|
||||
# ---------------------------------------------- whole pulls, fetch stubbed
|
||||
|
||||
def _row(job_id: str) -> dict[str, object]:
|
||||
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
|
||||
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
|
||||
"description": "", "remote": None, "salary": ""}
|
||||
|
||||
|
||||
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
|
||||
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
|
||||
def fake(client: object, target: str) -> object:
|
||||
if isinstance(answer, Exception):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_fetch_rows", fake)
|
||||
|
||||
|
||||
def _pull(store: Store) -> dict[str, object]:
|
||||
for batch in pull._boards(FakeConfig(), store): # type: ignore[arg-type]
|
||||
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
||||
|
||||
class Tally:
|
||||
results: ClassVar[list[dict[str, object]]] = []
|
||||
|
||||
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _open(store: Store) -> set[str]:
|
||||
return {r.source_id for r in store.roles()}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def acme(store: Store) -> Store:
|
||||
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
||||
board="workable")
|
||||
return store
|
||||
|
||||
|
||||
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
_pull(acme)
|
||||
# A 429 on page two: page one's role is stored again, the other stays open.
|
||||
_answer(monkeypatch, ([_row("1"), _row("3")], False))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
||||
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
||||
|
||||
|
||||
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1")], True))
|
||||
_pull(acme)
|
||||
_answer(monkeypatch, ValueError("gone"))
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == {"workable:acme:1"}
|
||||
assert (result["failed"], result["closed"]) == (1, 0)
|
||||
since, reason = pull.failing_boards(acme)["workable:acme"]
|
||||
assert since and reason == "ValueError: gone"
|
||||
|
||||
|
||||
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
||||
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
||||
_pull(acme)
|
||||
_answer(monkeypatch, ValueError("404 for good"))
|
||||
_pull(acme)
|
||||
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
||||
timespec="seconds")
|
||||
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
||||
{"since": old, "reason": "404 Not Found"})
|
||||
|
||||
result = _pull(acme)
|
||||
assert _open(acme) == set()
|
||||
assert result["closed"] == 2
|
||||
# Still failing, and still the date the failures began.
|
||||
assert pull.failing_boards(acme)["workable:acme"][0] == old
|
||||
|
||||
_answer(monkeypatch, ([_row("2")], True))
|
||||
_pull(acme)
|
||||
assert _open(acme) == {"workable:acme:2"}
|
||||
assert pull.failing_boards(acme) == {}
|
||||
+55
-2
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -139,16 +140,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
|
||||
return b"short" if params["page"] == 3 else b"full"
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setattr(search, "parse_jobindex",
|
||||
lambda body: full[:5] if body == b"short" else full)
|
||||
monkeypatch.setattr(search, "_jobindex_page",
|
||||
lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
|
||||
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
|
||||
|
||||
# Stopped by the page limit with every page full: not complete.
|
||||
pages.clear()
|
||||
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert (pages, complete) == ([1, 2], False)
|
||||
|
||||
|
||||
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""A full page whose parser dropped one item still has 20 items, so the
|
||||
search reads on instead of calling itself complete at 19 rows."""
|
||||
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
|
||||
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
|
||||
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
|
||||
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
|
||||
assert not complete
|
||||
|
||||
|
||||
def test_a_jobindex_answer_without_a_channel_raises() -> None:
|
||||
with pytest.raises(ValueError, match="channel"):
|
||||
search.parse_jobindex(b"<rss version='2.0'></rss>")
|
||||
|
||||
|
||||
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
|
||||
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
|
||||
asked: list[int] = []
|
||||
|
||||
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
|
||||
page = int(url.rsplit("/", 1)[1])
|
||||
asked.append(page)
|
||||
start = (page - 1) * search.ADZUNA_PAGE
|
||||
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
|
||||
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
|
||||
body: dict[str, Any] = {"results": results}
|
||||
if count is not None:
|
||||
body["count"] = count
|
||||
return json.dumps(body).encode()
|
||||
|
||||
monkeypatch.setattr(search, "_get", fake_get)
|
||||
monkeypatch.setenv("ADZUNA_APP_ID", "id")
|
||||
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
|
||||
return asked
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
|
||||
def test_an_adzuna_search_is_complete_when_it_read_its_count(
|
||||
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
|
||||
_adzuna(monkeypatch, count)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (len(rows), done) == (60, complete)
|
||||
|
||||
|
||||
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
|
||||
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
|
||||
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
|
||||
|
||||
|
||||
def test_provider_of() -> None:
|
||||
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
|
||||
assert search.provider_of("adzuna:berlin") == "adzuna"
|
||||
|
||||
+30
-2
@@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -395,7 +396,8 @@ def test_companies_lists_board_and_open_role_count(
|
||||
[row] = [c for c in res.json() if c["key"] == "acme"]
|
||||
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
|
||||
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
|
||||
"ats": "greenhouse", "careers_url": "", "country": "DK"}
|
||||
"ats": "greenhouse", "careers_url": "", "country": "DK",
|
||||
"failing_since": "", "failing_reason": ""}
|
||||
[other] = [c for c in res.json() if c["key"] == "other"]
|
||||
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
|
||||
"", None, None, "")
|
||||
@@ -404,6 +406,32 @@ def test_companies_lists_board_and_open_role_count(
|
||||
"", None, "successfactors", "https://novo.example/careers")
|
||||
|
||||
|
||||
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""What the pull recorded, on `/api/companies` and beside `app companies
|
||||
check`'s live result."""
|
||||
import plugins.ats as ats
|
||||
from app.cli import check_companies
|
||||
|
||||
store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
|
||||
store.add_company("Acme", key="acme", ats="greenhouse")
|
||||
store.state["ats:failing:greenhouse:acme"] = json.dumps(
|
||||
{"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"})
|
||||
|
||||
[row] = client.get("/api/companies").json()
|
||||
assert (row["failing_since"], row["failing_reason"]) == (
|
||||
"2026-09-01T06:00:00+00:00", "404 Not Found")
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
|
||||
assert check_companies(store) == 0
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
"greenhouse:acme (greenhouse:acme): now 1 roles; "
|
||||
"pull: failing since 2026-09-01: 404 Not Found",
|
||||
"checked 1 boards, 0 dead, 1 failing in the pull",
|
||||
]
|
||||
|
||||
|
||||
def test_set_affinity_validates_updates_and_enqueues_a_score(
|
||||
client: TestClient, store: Store, queue: Any) -> None:
|
||||
store.add_company("Acme", key="acme")
|
||||
@@ -909,7 +937,7 @@ def test_a_board_is_followed_only_if_it_answers(
|
||||
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import plugins.ats as ats
|
||||
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}])
|
||||
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
|
||||
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
|
||||
assert res.status_code == 201
|
||||
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
|
||||
|
||||
Reference in new issue
Block a user