Merge branch 'worktree-agent-ac3c9345edd29a1c6'

# Conflicts:
#	app/cli.py
#	app/jobs/sources.py
#	app/static/app.css
#	app/static/companies.js
#	app/web/api.py
#	tests/test_web.py
This commit is contained in:
Wolf committed 2026-09-29 14:15:10 +02:00
commit c6246af994
21 files changed
+1016 -154

No files matched your search

+29 -11
View File
@@ -87,7 +87,7 @@ def build_parser() -> argparse.ArgumentParser:
discover.add_argument("--add", action="store_true", discover.add_argument("--add", action="store_true",
help="add or update the company: its board when one is readable, " help="add or update the company: its board when one is readable, "
"else the host it runs on") "else the host it runs on")
companies_sub.add_parser("check", help="report boards that do not answer") companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True) searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -185,8 +185,17 @@ def run_job(args: argparse.Namespace) -> int:
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""), bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False) tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS: if chained or args.kind not in JOBS:
task_id = queue.start_task(cascade.chain(args.kind) if chained first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
else TASKS[args.kind]()) if args.kind == "pull":
# The same claim the `pull` job makes, so a pull here and one in
# the server's worker cannot both fetch.
run = jobs.pull.claim_run(store)
if not run:
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
"running": jobs.pull.running(store)}))
return 1
first = cascade.pull_chain(run)
task_id = queue.start_task(first)
queue.drain() queue.drain()
started = queue.jobs.of_task(task_id) started = queue.jobs.of_task(task_id)
if not started: if not started:
@@ -333,19 +342,28 @@ def detach_board(store: Store, key: str) -> int:
def check_companies(store: Store) -> int: def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a """`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not.""" line per board that did not answer now, answered incompletely, or has
from plugins.ats import fetch_board been failing in the pull, with what the pull recorded beside the live
result, and exits 1 if any did not answer now."""
from app import jobs
from plugins.ats import failure_reason, fetch_board
subs = [s for s in store.subscriptions(source="ats") if s.show] subs = [s for s in store.subscriptions(source="ats") if s.show]
dead = [] failing = jobs.pull.failing_boards(store)
dead = 0
for sub in subs: for sub in subs:
try: try:
fetch_board(sub.target) rows, complete = fetch_board(sub.target)
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
except Exception as exc: except Exception as exc:
dead.append((sub.target, sub.label, str(exc))) complete, live = False, f"not answering: {failure_reason(exc)}"
for target, label, reason in dead: dead += 1
print(f"{target} ({label}): {reason}") since, reason = failing.get(sub.target, ("", ""))
print(f"checked {len(subs)} boards, {len(dead)} dead") if complete and not since:
continue
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
return 1 if dead else 0 return 1 if dead else 0
+33 -6
View File
@@ -34,7 +34,7 @@ from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent from desk.tools import agent
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step" STEP = "app.jobs.cascade:step"
@@ -129,17 +129,44 @@ def chain(name: str) -> Task:
return task(CASCADES[name][0], cascade=name, at=0) return task(CASCADES[name][0], cascade=name, at=0)
def pull_chain(run: str) -> Task:
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
reads under that marker rather than claiming one of its own."""
return replace(chain("pull"), finder=pull.pull_task(run).finder)
def already_running(since: str) -> Notice:
"""The notice for a pull refused because another is in flight."""
return Notice(
code="pull.running", level="info", kind="expected", key="pull",
text=f"Not started: {pull.ALREADY_RUNNING}"
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
where="app.jobs.pull")
@job("pull", "Pulling every board", resource="http") @job("pull", "Pulling every board", resource="http")
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]: def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the pull cascade once this job is done. Its own work is nil. """Claim a pull run and ask for the pull cascade once this job is done.
Its own work is nil.
A pull whose claim is refused, because another run is still unsettled,
queues nothing, embed to summarise included, and says so in its result
and a notice: a second cascade would fetch every board again and could
send the same roles to the model twice.
Under `NULL` there is no queue to take the chain, so this raises rather Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` starts the chain than report a pull that never happened; `app job pull` claims the run and
on a queue of its own instead of calling this. starts the chain on a queue of its own instead of calling this.
""" """
if not context.task(chain("pull")): run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)):
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on") raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"])} return {"started": " > ".join(CASCADES["pull"]), "run": run}
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage #: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
+149 -39
View File
@@ -7,15 +7,28 @@ sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`. complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
under `search:`), rather than from what each `pull_batch` job happens to `ats:ids:<target>`, and the same under `search:`), rather than from what each
return. A batch job that raises partway through still leaves the boards it had `pull_batch` job happens to return. A batch job that raises partway through
already finished stamped, so one bad board in a batch of ten does not cost the still leaves the boards it had already finished stamped, so one bad board in a
other nine their closure. batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing.
A search closes the same way when its results ran out before its page limit, A search closes the same way when it listed everything that matches (see
since it then listed everything that matches. One that hit the limit did not, `plugins/search.py` for how each provider knows). One that did not closes its
so its roles close only once no search has seen them for `STALE_DAYS`. roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
schedule, the Run button and `app job pull` in another process cannot start a
second fetch while one is in flight; a claim older than `ABANDONED` with no
settle is a run that died and is taken over. The marker travels in every
batch's payload, the plugins stamp with the marker they were handed, and the
settle compares against the marker its own batches carried, so neither a
later run's marker nor its id lists reach this run's closure.
""" """
from __future__ import annotations from __future__ import annotations
@@ -23,6 +36,7 @@ from __future__ import annotations
import json import json
import logging import logging
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job from pipeline import NULL, RunContext, Tally, Task, job
@@ -43,37 +57,85 @@ SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry. #: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search") SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more. #: How long a search role stays open once no search lists it any more, and
#: how long a board may fail on every fetch before its roles close.
STALE_DAYS = 21 STALE_DAYS = 21
#: The `state` prefix the `ats` plugin records a failing board under:
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported. #: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded" SEARCHES_SEEDED = "searches:seeded"
#: The marker of the pull run claimed last, and of the one settled last. A run
#: is in flight while the two differ.
RUN = "pull:run"
SETTLED = "pull:settled"
#: How old an unsettled claim may be before it counts as a run that died (a
#: killed process, a killed cascade) and the next pull takes over. A pull of
#: every board takes minutes.
ABANDONED = timedelta(hours=1)
#: What a refused pull says, in its result and its notice.
ALREADY_RUNNING = "a pull is already running"
def _now() -> str: def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds") """A run marker: to the microsecond, so two pulls a second apart are two
runs and a stamp from one never matches the other."""
return datetime.now(UTC).isoformat(timespec="microseconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]: def claim_run(store: Store) -> str:
"""Start a pull run: a new marker in `pull:run`, returned, unless another
run is still unsettled and younger than `ABANDONED`, which returns "".
One conditional write, so two starters in two processes cannot both win."""
marker = _now()
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
def running(store: Store) -> str:
"""The marker of the pull run in flight, or "" when the last one settled."""
run = store.state.get(RUN, "")
return run if run and run != store.state.get(SETTLED, "") else ""
def release_run(store: Store, run: str) -> None:
"""Mark `run` settled, unless a later run has taken over since: letting go
of a claim that is no longer ours would release the one that is."""
if store.state.get(RUN, "") == run:
store.state[SETTLED] = run
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one """Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any source, each batch carrying the run marker its plugin stamps with and
batch runs — finders run synchronously, in whoever starts the task, ahead `close_boards` compares against. `run` is a marker the caller claimed
of every job it queues.""" already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store) seed_searches(config, store)
batches: list[dict[str, Any]] = [] targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES: for source in SOURCES]
subs = store.subscriptions(source=source, fetchable=True) if not any(found for _, found in targets):
if not subs: if run:
continue release_run(store, run)
store.state[f"{source}:run"] = _now() return []
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source} run = run or claim_run(store)
for i in range(0, len(subs), BATCH_SIZE)] if not run:
return batches log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http") @job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]: run: str = "", context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item """Fetch this batch's boards or searches and `upsert_role` every item
they list. they list.
@@ -85,6 +147,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
reads `options[subscription_field]` — so one job never re-fetches another reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth. between two boards rather than after the tenth.
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
""" """
from plugins.ats import ATSSource from plugins.ats import ATSSource
from plugins.search import SearchSource from plugins.search import SearchSource
@@ -99,7 +165,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None: if sub is None:
continue continue
context.check() context.check()
options = dict(config.sources.get(source) or {}) options = dict(config.sources.get(source) or {}) | {"run": run}
plugin: ATSSource | SearchSource plugin: ATSSource | SearchSource
if source == "search": if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}] options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
@@ -112,7 +178,18 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
item.source = source item.source = source
store.upsert_role(item, sub) store.upsert_role(item, sub)
stored += 1 stored += 1
return {"roles": stored} return {"roles": stored, "run": run}
def _run_of(tally: Tally, context: RunContext) -> str:
"""The marker this settle's own batches were queued with: off their
payloads, which a batch that raised still has, else off their results
(the only thing a settle called directly is given)."""
for batch in context.waited_for:
run = (getattr(batch, "payload", None) or {}).get("run")
if run:
return str(run)
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
def close_boards(config: Config, store: Store, tally: Tally, *, def close_boards(config: Config, store: Store, tally: Tally, *,
@@ -124,38 +201,70 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`pull_batch` job that raised partway through still leaves the boards it `pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its `ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its whole batch job never got to it — is never closed by its list, and a board
list only when that list was complete, and otherwise by `STALE_DAYS`. that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled,
which lets the next pull start.
A board or search switched off while its batch was in flight had its A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here. switched-off one is closed again here.
""" """
answered = searched = failed = closed = 0 answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
for source in SOURCES: for source in SOURCES:
for sub in store.subscriptions(source=source): for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch): if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id) closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "") if not run_marker:
continue
for sub in store.subscriptions(source=source, fetchable=True): for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "") stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker): if not (stamp and stamp == run_marker):
failed += 1 failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue continue
if source == "search": if source == "search":
searched += 1 searched += 1
if not store.state.get(f"search:complete:{sub.target}", ""):
continue
else: else:
answered += 1 answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]")) seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id) closed += store.mark_closed(source, seen_ids, sub.id)
if searched: if searched:
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS) closed += store.close_unseen("search", stale)
closed += store.close_unseen("search", stale.isoformat(timespec="seconds")) if run_marker:
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results) roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles, return {"boards": answered, "searches": searched, "failed": failed,
"closed": closed} "incomplete": incomplete, "roles": roles, "closed": closed}
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
def search_target(provider: str, name: str) -> str: def search_target(provider: str, name: str) -> str:
@@ -184,8 +293,9 @@ def seed_searches(config: Config, store: Store) -> list[str]:
return added return added
def pull_task() -> Task: def pull_task(run: str = "") -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage """The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts.""" of the chain the `pull` job starts. `run` is a marker `claim_run` already
return Task(kind="pull_batch", finder=_boards, name="pull", gave the caller; without one the finder claims its own."""
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
label="Reading every board", settle=SETTLE) label="Reading every board", settle=SETTLE)
+1 -1
View File
@@ -68,7 +68,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
rows: list[Any] | None = None rows: list[Any] | None = None
if check: if check:
try: try:
rows = fetch_board(board) rows, _complete = fetch_board(board)
except Exception as exc: except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":") host, _, slug = board.partition(":")
+1
View File
@@ -1211,6 +1211,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
border-radius: .3rem; padding: .35rem .5rem; border-radius: .3rem; padding: .35rem .5rem;
} }
.crow .board-edit input { flex: 1 1 8rem; min-width: 0; } .crow .board-edit input { flex: 1 1 8rem; min-width: 0; }
.crow .sub .failing { color: var(--warn); }
.linky { .linky {
padding: 0; border: 0; background: none; font-size: .8rem; padding: 0; border: 0; background: none; font-size: .8rem;
+11
View File
@@ -86,6 +86,17 @@
sub.appendChild(link); sub.appendChild(link);
} }
sub.append(` · ${count(company.open_roles, "open role")}`); sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its
// roles close.
const failing = document.createElement("span");
failing.className = "failing";
failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}`
+ (company.failing_reason ? `: ${company.failing_reason}` : "");
sub.append(" · ");
sub.appendChild(failing);
}
sub.append(" ");
const edit = document.createElement("button"); const edit = document.createElement("button");
edit.type = "button"; edit.type = "button";
edit.className = "linky"; edit.className = "linky";
+20
View File
@@ -1319,6 +1319,26 @@ class Store:
`self.state`: it can keep a counter without being able to read roles.""" `self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self) return _StateMap(self)
def state_items(self, prefix: str) -> dict[str, str]:
"""Every `state` key that starts with `prefix`, with its value, in one
query, for a reader that wants one kind of key across every target."""
return {str(r["key"]): str(r["value"]) for r in self._rows(
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
(len(prefix), prefix))}
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
"""Set `key` to `value` only while it is free: unset, equal to the
value under `released` (the claim last let go of), or no later than
`stale`. Returns whether it was set. One statement, so of two
processes claiming at once exactly one wins."""
return self._write(
"""INSERT INTO state (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
WHERE state.value <= ?
OR state.value = COALESCE(
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
(key, value, stale, released)) > 0
def _check_status(status: str) -> None: def _check_status(status: str) -> None:
if status not in STATUSES: if status not in STATUSES:
+15 -8
View File
@@ -265,7 +265,9 @@ class CompanyOut(BaseModel):
still open. `ats` is the host its careers site runs on whether or not the still open. `ats` is the host its careers site runs on whether or not the
app reads it, and `ats_slug` its name there, so a followed company with app reads it, and `ats_slug` its name there, so a followed company with
no `board` and an `ats` is one the app cannot read yet; `careers_url` is no `board` and an `ats` is one the app cannot read yet; `careers_url` is
where a person can.""" where a person can. `failing_since` and `failing_reason` say since when
the board has failed on every fetch and why it last did, both empty
while it answers."""
key: str key: str
name: str name: str
@@ -278,20 +280,22 @@ class CompanyOut(BaseModel):
ats_slug: str = "" ats_slug: str = ""
careers_url: str = "" careers_url: str = ""
country: str = "" country: str = ""
failing_since: str = ""
failing_reason: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]: def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
"""`company_key -> (subscription id, ats host, on)`, read off each """`company_key -> (subscription id, ats host, on, target)`, read off each
company's own subscription. Same join as `Store.company_boards`, with the company's own subscription. Same join as `Store.company_boards`, with the
id and switch state a company row needs to draw its own on/off control id and switch state a company row needs to draw its own on/off control
rather than fetching `/api/sources` as well. A detached board (`show` rather than fetching `/api/sources` as well. A detached board (`show`
0) is no board.""" 0) is no board."""
boards: dict[str, tuple[int, str, bool]] = {} boards: dict[str, tuple[int, str, bool, str]] = {}
for sub in store.subscriptions(source="ats"): for sub in store.subscriptions(source="ats"):
if not sub.show: if not sub.show:
continue continue
boards.setdefault(company_key(sub.company), (sub.id, sub.board, boards.setdefault(company_key(sub.company), (
bool(sub.subscribed and sub.fetch))) sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target))
return boards return boards
@@ -301,14 +305,17 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
absent) has it; 0 throughout when there is no agent.""" absent) has it; 0 throughout when there is no agent."""
boards = _company_boards(store) boards = _company_boards(store)
counts = store.open_role_counts() counts = store.open_role_counts()
failing = jobs.pull.failing_boards(store)
out = [] out = []
for c in store.companies(_agent_id(store, agent)): for c in store.companies(_agent_id(store, agent)):
sub_id, host, on = boards.get(c.key, (None, "", None)) sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
since, reason = failing.get(target, ("", ""))
out.append(CompanyOut(key=c.key, name=c.name, board=host, out.append(CompanyOut(key=c.key, name=c.name, board=host,
subscription_id=sub_id, on=on, affinity=c.affinity, subscription_id=sub_id, on=on, affinity=c.affinity,
open_roles=counts.get(c.key, 0), ats=c.ats, open_roles=counts.get(c.key, 0), ats=c.ats,
ats_slug=c.ats_slug, careers_url=c.careers_url, ats_slug=c.ats_slug, careers_url=c.careers_url,
country=c.country)) country=c.country,
failing_since=since, failing_reason=reason))
return out return out
+8 -2
View File
@@ -38,7 +38,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning
## Ingestion ## Ingestion
**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker and the ids it saw, and the pull's settle closes a board only when its stamp matches the marker written before any batch ran. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list. **Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps each board it parses without raising with the run's marker, the ids it saw and whether they are the whole board, and the pull's settle closes a board only when its stamp matches the marker its own batches carried and the read was complete. So a batch that raises part way still closes the boards it finished, and a board no batch reached stays open instead of being closed on an empty id list.
**One pull runs at a time, claimed in the database.** The schedule, the Run button and `app job pull`, which drains a queue of its own in another process, could each start a pull while one was fetching; the second overwrote the stored run marker, so the first run's boards failed their stamp check and stayed open, or a settle closed on the other run's id lists, and with extract on two cascades could send the same roles to the model twice. `claim_run` sets `pull:run` to a new marker in one conditional statement, only while it equals `pull:settled` or is older than an hour, so of two starters in two processes one wins; the settle writes `pull:settled` when it ends. A refused `pull` job queues no stage and says "a pull is already running" in its result and a notice, and the CLI prints the same and exits 1. The marker travels in every batch's payload, the plugins stamp with the marker they were handed, and the settle reads its marker off its own batches' payloads, which a batch that raised still has. The hour is the escape for a run that died: a killed process or a killed cascade never settles, and blocking every pull until someone clears a key would be worse than one duplicated fetch after an hour. The guard lives in `app` because `pipeline`'s queue dedups a job against a live job of its kind, not a cascade against a live cascade, and `solo=True` on the `pull` job would not help, since that job ends in milliseconds once it has asked for the chain.
**Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards. **Boards follow companies; searches find them.** `ats` only reads companies someone listed, so it cannot surface a role at a company nobody knew of, and the seed list came from a US-centred project: of 29,000 open roles, 31 were in Denmark. The `search` plugin runs saved queries on boards that span companies, as `search` subscriptions whose `params` are the query, and pulls them in the same task as the boards.
@@ -50,7 +52,7 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API. **Jobindex and Adzuna, not Jobnet or Jooble.** Jobindex, Denmark's largest board, has an RSS twin for every search page, with phrase and required-word syntax and a region id; no key. Adzuna has a documented API with a free key for the UK and most of the EU, but no Denmark. Jobnet's search endpoint answers 401 outside its own site's session, and Jooble gives a key only on a manual application and answered a Cloudflare challenge. LinkedIn and Indeed have no public search API.
**A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. A search whose results ran out before its page limit did list everything, and closes like a board; the rest close once no search has listed them for 21 days. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists. **A search closes a role only when it listed everything.** A search result is not a complete list, so a role missing from one run has not closed. An Adzuna search listed everything when it read as many distinct results as the `count` it states; a Jobindex search, whose feed states no count, when a page within its page limit came back with fewer than 20 items, counted before the parser drops any, so one malformed item does not pass for the end. Such a search closes like a board; the rest close once no search has listed them for 21 days. A Jobindex answer with no RSS channel raises instead of reading as an empty, complete search. A role two searches found belongs to the last one that listed it, so one search never closes a role the other still lists.
**Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later. **Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later.
@@ -70,6 +72,10 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role. **Workable's list endpoint has no URL and no text.** The parser was first written to Workable's published job object, since every slug tried in the build answered an empty list; the first populated board (Hugging Face, 2026-09-28) showed the v3 shape instead: `location` an object, `department` a list, a `shortcode` and no URL, and no description. The job page is `apply.workable.com/{slug}/j/{shortcode}/`, and `description` stays empty as for SmartRecruiters rather than costing one request per role.
**Workable and SmartRecruiters are read to the end, and a board closes roles only when it read the total it states.** Workable answers 10 roles a page with a `nextPage` token and SmartRecruiters at most 100 a page by `offset`, and both parsers read page one only, so the settle took page one for the whole board and closed what fell off it: Delivery Hero stated 980 roles and 100 were open. `_paged` follows the pages 0.5 s apart, up to 100 pages, and the plugin stamps `ats:complete` only when the distinct ids read equal `total` or `totalFound`. The count is taken on the raw results, so a row the parser drops does not block closure, and on distinct ids, so a role repeated by a shifted offset does not fake completeness. A 429 after page one ends the read: the roles read are stored and nothing closes, because a partial list is no evidence that the rest closed. A 429 on page one is a failure like any other. The other five hosts answer with the whole board and are complete whenever they parse.
**A board that fails on every fetch for 21 days closes its roles.** A board whose fetch raised was only logged, so a company that moved ATS kept its roles open for good behind a 404. The `ats` plugin now records `ats:failing:<target>` in `state` with the time of the first failure in a row and the latest reason, and a fetch that answers removes it. Once the first failure is `STALE_DAYS` old, the same window a search role gets, the settle closes the board's roles; the next answer reopens what it lists through `upsert_role`, as for a board switched back on. The record is plugin state rather than a column because the plugin only sees `self.state`. `/api/companies` returns it as `failing_since` and `failing_reason`, the Companies list shows "not answering since", and `app companies check` prints it beside its live result.
**`pull` is a job that starts the cascade, on a default schedule of every 6h.** A schedule enqueues a job kind and a cascade is started by one, so the `pull` job only asks for the chain; declared beside it in `app/jobs/cascade.py`, the schedule is a starting point that an edit under Runs overrides for good. `pull` is not also a task, which would draw two buttons under Runs. **`pull` is a job that starts the cascade, on a default schedule of every 6h.** A schedule enqueues a job kind and a cascade is started by one, so the `pull` job only asks for the chain; declared beside it in `app/jobs/cascade.py`, the schedule is a starting point that an edit under Runs overrides for good. `pull` is not also a task, which would draw two buttons under Runs.
## Ranking ## Ranking
+154 -26
View File
@@ -18,7 +18,12 @@ The body cap is not a straight port — see `MAX_BYTES` below.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:acme`. Every board is fetched independently; one that fails or times `lever:acme`. Every board is fetched independently; one that fails or times
out is logged and skipped, and the rest continue. out is logged, recorded as failing under `ats:failing:<target>` (see
`ATSSource.fetch`), and skipped, and the rest continue.
Workable and SmartRecruiters page their list and state a total; every other
host answers with the whole board at once. A paged board counts as complete
only when it read as many distinct roles as the total says (see `_paged`).
""" """
from __future__ import annotations from __future__ import annotations
@@ -27,6 +32,7 @@ import html
import json import json
import logging import logging
import re import re
import time
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections.abc import Callable, Iterator from collections.abc import Callable, Iterator
from datetime import UTC, datetime from datetime import UTC, datetime
@@ -129,10 +135,11 @@ class _TooLarge(ValueError):
def _get(client: httpx.Client, url: str, host: str, slug: str, *, def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET") -> bytes: method: str = "GET", payload: dict[str, Any] | None = None) -> bytes:
"""Fetch one board feed: https only, the host's own allow-list, at most """Fetch one board feed: https only, the host's own allow-list, at most
three redirect hops each checked the same way, and the body cut off at three redirect hops each checked the same way, and the body cut off at
`MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`.""" `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none."""
allow = _allowed_host(host, slug) allow = _allowed_host(host, slug)
current = url current = url
for _hop in range(4): for _hop in range(4):
@@ -142,7 +149,7 @@ def _get(client: httpx.Client, url: str, host: str, slug: str, *,
response = client.request( response = client.request(
method, current, timeout=TIMEOUT, follow_redirects=False, method, current, timeout=TIMEOUT, follow_redirects=False,
headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"}, headers={"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"},
json={} if method == "POST" else None, json=(payload or {}) if method == "POST" else None,
) )
if response.is_redirect: if response.is_redirect:
location = response.headers.get("location") or "" location = response.headers.get("location") or ""
@@ -268,15 +275,20 @@ def _parse_ashby(body: bytes) -> list[Row]:
def _parse_workable(body: bytes, slug: str) -> list[Row]: def _parse_workable(body: bytes, slug: str) -> list[Row]:
"""POST https://apply.workable.com/api/v3/accounts/{slug}/jobs with `{}` -> return _workable_rows(json.loads(body), slug)
`{"total": int, "results": [...]}`, as Hugging Face's board answered on
2026-09-28. A result carries no URL and no text: the job page is
def _workable_rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
Face's board answered on 2026-09-28. Ten results a page; the next page is
the same POST with `{"token": nextPage}`, and the last page has no
`nextPage` (see `_paged`). A result carries no URL and no text: the job page is
`https://apply.workable.com/{slug}/j/{shortcode}/`, and the description `https://apply.workable.com/{slug}/j/{shortcode}/`, and the description
is only on the per-job endpoint, one request per role, so `description` is only on the per-job endpoint, one request per role, so `description`
stays empty here as it does for SmartRecruiters. `location` is an object stays empty here as it does for SmartRecruiters. `location` is an object
and `department` a list; `remote` is a flag beside `workplace` and `department` a list; `remote` is a flag beside `workplace`
(remote, hybrid, on_site).""" (remote, hybrid, on_site)."""
data = json.loads(body)
rows = [] rows = []
for j in data.get("results") or []: for j in data.get("results") or []:
title = (j.get("title") or "").strip() title = (j.get("title") or "").strip()
@@ -303,11 +315,16 @@ def _parse_workable(body: bytes, slug: str) -> list[Row]:
def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]: def _parse_smartrecruiters(body: bytes, slug: str) -> list[Row]:
"""https://api.smartrecruiters.com/v1/companies/{slug}/postings -> return _smartrecruiters_rows(json.loads(body), slug)
`{"content": [...]}`. The list omits the job description (only the
def _smartrecruiters_rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
page, whatever `limit` asks for; the next is `?offset=` further on (see
`_paged`). The list omits the job description (only the
per-posting detail endpoint has it, and fetching one per row would be per-posting detail endpoint has it, and fetching one per row would be
hundreds of extra requests per board), so `description` stays empty here.""" hundreds of extra requests per board), so `description` stays empty here."""
data = json.loads(body)
rows = [] rows = []
for j in data.get("content") or []: for j in data.get("content") or []:
title = (j.get("name") or "").strip() title = (j.get("name") or "").strip()
@@ -484,9 +501,91 @@ def careers_url(target: str) -> str:
return _BOARDS[host][1].format(slug=slug) return _BOARDS[host][1].format(slug=slug)
def _fetch_rows(client: httpx.Client, target: str) -> list[Row]: #: Pages read from one paged board at most, and the pause between two of
"""One board, fetched and parsed. Raises on any failure — refused host, #: them. Workable answers ten roles a page and SmartRecruiters a hundred, so
timeout, a feed shape none of the parsers understand. #: the cap is 1,000 and 10,000 roles; the pause keeps Workable, which answers
#: a burst with 429, from seeing one.
MAX_PAGES = 100
PAGE_DELAY = 0.5
#: SmartRecruiters' page size, which is also the most it serves per page.
SMARTRECRUITERS_PAGE = 100
#: A paged host's one page: `(client, slug, cursor)` -> the decoded page, its
#: raw results, the total it states, and the cursor of the next page (None
#: after the last). The first call gets cursor None.
Page = tuple[dict[str, Any], list[dict[str, Any]], Any, Any]
def _workable_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
url = _BOARDS["workable"][0].format(slug=slug)
data = json.loads(_get(client, url, "workable", slug, method="POST",
payload={"token": cursor} if cursor else {}))
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
def _smartrecruiters_page(client: httpx.Client, slug: str, cursor: Any) -> Page:
offset = int(cursor or 0)
url = (_BOARDS["smartrecruiters"][0].format(slug=slug)
+ f"?offset={offset}&limit={SMARTRECRUITERS_PAGE}")
data = json.loads(_get(client, url, "smartrecruiters", slug))
content = data.get("content") or []
return data, content, data.get("totalFound"), offset + len(content) if content else None
#: host -> (one page, that page's rows) for the hosts that page.
_PAGED: dict[str, tuple[Callable[[httpx.Client, str, Any], Page],
Callable[[dict[str, Any], str], list[Row]]]] = {
"workable": (_workable_page, _workable_rows),
"smartrecruiters": (_smartrecruiters_page, _smartrecruiters_rows),
}
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""Every page of a Workable or SmartRecruiters board, `PAGE_DELAY` apart,
until the host has no next page, the stated total is reached, or
`MAX_PAGES`. Returns the rows and whether the board is complete: as many
distinct results read as the host's stated total. Counted on the raw
results, so a row the parser drops does not make a board incomplete, and
on distinct ids, so a page that shifted under an offset and repeated a
role does not make it complete.
A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure."""
host, _, slug = target.partition(":")
page_of, rows_of = _PAGED[host]
rows: dict[str, Row] = {}
seen: set[str] = set()
total: Any = None
cursor: Any = None
for page in range(MAX_PAGES):
if page:
time.sleep(PAGE_DELAY)
try:
data, results, total, cursor = page_of(client, slug, cursor)
except httpx.HTTPStatusError as exc:
if not page or exc.response.status_code != 429:
raise
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
target, page + 1, len(seen), total)
return list(rows.values()), False
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
for row in rows_of(data, slug):
rows.setdefault(row["source_id"], row)
if cursor is None or (isinstance(total, int) and len(seen) >= total):
break
complete = isinstance(total, int) and len(seen) == total
if not complete:
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), total)
return list(rows.values()), complete
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""One board, fetched and parsed: its rows and whether they are the whole
board. Raises on any failure — refused host, timeout, a feed shape none
of the parsers understand. A host that does not page answers with the
whole board, so it is complete whenever it parses; a paged one is
complete when `_paged` says so.
A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES` A Greenhouse board whose `?content=true` body still exceeds `MAX_BYTES`
gets one retry without it: the description is lost, but the titles land gets one retry without it: the description is lost, but the titles land
@@ -494,9 +593,11 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
if not is_known_target(target): if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host") raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
if host in _PAGED:
return _paged(client, target)
url = _BOARDS[host][0].format(slug=slug) url = _BOARDS[host][0].format(slug=slug)
try: try:
body = _get(client, url, host, slug, method="POST" if host == "workable" else "GET") body = _get(client, url, host, slug, method="GET")
except _TooLarge: except _TooLarge:
if host != "greenhouse" or "?" not in url: if host != "greenhouse" or "?" not in url:
raise raise
@@ -504,18 +605,28 @@ def _fetch_rows(client: httpx.Client, target: str) -> list[Row]:
"retrying without it", target, MAX_BYTES) "retrying without it", target, MAX_BYTES)
url = url.split("?", 1)[0] url = url.split("?", 1)[0]
body = _get(client, url, host, slug, method="GET") body = _get(client, url, host, slug, method="GET")
return _PARSERS[host](body, slug) return _PARSERS[host](body, slug), True
def fetch_board(target: str) -> list[Row]: def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just """Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, and what does it list". For `app companies "does this board answer, what does it list, and is that all of it". For
add|check`, which ask that question about one board at a time outside any `app companies add|check`, which ask that question about one board at a
pull. Raises the same way `_fetch_rows` does.""" time outside any pull. Raises the same way `_fetch_rows` does."""
with httpx.Client() as client: with httpx.Client() as client:
return _fetch_rows(client, target) return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
# -------------------------------------------------------------- the plugin # -------------------------------------------------------------- the plugin
class ATSSource(Source): class ATSSource(Source):
@@ -543,19 +654,33 @@ class ATSSource(Source):
def fetch(self) -> Iterator[Item]: def fetch(self) -> Iterator[Item]:
with httpx.Client() as client: with httpx.Client() as client:
# Written by the pull job before any board is fetched (see # Handed in by the batch that runs this plugin, as the marker of
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry # the pull run that queued it (see `app/jobs/pull.py`), and never
# run — has no run to stamp itself against, so it makes one up. # read from state, where a later run may have replaced it. A
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat( # plugin run on its own — a test, a dry run — has no run to stamp
# itself against, so it makes one up.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds") timespec="seconds")
for target, label in self.targets(): for target, label in self.targets():
host, _, slug = target.partition(":") host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try: try:
rows = _fetch_rows(client, target) rows, complete = _fetch_rows(client, target)
except Exception as exc: except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc) log.warning("ats: %s failed (%s)", target, exc)
# The first failure in a row sets `since`, later ones only
# the reason; `app.jobs.pull` closes the board's roles
# once `since` is `STALE_DAYS` old.
try:
since = json.loads(self.state.get(failing, "") or "{}").get("since")
except ValueError:
since = None
self.state[failing] = json.dumps({
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
continue continue
self.state.pop(failing, None)
seen_ids = [] seen_ids = []
for row in rows: for row in rows:
@@ -586,6 +711,9 @@ class ATSSource(Source):
) )
# Stamped only once the board has parsed without raising, so a # Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor # board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. # has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out.
self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
+50 -18
View File
@@ -24,9 +24,12 @@ browser session of its own site) and Jooble (a key needs a manual
application, and its API host answered a Cloudflare challenge). application, and its API host answered a Cloudflare challenge).
A search result list is never a board's complete list, so a role missing from A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that ran out of results before its page one run is not closed here. A search that read everything that matches was
limit was complete, and the pull closes what it stopped listing; the rest complete, and the pull closes what it stopped listing; the rest close once no
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
how many results match, so an Adzuna search is complete when it read that
many; Jobindex states no count, so a Jobindex search is complete when a page
within the page limit came back short.
""" """
from __future__ import annotations from __future__ import annotations
@@ -124,20 +127,28 @@ def _text(fragment: str) -> str:
def parse_jobindex(body: bytes) -> list[Row]: def parse_jobindex(body: bytes) -> list[Row]:
"""One page of a Jobindex RSS search. An item is `<title>` "Role, Company", return _jobindex_page(body)[0]
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
"""One page of a Jobindex RSS search, and how many items it had before
any was dropped. An item is `<title>` "Role, Company",
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
whose `<h4>` is the title alone, whose logo's `alt` is the company on a whose `<h4>` is the title alone, whose logo's `alt` is the company on a
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
("robot job") has no logo, so its company is what follows the title's ("robot job") has no logo, so its company is what follows the title's
last comma, or '' when there is none.""" last comma, or '' when there is none. A body with no `<channel>` is not
a search result and raises, since an empty list would read as a search
that ran out and close everything it listed before."""
root = ET.fromstring(body) root = ET.fromstring(body)
channel = root.find("channel") channel = root.find("channel")
if channel is None: if channel is None:
return [] raise ValueError("jobindex answered without an RSS channel")
area_match = _SEARCH_AREA.search(channel.findtext("title") or "") area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
search_area = area_match.group(1).strip() if area_match else "" search_area = area_match.group(1).strip() if area_match else ""
rows = [] rows = []
for item in channel.findall("item"): items = channel.findall("item")
for item in items:
link = (item.findtext("link") or "").strip() link = (item.findtext("link") or "").strip()
full_title = (item.findtext("title") or "").strip() full_title = (item.findtext("title") or "").strip()
if not full_title or not _web_url(link): if not full_title or not _web_url(link):
@@ -169,21 +180,24 @@ def parse_jobindex(body: bytes) -> list[Row]:
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
"salary_min": None, "salary_max": None, "currency": "", "salary_min": None, "salary_max": None, "currency": "",
}) })
return rows return rows, len(items)
def search_jobindex(client: httpx.Client, params: dict[str, Any], def search_jobindex(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]: keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page or the page """Every page of one search, newest first, until a short page or the page
limit. Returns the rows and whether the results ran out before the limit.""" limit. Returns the rows and whether the results ran out: a page with
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
a dropped item does not pass for the end. Jobindex states no total, so a
search that read the page limit full is never complete."""
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
if params.get("area"): if params.get("area"):
query["geoareaid"] = params["area"] query["geoareaid"] = params["area"]
rows: list[Row] = [] rows: list[Row] = []
for page in range(1, _pages(params) + 1): for page in range(1, _pages(params) + 1):
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page})) got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
rows += got rows += got
if len(got) < JOBINDEX_PAGE: if items < JOBINDEX_PAGE:
return rows, True return rows, True
return rows, False return rows, False
@@ -202,7 +216,10 @@ def parse_adzuna(body: bytes, country: str) -> list[Row]:
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
and snippets carry `<strong>` highlights. A salary Adzuna estimated and snippets carry `<strong>` highlights. A salary Adzuna estimated
(`salary_is_predicted`) is not the posting's, so it is dropped.""" (`salary_is_predicted`) is not the posting's, so it is dropped."""
data = json.loads(body) return _adzuna_rows(json.loads(body), country)
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
rows = [] rows = []
for job in data.get("results") or []: for job in data.get("results") or []:
title = _text(job.get("title") or "") title = _text(job.get("title") or "")
@@ -237,6 +254,9 @@ def _parse_iso(value: Any) -> datetime | None:
def search_adzuna(client: httpx.Client, params: dict[str, Any], def search_adzuna(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]: keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, until a short page, the stated `count` or
the page limit. Returns the rows and whether they are all `count` says
match, or, on an answer with no `count`, whether a page came back short."""
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
if not (app_id and app_key): if not (app_id and app_key):
@@ -252,13 +272,24 @@ def search_adzuna(client: httpx.Client, params: dict[str, Any],
if params.get("where"): if params.get("where"):
query["where"] = params["where"] query["where"] = params["where"]
rows: list[Row] = [] rows: list[Row] = []
seen: set[str] = set()
count: Any = None
ran_out = False
for page in range(1, _pages(params) + 1): for page in range(1, _pages(params) + 1):
url = ADZUNA_SEARCH.format(country=country, page=page) url = ADZUNA_SEARCH.format(country=country, page=page)
got = parse_adzuna(_get(client, url, query), country) data = json.loads(_get(client, url, query))
rows += got results = data.get("results") or []
if len(got) < ADZUNA_PAGE: count = data.get("count", count)
return rows, True seen |= {str(job.get("id") or "") for job in results}
return rows, False rows += _adzuna_rows(data, country)
if len(results) < ADZUNA_PAGE:
ran_out = True
break
if isinstance(count, int) and len(seen) >= count:
break
# Adzuna states how many results match; the search is complete when it
# read that many distinct ones. Without a count, a short page is the end.
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
@@ -309,7 +340,8 @@ class SearchSource(Source):
def fetch(self) -> Iterator[Item]: def fetch(self) -> Iterator[Item]:
with httpx.Client() as client: with httpx.Client() as client:
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( # The pull run's marker, handed in by its batch as for `ats`.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds") timespec="seconds")
for target, label in self.targets(): for target, label in self.targets():
provider = provider_of(target) provider = provider_of(target)
+16
View File
@@ -0,0 +1,16 @@
{
"offset": 2,
"limit": 2,
"totalFound": 3,
"content": [
{
"id": "744000152001234",
"name": "Warehouse Operations Lead",
"company": {"identifier": "Acme", "name": "Acme"},
"releasedDate": "2026-09-25T07:00:00.000Z",
"location": {"city": "Tilburg", "country": "nl", "remote": false, "fullLocation": "Tilburg, Netherlands"},
"department": {"id": "3", "label": "Operations"},
"function": {"id": "operations", "label": "Operations"}
}
]
}
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"offset": 0, "offset": 0,
"limit": 100, "limit": 2,
"totalFound": 2, "totalFound": 3,
"content": [ "content": [
{ {
"id": "744000151326819", "id": "744000151326819",
+25
View File
@@ -0,0 +1,25 @@
{
"total": 3,
"results": [
{
"id": 6125077,
"shortcode": "0A1B2C3D4E",
"title": "Maintenance Site Team Leader",
"remote": false,
"location": {
"country": "Ireland",
"countryCode": "IE",
"city": "Dublin",
"region": "County Dublin",
"display": "Dublin, County Dublin, Ireland"
},
"state": "published",
"published": "2026-09-28T00:00:00.000Z",
"type": "full",
"department": [
"Maintenance"
],
"workplace": "on_site"
}
]
}
+2 -1
View File
@@ -1,5 +1,6 @@
{ {
"total": 2, "total": 3,
"nextPage": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0=",
"results": [ "results": [
{ {
"id": 6125026, "id": 6125026,
+166 -1
View File
@@ -4,9 +4,12 @@ under `tests/fixtures/ats/`."""
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from pathlib import Path from pathlib import Path
from typing import Any
import httpx
import pytest import pytest
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
@@ -160,13 +163,14 @@ def test_fetch_rows_retries_greenhouse_without_content_on_a_too_large_body(
return (FIXTURES / "greenhouse.json").read_bytes() return (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "_get", fake_get)
rows = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type] rows, complete = ats._fetch_rows(object(), "greenhouse:acme") # type: ignore[arg-type]
assert calls == [ assert calls == [
"https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true", "https://boards-api.greenhouse.io/v1/boards/acme/jobs?content=true",
"https://boards-api.greenhouse.io/v1/boards/acme/jobs", "https://boards-api.greenhouse.io/v1/boards/acme/jobs",
] ]
assert [r["source_id"] for r in rows] == ["1001", "1002"] assert [r["source_id"] for r in rows] == ["1001", "1002"]
assert complete # a host that does not page answers with the whole board
def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts( def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
@@ -177,3 +181,164 @@ def test_fetch_rows_does_not_retry_a_too_large_body_on_other_hosts(
monkeypatch.setattr(ats, "_get", fake_get) monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(ats._TooLarge): with pytest.raises(ats._TooLarge):
ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type] ats._fetch_rows(object(), "lever:acme") # type: ignore[arg-type]
# ----------------------------------------------------------------- paging
def _page(name: str) -> bytes:
return (FIXTURES / name).read_bytes()
def _status(code: int) -> httpx.HTTPStatusError:
request = httpx.Request("POST", "https://apply.workable.com/")
return httpx.HTTPStatusError(str(code), request=request,
response=httpx.Response(code, request=request))
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def test_workable_follows_next_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable-2.json" if payload else "workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert calls == [{}, {"token": "WzE3ODk2ODk2MDAwMDAsNjEyNTA0MV0="}]
assert [r["source_id"] for r in rows] == ["6125026", "6125041", "6125077"]
assert complete
def test_smartrecruiters_pages_by_offset_to_total_found(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
urls: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
urls.append(url)
return _page("smartrecruiters-2.json" if "offset=2" in url else "smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
base = "https://api.smartrecruiters.com/v1/companies/Acme/postings"
assert urls == [f"{base}?offset=0&limit=2", f"{base}?offset=2&limit=2"]
assert len(rows) == 3 and rows[2]["location"] == "Tilburg, Netherlands"
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""totalFound says 3; the second page is empty, so two were read."""
empty = json.dumps({"offset": 2, "limit": 2, "totalFound": 3, "content": []}).encode()
def fake_get(client, url, host, slug, *, method="GET", payload=None):
return empty if "offset=2" in url else _page("smartrecruiters.json")
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_a_repeated_row_does_not_count_twice_toward_the_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
"""A page that shifted under an offset repeats a role and misses one: three
rows read, two distinct, against a total of three."""
def fake_get(client, url, host, slug, *, method="GET", payload=None):
page = json.loads(_page("smartrecruiters.json"))
if "offset=2" in url:
page["content"] = page["content"][1:]
return json.dumps(page).encode()
monkeypatch.setattr(ats, "SMARTRECRUITERS_PAGE", 2)
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "smartrecruiters:Acme") # type: ignore[arg-type]
assert (len(rows), complete) == (2, False)
def test_the_page_cap_leaves_a_board_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
calls: list[dict[str, Any] | None] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None):
calls.append(payload)
return _page("workable.json") # always another page
monkeypatch.setattr(ats, "MAX_PAGES", 3)
monkeypatch.setattr(ats, "_get", fake_get)
_rows_read, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert (len(calls), complete) == (3, False)
def test_a_429_mid_way_keeps_what_was_read_and_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
if payload:
raise _status(429)
return _page("workable.json")
monkeypatch.setattr(ats, "_get", fake_get)
rows, complete = ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
assert ([r["source_id"] for r in rows], complete) == (["6125026", "6125041"], False)
def test_a_429_on_the_first_page_is_a_failure(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
def fake_get(client, url, host, slug, *, method="GET", payload=None):
raise _status(429)
monkeypatch.setattr(ats, "_get", fake_get)
with pytest.raises(httpx.HTTPStatusError):
ats._fetch_rows(object(), "workable:exotec") # type: ignore[arg-type]
# -------------------------------------------------------- failing boards
def _source(monkeypatch: pytest.MonkeyPatch, answer: object) -> ats.ATSSource:
"""One board, `greenhouse:gone`, whose fetch answers `answer` or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
source = ats.ATSSource({"boards": [{"url": "greenhouse:gone", "label": "Gone"}],
"run": "run-1"})
return source
def test_a_failing_board_records_since_and_the_latest_reason(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, _status(404))
assert list(source.fetch()) == []
first = json.loads(source.state["ats:failing:greenhouse:gone"])
assert first["reason"] == "404 Not Found" and first["since"]
assert "ats:ok:greenhouse:gone" not in source.state
# A later failure keeps when the failures began and updates the reason.
source.state["ats:failing:greenhouse:gone"] = json.dumps(
first | {"since": "2026-09-01T00:00:00+00:00"})
later = _source(monkeypatch, httpx.ConnectTimeout("timed out"))
later.state = source.state
list(later.fetch())
assert json.loads(source.state["ats:failing:greenhouse:gone"]) == {
"since": "2026-09-01T00:00:00+00:00", "reason": "ConnectTimeout: timed out"}
def test_an_answer_clears_the_failure_and_stamps_completeness(
monkeypatch: pytest.MonkeyPatch) -> None:
source = _source(monkeypatch, (_rows("greenhouse"), False))
source.state["ats:failing:greenhouse:gone"] = json.dumps(
{"since": "2026-09-01T00:00:00+00:00", "reason": "404 Not Found"})
assert len(list(source.fetch())) == 2
assert "ats:failing:greenhouse:gone" not in source.state
assert source.state["ats:ok:greenhouse:gone"] == "run-1"
assert source.state["ats:complete:greenhouse:gone"] == ""
+217 -17
View File
@@ -4,8 +4,11 @@ touches the network."""
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from datetime import UTC, datetime import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import ClassVar from typing import ClassVar
@@ -16,9 +19,10 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402 from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext # noqa: E402 from pipeline import Cancelled, RunContext, Tally # noqa: E402
from plugins import ats # noqa: E402
from plugins.ats import ATSSource # noqa: E402 from plugins.ats import ATSSource # noqa: E402
from app.jobs import pull # noqa: E402 from app.jobs import cascade, pull # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
@@ -53,12 +57,14 @@ def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
assert sum(len(b["targets"]) for b in batches) == 3 assert sum(len(b["targets"]) for b in batches) == 3
assert store.state.get("ats:run", "") != "" run = store.state.get(pull.RUN, "")
assert run and {b["run"] for b in batches} == {run}
assert pull.running(store) == run
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None: def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type] assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
assert store.state.get("ats:run", "") == "" assert store.state.get(pull.RUN, "") == ""
def test_run_batch_upserts_every_item_the_plugin_yields( def test_run_batch_upserts_every_item_the_plugin_yields(
@@ -69,15 +75,15 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
def fake_fetch(self: ATSSource) -> object: def fake_fetch(self: ATSSource) -> object:
yield _item("greenhouse", "acme", "1") yield _item("greenhouse", "acme", "1")
yield _item("greenhouse", "acme", "2") yield _item("greenhouse", "acme", "2")
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]' self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch) monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
store.state["ats:run"] = "run-1"
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"]) result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
assert result == {"roles": 2} assert result == {"roles": 2, "run": "run-1"}
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
assert store.roles()[0].subscription_id is not None assert store.roles()[0].subscription_id is not None
@@ -124,17 +130,17 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
# acme answered this run and now lists only "1"; other's board failed # acme answered this run and now lists only "1"; other's board failed
# (or its batch never got to it) and stamped nothing. # (or its batch never got to it) and stamped nothing.
store.state["ats:run"] = "run-1"
store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:complete:greenhouse:acme"] = "1"
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
class FakeTally: class FakeTally:
results = ({"roles": 1},) results = ({"roles": 1, "run": "run-1"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1, assert result == {"boards": 1, "searches": 0, "failed": 1, "incomplete": 0,
"closed": 1} "roles": 1, "closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -146,15 +152,209 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
# A stale stamp from a run that is not this one — e.g. this run's batch # A stale stamp from a run that is not this one — e.g. this run's batch
# for this board never finished. # for this board never finished.
store.state["ats:run"] = "run-2"
store.state["ats:ok:greenhouse:acme"] = "run-1" store.state["ats:ok:greenhouse:acme"] = "run-1"
store.state["ats:ids:greenhouse:acme"] = "[]" store.state["ats:ids:greenhouse:acme"] = "[]"
class FakeTally: class FakeTally:
results: tuple[dict[str, int], ...] = () results = ({"roles": 0, "run": "run-2"},)
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0, assert result == {"boards": 0, "searches": 0, "failed": 1, "incomplete": 0,
"closed": 0} "roles": 0, "closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"} assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
# ---------------------------------------------- whole pulls, fetch stubbed
def _row(job_id: str) -> dict[str, object]:
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
"description": "", "remote": None, "salary": ""}
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object) -> None:
"""Every board's fetch returns `answer`, `(rows, complete)`, or raises it."""
def fake(client: object, target: str) -> object:
if isinstance(answer, Exception):
raise answer
return answer
monkeypatch.setattr(ats, "_fetch_rows", fake)
def _pull(store: Store) -> dict[str, object]:
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
def _open(store: Store) -> set[str]:
return {r.source_id for r in store.roles()}
@pytest.fixture
def acme(store: Store) -> Store:
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
board="workable")
return store
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
# A 429 on page two: page one's role is stored again, the other stays open.
_answer(monkeypatch, ([_row("1"), _row("3")], False))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
def test_a_failing_board_keeps_its_roles_before_stale_days(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
_answer(monkeypatch, ValueError("gone"))
result = _pull(acme)
assert _open(acme) == {"workable:acme:1"}
assert (result["failed"], result["closed"]) == (1, 0)
since, reason = pull.failing_boards(acme)["workable:acme"]
assert since and reason == "ValueError: gone"
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1"), _row("2")], True))
_pull(acme)
_answer(monkeypatch, ValueError("404 for good"))
_pull(acme)
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
timespec="seconds")
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
{"since": old, "reason": "404 Not Found"})
result = _pull(acme)
assert _open(acme) == set()
assert result["closed"] == 2
# Still failing, and still the date the failures began.
assert pull.failing_boards(acme)["workable:acme"][0] == old
_answer(monkeypatch, ([_row("2")], True))
_pull(acme)
assert _open(acme) == {"workable:acme:2"}
assert pull.failing_boards(acme) == {}
# ------------------------------------------------------ one pull at a time
def _ago(**delta: float) -> str:
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
first = pull.claim_run(acme)
assert first and pull.running(acme) == first
asked: list[object] = []
said: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True,
on_notice=said.append)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
assert asked == []
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
# A chain started without the job's claim (a test, a script) finds
# nothing to queue either, and the running run's marker stands.
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
assert acme.state[pull.RUN] == first
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
_pull(acme)
assert pull.running(acme) == ""
asked: list[object] = []
context = RunContext(on_task=lambda task: asked.append(task) or True)
result = cascade.run_pull(FakeConfig(), acme, context=context)
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
# The chain's finder reads under the job's claim instead of claiming again.
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
assert batch["run"] == result["run"]
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
"""Eight starters, each on its own connection to one file, as two
processes would be, released together: one claim wins."""
Store(tmp_path / "shared.db")
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
barrier = threading.Barrier(len(stores))
def claim(store: Store) -> str:
barrier.wait()
return pull.claim_run(store)
with ThreadPoolExecutor(len(stores)) as pool:
claims = list(pool.map(claim, stores))
assert sum(bool(c) for c in claims) == 1
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
hours = pull.ABANDONED.total_seconds() / 3600
acme.state[pull.RUN] = _ago(hours=hours / 2)
assert pull.claim_run(acme) == ""
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
taken = pull.claim_run(acme)
assert taken and acme.state[pull.RUN] == taken
def test_a_batch_stamps_with_the_marker_it_was_handed(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_answer(monkeypatch, ([_row("1")], True))
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
acme.state[pull.RUN] = "a later run"
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
assert acme.state["ats:ok:workable:acme"] == batch["run"]
def test_run_a_settle_does_not_close_on_run_b_ids(
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""Run A read the board in full; run B, which took over, read it again and
listed less before A's settle ran. A's settle compares with A's marker, so
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
B's own settle is the one that closes."""
_answer(monkeypatch, ([_row("1"), _row("2")], True))
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
_answer(monkeypatch, ([_row("1")], True))
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
assert b["run"] != a["run"]
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
assert (result["failed"], result["closed"]) == (1, 0)
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
assert pull.running(acme) == b["run"]
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
assert result["closed"] == 1
assert _open(acme) == {"workable:acme:1"}
assert pull.running(acme) == ""
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
"""A batch that raised has no result, but its payload still names the run."""
run = pull.claim_run(acme)
acme.state["ats:ok:workable:acme"] = run
acme.state["ats:complete:workable:acme"] = "1"
acme.state["ats:ids:workable:acme"] = "[]"
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
"result": None, "state": "failed"})()
context = RunContext(waited_for=(failed,))
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
assert result["boards"] == 1 and pull.running(acme) == ""
+23 -1
View File
@@ -633,7 +633,7 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
published_at=datetime(2026, 9, 20, tzinfo=UTC), published_at=datetime(2026, 9, 20, tzinfo=UTC),
raw={"company": "Acme", "board": "greenhouse", "slug": "acme"}) raw={"company": "Acme", "board": "greenhouse", "slug": "acme"})
yield item yield item
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") self.state["ats:ok:greenhouse:acme"] = self.options["run"]
self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' self.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
monkeypatch.setattr(ATSSource, "fetch", fake_fetch) monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
@@ -658,6 +658,28 @@ def test_the_pull_job_runs_pull_embed_score_extract_score_then_summarise(
assert work == ["pull", "pull_batch", "embed", "score", "score"] assert work == ["pull", "pull_batch", "embed", "score", "score"]
assert all(j.state == "done" for j in jobs) assert all(j.state == "done" for j in jobs)
assert len(store.scores(agent.id)) == 1 assert len(store.scores(agent.id)) == 1
# The board was stamped with the run the `pull` job claimed, and the
# settle let that run go, so the next pull may start.
run = queue.jobs.get(started.id).result["run"] # type: ignore[union-attr]
assert store.state["ats:ok:greenhouse:acme"] == run == store.state["pull:settled"]
def test_the_pull_job_queues_nothing_while_another_pull_runs(
store: Store, config: Config, tmp_path: Path) -> None:
from app.jobs import pull
store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
running = pull.claim_run(store)
queue = JobQueue(Jobs(tmp_path / "roledesk.db"), config, store,
workers={"cpu": 1, "http": 1, "llm": 1})
started = queue.enqueue("pull")
queue.drain(timeout=60)
done = queue.jobs.get(started.id) # type: ignore[union-attr]
assert done is not None and done.state == "done"
assert done.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
assert [j.kind for j in queue.jobs.of_cascade(done.cascade_id)] == ["pull"]
def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None: def test_pull_is_a_job_on_a_six_hour_schedule_and_the_stages_are_tasks() -> None:
+62 -16
View File
@@ -4,6 +4,7 @@ searches. No network: Jobindex's feed is a trimmed copy of a real one
from __future__ import annotations from __future__ import annotations
import json
import sys import sys
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -20,6 +21,7 @@ from plugins import search # noqa: E402
from plugins.search import SearchSource # noqa: E402 from plugins.search import SearchSource # noqa: E402
from app.config import SearchSeed # noqa: E402 from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402 from app.jobs import pull # noqa: E402
from pipeline import Tally # noqa: E402
from app.models import country_of, remote_of # noqa: E402 from app.models import country_of, remote_of # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
@@ -139,16 +141,68 @@ def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> N
return b"short" if params["page"] == 3 else b"full" return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get) monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "parse_jobindex", monkeypatch.setattr(search, "_jobindex_page",
lambda body: full[:5] if body == b"short" else full) lambda body: (full[:5], 5) if body == b"short" else (full, len(full)))
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type] rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True) assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
# Stopped by the page limit with every page full: not complete.
pages.clear() pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type] rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False) assert (pages, complete) == ([1, 2], False)
def test_a_dropped_jobindex_item_does_not_pass_for_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
"""A full page whose parser dropped one item still has 20 items, so the
search reads on instead of calling itself complete at 19 rows."""
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
monkeypatch.setattr(search, "_get", lambda client, url, params: b"")
monkeypatch.setattr(search, "_jobindex_page", lambda body: (full[1:], len(full)))
_rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert not complete
def test_a_jobindex_answer_without_a_channel_raises() -> None:
with pytest.raises(ValueError, match="channel"):
search.parse_jobindex(b"<rss version='2.0'></rss>")
def _adzuna(monkeypatch: pytest.MonkeyPatch, count: int | None) -> list[int]:
"""Adzuna stubbed to 60 results, `ADZUNA_PAGE` a page, stating `count`."""
asked: list[int] = []
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
page = int(url.rsplit("/", 1)[1])
asked.append(page)
start = (page - 1) * search.ADZUNA_PAGE
results = [{"id": str(n), "title": f"Role {n}", "redirect_url": f"https://x.example/{n}"}
for n in range(start, min(start + search.ADZUNA_PAGE, 60))]
body: dict[str, Any] = {"results": results}
if count is not None:
body["count"] = count
return json.dumps(body).encode()
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setenv("ADZUNA_APP_ID", "id")
monkeypatch.setenv("ADZUNA_APP_KEY", "key")
return asked
@pytest.mark.parametrize(("count", "complete"), [(60, True), (75, False), (None, True)])
def test_an_adzuna_search_is_complete_when_it_read_its_count(
monkeypatch: pytest.MonkeyPatch, count: int | None, complete: bool) -> None:
_adzuna(monkeypatch, count)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (len(rows), done) == (60, complete)
def test_an_adzuna_search_stops_at_its_count(monkeypatch: pytest.MonkeyPatch) -> None:
asked = _adzuna(monkeypatch, search.ADZUNA_PAGE)
rows, done = search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
assert (asked, len(rows), done) == ([1], search.ADZUNA_PAGE, True)
def test_provider_of() -> None: def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex" assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna" assert search.provider_of("adzuna:berlin") == "adzuna"
@@ -176,14 +230,9 @@ def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool
def _pull(store: Store) -> dict[str, Any]: def _pull(store: Store) -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
for batch in batches: for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
def test_a_complete_search_closes_what_it_stopped_listing( def test_a_complete_search_closes_what_it_stopped_listing(
@@ -253,13 +302,10 @@ def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
board="jobindex", params={"q": "backend"}) board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False) _stub_search(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] result = pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False) store.update_subscription(sub.id, subscribed=False, fetch=False)
class Tally: pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
results: ClassVar[list[dict[str, Any]]] = []
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
assert store.roles() == [] assert store.roles() == []
# And a batch that starts after the switch reads nothing. # And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type] assert pull.run_batch(FakeConfig(), store, **batches[0])["roles"] == 0 # type: ignore[arg-type]
+2 -2
View File
@@ -36,11 +36,11 @@ class Boards:
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()} self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
self.fetched: list[str] = [] self.fetched: list[str] = []
def __call__(self, target: str) -> list[dict[str, Any]]: def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
self.fetched.append(target) self.fetched.append(target)
if target not in self.listed: if target not in self.listed:
raise OSError("connection refused") raise OSError("connection refused")
return [{"source_id": str(i)} for i in range(self.listed[target])] return [{"source_id": str(i)} for i in range(self.listed[target])], True
@pytest.fixture @pytest.fixture
+30 -3
View File
@@ -4,6 +4,7 @@ worker beyond starting and stopping it with the app."""
from __future__ import annotations from __future__ import annotations
import json
from collections.abc import Iterator from collections.abc import Iterator
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -396,7 +397,7 @@ def test_companies_lists_board_and_open_role_count(
assert row == {"key": "acme", "name": "Acme", "board": "greenhouse", assert row == {"key": "acme", "name": "Acme", "board": "greenhouse",
"subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1, "subscription_id": sub.id, "on": True, "affinity": 0, "open_roles": 1,
"ats": "greenhouse", "ats_slug": "", "careers_url": "", "ats": "greenhouse", "ats_slug": "", "careers_url": "",
"country": "DK"} "country": "DK", "failing_since": "", "failing_reason": ""}
[other] = [c for c in res.json() if c["key"] == "other"] [other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "") "", None, None, "")
@@ -405,6 +406,32 @@ def test_companies_lists_board_and_open_role_count(
"", None, "successfactors", "https://novo.example/careers") "", None, "successfactors", "https://novo.example/careers")
def test_companies_say_since_when_a_board_fails(client: TestClient, store: Store,
capsys: pytest.CaptureFixture[str],
monkeypatch: pytest.MonkeyPatch) -> None:
"""What the pull recorded, on `/api/companies` and beside `app companies
check`'s live result."""
import plugins.ats as ats
from app.cli import check_companies
store.add_subscription("ats", "greenhouse:acme", company="Acme", board="greenhouse")
store.add_company("Acme", key="acme", ats="greenhouse")
store.state["ats:failing:greenhouse:acme"] = json.dumps(
{"since": "2026-09-01T06:00:00+00:00", "reason": "404 Not Found"})
[row] = client.get("/api/companies").json()
assert (row["failing_since"], row["failing_reason"]) == (
"2026-09-01T06:00:00+00:00", "404 Not Found")
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
assert check_companies(store) == 0
assert capsys.readouterr().out.splitlines() == [
"greenhouse:acme (greenhouse:acme): now 1 roles; "
"pull: failing since 2026-09-01: 404 Not Found",
"checked 1 boards, 0 dead, 1 failing in the pull",
]
def test_set_affinity_validates_updates_and_enqueues_a_score( def test_set_affinity_validates_updates_and_enqueues_a_score(
client: TestClient, store: Store, queue: Any) -> None: client: TestClient, store: Store, queue: Any) -> None:
store.add_company("Acme", key="acme") store.add_company("Acme", key="acme")
@@ -913,7 +940,7 @@ def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}]) monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"}) res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201 assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme" assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
@@ -940,7 +967,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: []) # between openings monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme") store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"}) res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})