Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+18 -9
View File
@@ -78,7 +78,7 @@ def build_parser() -> argparse.ArgumentParser:
discover.add_argument("domain", help="its website, e.g. pleo.io")
discover.add_argument("--add", action="store_true",
help="add the company, with its board when one is readable")
companies_sub.add_parser("check", help="report boards that do not answer")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -322,19 +322,28 @@ def add_company(store: Store, board: str, name: str) -> int:
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not."""
from plugins.ats import fetch_board
line per board that did not answer now, answered incompletely, or has
been failing in the pull, with what the pull recorded beside the live
result, and exits 1 if any did not answer now."""
from app import jobs
from plugins.ats import failure_reason, fetch_board
subs = store.subscriptions(source="ats")
dead = []
failing = jobs.pull.failing_boards(store)
dead = 0
for sub in subs:
try:
fetch_board(sub.target)
rows, complete = fetch_board(sub.target)
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
except Exception as exc:
dead.append((sub.target, sub.label, str(exc)))
for target, label, reason in dead:
print(f"{target} ({label}): {reason}")
print(f"checked {len(subs)} boards, {len(dead)} dead")
complete, live = False, f"not answering: {failure_reason(exc)}"
dead += 1
since, reason = failing.get(sub.target, ("", ""))
if complete and not since:
continue
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
return 1 if dead else 0
+52 -19
View File
@@ -7,15 +7,19 @@ sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
under `search:`), rather than from what each `pull_batch` job happens to
return. A batch job that raises partway through still leaves the boards it had
already finished stamped, so one bad board in a batch of ten does not cost the
other nine their closure.
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
`ats:ids:<target>`, and the same under `search:`), rather than from what each
`pull_batch` job happens to return. A batch job that raises partway through
still leaves the boards it had already finished stamped, so one bad board in a
batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing.
A search closes the same way when its results ran out before its page limit,
since it then listed everything that matches. One that hit the limit did not,
so its roles close only once no search has seen them for `STALE_DAYS`.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
"""
from __future__ import annotations
@@ -43,15 +47,22 @@ SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more.
#: How long a search role stays open once no search lists it any more, and
#: how long a board may fail on every fetch before its roles close.
STALE_DAYS = 21
#: The `state` prefix the `ats` plugin records a failing board under:
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
"""A run marker: to the microsecond, so two pulls a second apart are two
runs and a stamp from one never matches the other."""
return datetime.now(UTC).isoformat(timespec="microseconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
@@ -124,14 +135,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
"""
answered = searched = failed = closed = 0
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
@@ -141,21 +156,39 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue
if source == "search":
searched += 1
if not store.state.get(f"search:complete:{sub.target}", ""):
continue
else:
answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
closed += store.close_unseen("search", stale)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
"closed": closed}
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
def search_target(provider: str, name: str) -> str:
+1 -1
View File
@@ -48,7 +48,7 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
try:
rows = fetch_board(board)
rows, _complete = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
if not rows:
+1
View File
@@ -1200,6 +1200,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
gap: .4rem; font-size: .8rem; color: var(--dim);
}
.srow[data-muted="true"], .crow[data-hidden="true"] { opacity: .6; }
.crow .sub .failing { color: var(--warn); }
.linky {
padding: 0; border: 0; background: none; font-size: .8rem;
+10
View File
@@ -73,6 +73,16 @@
sub.appendChild(link);
}
sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its
// roles close.
const failing = document.createElement("span");
failing.className = "failing";
failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}`
+ (company.failing_reason ? `: ${company.failing_reason}` : "");
sub.append(" · ");
sub.appendChild(failing);
}
return sub;
}
+7
View File
@@ -1230,6 +1230,13 @@ class Store:
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
def state_items(self, prefix: str) -> dict[str, str]:
"""Every `state` key that starts with `prefix`, with its value, in one
query, for a reader that wants one kind of key across every target."""
return {str(r["key"]): str(r["value"]) for r in self._rows(
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
(len(prefix), prefix))}
def _check_status(status: str) -> None:
if status not in STATUSES:
+15 -8
View File
@@ -264,7 +264,9 @@ class CompanyOut(BaseModel):
board's subscription, whether it is on, and how many of its roles are
still open. `ats` is the host its careers site runs on whether or not the
app reads it, so a followed company with no `board` and an `ats` is one
the app cannot read yet; `careers_url` is where a person can."""
the app cannot read yet; `careers_url` is where a person can.
`failing_since` and `failing_reason` say since when the board has failed
on every fetch and why it last did, both empty while it answers."""
key: str
name: str
@@ -276,17 +278,19 @@ class CompanyOut(BaseModel):
ats: str = ""
careers_url: str = ""
country: str = ""
failing_since: str = ""
failing_reason: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]:
"""`company_key -> (subscription id, ats host, on)`, read off each
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
"""`company_key -> (subscription id, ats host, on, target)`, read off each
company's own subscription. Same join as `Store.company_boards`, with the
id and switch state a company row needs to draw its own on/off control
rather than fetching `/api/sources` as well."""
boards: dict[str, tuple[int, str, bool]] = {}
boards: dict[str, tuple[int, str, bool, str]] = {}
for sub in store.subscriptions(source="ats"):
boards.setdefault(company_key(sub.company), (sub.id, sub.board,
bool(sub.subscribed and sub.fetch)))
boards.setdefault(company_key(sub.company), (
sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target))
return boards
@@ -296,13 +300,16 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
absent) has it; 0 throughout when there is no agent."""
boards = _company_boards(store)
counts = store.open_role_counts()
failing = jobs.pull.failing_boards(store)
out = []
for c in store.companies(_agent_id(store, agent)):
sub_id, host, on = boards.get(c.key, (None, "", None))
sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
since, reason = failing.get(target, ("", ""))
out.append(CompanyOut(key=c.key, name=c.name, board=host,
subscription_id=sub_id, on=on, affinity=c.affinity,
open_roles=counts.get(c.key, 0), ats=c.ats,
careers_url=c.careers_url, country=c.country))
careers_url=c.careers_url, country=c.country,
failing_since=since, failing_reason=reason))
return out