Paged boards are read to their stated total, and a board failing for 21 days closes

Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus committed 2026-09-29 14:02:49 +02:00
1 parent 0558ef19ed
commit c7da12b93d
18 files changed
+695 -91

No files matched your search

+18 -9
View File
@@ -78,7 +78,7 @@ def build_parser() -> argparse.ArgumentParser:
discover.add_argument("domain", help="its website, e.g. pleo.io")
discover.add_argument("--add", action="store_true",
help="add the company, with its board when one is readable")
companies_sub.add_parser("check", help="report boards that do not answer")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -322,19 +322,28 @@ def add_company(store: Store, board: str, name: str) -> int:
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not."""
from plugins.ats import fetch_board
line per board that did not answer now, answered incompletely, or has
been failing in the pull, with what the pull recorded beside the live
result, and exits 1 if any did not answer now."""
from app import jobs
from plugins.ats import failure_reason, fetch_board
subs = store.subscriptions(source="ats")
dead = []
failing = jobs.pull.failing_boards(store)
dead = 0
for sub in subs:
try:
fetch_board(sub.target)
rows, complete = fetch_board(sub.target)
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
except Exception as exc:
dead.append((sub.target, sub.label, str(exc)))
for target, label, reason in dead:
print(f"{target} ({label}): {reason}")
print(f"checked {len(subs)} boards, {len(dead)} dead")
complete, live = False, f"not answering: {failure_reason(exc)}"
dead += 1
since, reason = failing.get(sub.target, ("", ""))
if complete and not since:
continue
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
return 1 if dead else 0