Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
This commit is contained in:
1 parent
0558ef19ed
commit
c7da12b93d
18 files changed
+695
-91
No files matched your search
+52
-19
@@ -7,15 +7,19 @@ sized for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which each plugin itself stamps per target
|
||||
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
||||
under `search:`), rather than from what each `pull_batch` job happens to
|
||||
return. A batch job that raises partway through still leaves the boards it had
|
||||
already finished stamped, so one bad board in a batch of ten does not cost the
|
||||
other nine their closure.
|
||||
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
||||
`ats:ids:<target>`, and the same under `search:`), rather than from what each
|
||||
`pull_batch` job happens to return. A batch job that raises partway through
|
||||
still leaves the boards it had already finished stamped, so one bad board in a
|
||||
batch of ten does not cost the other nine their closure. A board that answered
|
||||
but was not read to its stated total (a paged host that hit a 429 or the page
|
||||
cap) stores what it read and closes nothing.
|
||||
|
||||
A search closes the same way when its results ran out before its page limit,
|
||||
since it then listed everything that matches. One that hit the limit did not,
|
||||
so its roles close only once no search has seen them for `STALE_DAYS`.
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,15 +47,22 @@ SETTLE = "app.jobs.pull:close_boards"
|
||||
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
||||
SOURCES = ("ats", "search")
|
||||
|
||||
#: How long a search role stays open once no search lists it any more.
|
||||
#: How long a search role stays open once no search lists it any more, and
|
||||
#: how long a board may fail on every fetch before its roles close.
|
||||
STALE_DAYS = 21
|
||||
|
||||
#: The `state` prefix the `ats` plugin records a failing board under:
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
"""A run marker: to the microsecond, so two pulls a second apart are two
|
||||
runs and a stamp from one never matches the other."""
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
@@ -124,14 +135,18 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed. A search closes by its
|
||||
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
"""
|
||||
answered = searched = failed = closed = 0
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
@@ -141,21 +156,39 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
if not store.state.get(f"search:complete:{sub.target}", ""):
|
||||
continue
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
||||
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
||||
closed += store.close_unseen("search", stale)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
||||
"closed": closed}
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
def search_target(provider: str, name: str) -> str:
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
||||
if store.company(key) is not None:
|
||||
raise ValueError(f"{name} is already followed")
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
rows, _complete = fetch_board(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
if not rows:
|
||||
|
||||
Reference in new issue
Block a user