593 lines
29 KiB
Python
593 lines
29 KiB
Python
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
|
they list.
|
|
|
|
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
|
`http` resource pool, because a board is one HTTP round trip and that pool is
|
|
sized for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which each plugin itself stamps per target
|
|
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
|
`ats:ids:<target>`, and the same under `search:`), rather than from what each
|
|
`pull_batch` job happens to return. A batch job that raises partway through
|
|
still leaves the boards it had already finished stamped, so one bad board in a
|
|
batch of ten does not cost the other nine their closure. A board that answered
|
|
but was not read to its stated total (a paged host that hit a 429 or the page
|
|
cap) stores what it read and closes nothing; a paged board over the page cap
|
|
is swept across pulls and closes once the sweep reached its end (see
|
|
`plugins/readers/_runner.py`).
|
|
|
|
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
|
`app.jobs.reach`, or its subscription's `params`) lists only the roles in
|
|
reach, so its close keeps open the roles a person acted on: those with a vote
|
|
or an application (`_close_unlisted`). When the reach wanted remote roles and
|
|
the reader could not ask for them (`ats:remote_unasked:<target>`), the read
|
|
says nothing of the remote roles either, so they stay open too and close only
|
|
once the board has not listed them for `STALE_DAYS` (`_close_aged`). Any read
|
|
that lists under a tenth of what the same read listed last time is recorded
|
|
by the runner as a failure, not as a list to close against (`_runner.read`).
|
|
|
|
A board swept across pulls (`ats:swept:<target>`, a complete read that finished
|
|
a sweep an earlier pull began) closes a role only when two finished sweeps in a
|
|
row missed it (`_close_swept`): offset paging drifts when roles close between
|
|
pulls, so one sweep can miss a role that is still open. The ids a finished
|
|
sweep missed are kept in `ats:missing:<target>`, and a complete read that was
|
|
not swept clears them.
|
|
|
|
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
|
|
the roles matching a word, which says nothing of the roles it did not list,
|
|
so it closes by list nothing: a role of such a board closes once the board
|
|
has not listed it for `STALE_DAYS`, the rule searches have.
|
|
|
|
A search never closes by its list, even one stamped complete because it
|
|
read as many results as its provider stated: a search is never a company's
|
|
whole list, so a role that drops out of its results may still be open. Its
|
|
roles close once no search has listed them for `STALE_DAYS`
|
|
(`Store.close_unseen`). A board or a search that has failed on every fetch
|
|
for `STALE_DAYS` (`ats:failing:<target>`, `search:failing:<target>`) closes
|
|
all of its roles, and its next answer reopens what it lists. A target that
|
|
raises in its batch after its read, a row that does not store, is recorded
|
|
as failing the same way and the batch goes on with the next one
|
|
(`run_batch`).
|
|
|
|
The settle says what needs a look in one notice (`settled`): the targets
|
|
that did not answer or were read incomplete, and the boards that closed more
|
|
than `CLOSE_NOTICE` roles or more than half of theirs.
|
|
|
|
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
|
set only while the previous run is settled, in one statement), so the
|
|
schedule, the Run button and `app job pull` in another process cannot start a
|
|
second fetch while one is in flight; a claim older than `ABANDONED` with no
|
|
settle is a run that died and is taken over. The marker travels in every
|
|
batch's payload, the plugins stamp with the marker they were handed, and the
|
|
settle compares against the marker its own batches carried, so neither a
|
|
later run's marker nor its id lists reach this run's closure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
from functools import partial
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, Cancelled, Notice, RunContext, Tally, Task, job
|
|
|
|
from app.config import Config
|
|
from app.jobs.reach import reaches
|
|
from app.models import company_key
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Targets per `pull_batch` job. A batch reads its boards one after another,
|
|
#: so the parallelism is the number of batches the `http` pool runs at once
|
|
#: (8 workers by default, see `config.yaml`), not this number. On the live
|
|
#: database of 2026-09-29, 175 boards and 4 searches make 19 batches, enough
|
|
#: to keep eight workers busy, while a batch that fails or is killed costs at
|
|
#: most ten targets' progress. A board's request is bounded by
|
|
#: `plugins.ats.DEADLINE` plus one read timeout.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "app.jobs.pull:close_boards"
|
|
|
|
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
|
SOURCES = ("ats", "search")
|
|
|
|
#: How long a search role stays open once no search lists it any more, and
|
|
#: how long a board may fail on every fetch before its roles close.
|
|
STALE_DAYS = 21
|
|
|
|
#: The `state` prefix the `ats` plugin records a failing board under:
|
|
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
|
FAILING = "ats:failing:"
|
|
|
|
#: The same for a search, kept by the `search` plugin.
|
|
SEARCH_FAILING = "search:failing:"
|
|
|
|
#: A board that closes more roles than this in one settle is named in the
|
|
#: settle's notice, and so is one that closes more than half of its open
|
|
#: roles, once it had at least `CLOSE_SHARE_MIN` open: a board of two closing
|
|
#: one is an ordinary day.
|
|
CLOSE_NOTICE = 50
|
|
CLOSE_SHARE_MIN = 10
|
|
|
|
#: How many targets a notice names in its text; its detail names them all.
|
|
NOTICE_NAMES = 10
|
|
|
|
#: The `state` key that records the starter searches were imported.
|
|
SEARCHES_SEEDED = "searches:seeded"
|
|
|
|
#: The marker of the pull run claimed last, and of the one settled last. A run
|
|
#: is in flight while the two differ.
|
|
RUN = "pull:run"
|
|
SETTLED = "pull:settled"
|
|
|
|
#: How old an unsettled claim may be before it counts as a run that died (a
|
|
#: killed process, a killed cascade) and the next pull takes over. A pull of
|
|
#: every board takes minutes.
|
|
ABANDONED = timedelta(hours=1)
|
|
|
|
#: What a refused pull says, in its result and its notice.
|
|
ALREADY_RUNNING = "a pull is already running"
|
|
|
|
|
|
def _now() -> str:
|
|
"""A run marker: to the microsecond, so two pulls a second apart are two
|
|
runs and a stamp from one never matches the other."""
|
|
return datetime.now(UTC).isoformat(timespec="microseconds")
|
|
|
|
|
|
def claim_run(store: Store) -> str:
|
|
"""Start a pull run: a new marker in `pull:run`, returned, unless another
|
|
run is still unsettled and younger than `ABANDONED`, which returns "".
|
|
One conditional write, so two starters in two processes cannot both win."""
|
|
marker = _now()
|
|
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
|
|
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
|
|
|
|
|
|
def running(store: Store) -> str:
|
|
"""The marker of the pull run in flight, or "" when the last one settled."""
|
|
run = store.state.get(RUN, "")
|
|
return run if run and run != store.state.get(SETTLED, "") else ""
|
|
|
|
|
|
def release_run(store: Store, run: str) -> None:
|
|
"""Mark `run` settled, unless a later run has taken over since: letting go
|
|
of a claim that is no longer ours would release the one that is."""
|
|
if store.state.get(RUN, "") == run:
|
|
store.state[SETTLED] = run
|
|
|
|
|
|
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
|
source, each batch carrying the run marker its plugin stamps with and
|
|
`close_boards` compares against. `run` is a marker the caller claimed
|
|
already (the `pull` job, `app job pull`); without one the finder claims
|
|
its own, and queues nothing when another run is in flight. A run with
|
|
nothing to read is settled here, since no settle will see a batch of it,
|
|
and so is a run whose finder raises: no batch and no settle would follow,
|
|
and the next pull would wait `ABANDONED` for a claim nobody holds."""
|
|
held = run
|
|
try:
|
|
seed_searches(config, store)
|
|
subs = {source: store.subscriptions(source=source, fetchable=True)
|
|
for source in SOURCES}
|
|
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
|
if not any(found for _, found in targets):
|
|
if run:
|
|
release_run(store, run)
|
|
return []
|
|
run = run or claim_run(store)
|
|
if not run:
|
|
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
|
running(store))
|
|
return []
|
|
held = run
|
|
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
|
reach_of = reaches(store, subs["ats"])
|
|
batches: list[dict[str, Any]] = [
|
|
{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
|
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
|
for batch in batches:
|
|
if batch["source"] == "ats":
|
|
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
|
return batches
|
|
except Exception:
|
|
if held:
|
|
release_run(store, held)
|
|
raise
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
|
run: str = "", reach: dict[str, Any] | None = None,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards or searches and `upsert_role` every item
|
|
they list.
|
|
|
|
`targets` is keyword-only without a default, which is what makes it a
|
|
required payload key to the queue: `/api/jobs/kinds` lists no batch that
|
|
could only fail without one.
|
|
|
|
One plugin per target, scoped to just that entry — `Source.targets()`
|
|
reads `options[subscription_field]` — so one job never re-fetches another
|
|
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
|
between two boards rather than after the tenth.
|
|
|
|
`run` is the marker of the run that queued this batch, handed to the
|
|
plugin as its `run` option, so its stamps belong to that run whatever the
|
|
stored marker says by the time the batch runs. `reach` is `target ->` the
|
|
reach the run computed for a board (`app.jobs.reach`), handed to the
|
|
plugin as its `reach` option. `STALE_DAYS` goes as `stale_days`, after
|
|
which the runner's 10% guard accepts the answer it kept refusing.
|
|
|
|
Each target is guarded on its own. A read that fails is the plugin's to
|
|
record; anything raised after it, a row `upsert_role` refuses or a bug
|
|
past the read, is recorded here under `<source>:failing:<target>` with
|
|
its reason, and the batch goes on with the next target. Otherwise one
|
|
bad row would cost the targets after it in the batch their read on every
|
|
pull, since batches are stable, and leave no trace on any of them. A
|
|
Kill (`Cancelled`) still ends the batch.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
from plugins.readers import _runner
|
|
from plugins.search import SearchSource
|
|
|
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
|
# Job boards or Company boards since then is skipped.
|
|
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
|
if s.target in targets}
|
|
stored = 0
|
|
for target in targets:
|
|
sub = subs.get(target)
|
|
if sub is None:
|
|
continue
|
|
context.check()
|
|
options = dict(config.sources.get(source) or {}) | {"run": run,
|
|
"stale_days": STALE_DAYS}
|
|
items: Any = None
|
|
try:
|
|
plugin: ATSSource | SearchSource
|
|
if source == "search":
|
|
options["searches"] = [{"url": target, "label": sub.label,
|
|
"params": sub.params}]
|
|
plugin = SearchSource(options)
|
|
else:
|
|
# A board's params are its filter; a reader's key is kept with
|
|
# the searches' (`sources.search.<name>`).
|
|
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
|
"params": sub.params}]
|
|
options["keys"] = dict(config.sources.get("search") or {})
|
|
options["reach"] = (reach or {}).get(target)
|
|
plugin = ATSSource(options)
|
|
plugin.state = store.state
|
|
items = plugin.fetch()
|
|
for item in items:
|
|
item.source = source
|
|
store.upsert_role(item, sub)
|
|
stored += 1
|
|
except Cancelled:
|
|
raise
|
|
except Exception as exc:
|
|
log.warning("pull: %s failed after its read (%s); on to the next target",
|
|
target, exc, exc_info=True)
|
|
_runner.record_failure(store.state, f"{source}:failing:{target}", exc)
|
|
finally:
|
|
# The plugin's generator holds its HTTP client until it is closed.
|
|
if items is not None and hasattr(items, "close"):
|
|
items.close()
|
|
return {"roles": stored, "run": run}
|
|
|
|
|
|
def _run_of(tally: Tally, context: RunContext) -> str:
|
|
"""The marker this settle's own batches were queued with: off their
|
|
payloads, which a batch that raised still has, else off their results
|
|
(the only thing a settle called directly is given)."""
|
|
for batch in context.waited_for:
|
|
run = (getattr(batch, "payload", None) or {}).get("run")
|
|
if run:
|
|
return str(run)
|
|
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, tally what happened, and say what needs a
|
|
look.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed by its list, and a board
|
|
that answered incompletely is not either. A search never closes by its
|
|
list, complete or not: its roles close once no search has listed them
|
|
for `STALE_DAYS` (`Store.close_unseen`, run when a search answered). A
|
|
board or a search that has failed on every fetch since `STALE_DAYS` ago
|
|
closes all its roles (`failing_boards`). How a board's read that
|
|
answered closes is `_close_read`'s.
|
|
|
|
The stamps are compared with this run's own marker, the one its batches
|
|
carried, never with the stored `pull:run`, which a later run may have
|
|
replaced. A settle with no batch behind it (a refused pull, nothing to
|
|
read) closes nothing by list or by age and says nothing. Last, the run is
|
|
marked settled, which lets the next pull start.
|
|
|
|
A board or search switched off while its batch was in flight had its
|
|
roles reopened by that batch, after the switch closed them, so every
|
|
switched-off one is closed again here; those closes are asked for and not
|
|
named in the notice.
|
|
|
|
The counts go into the result (`unanswered` counts boards and searches
|
|
whose stamp is not this run's), and one notice (`settled`) names every
|
|
target unanswered, every board read incomplete other than one mid-sweep,
|
|
and every board that closed more than `CLOSE_NOTICE` roles or more than
|
|
half the open roles it had, when there is any. The result alone was
|
|
seen by nobody: a settle closed 12,588 roles with nothing on screen.
|
|
|
|
The `prep` stage that follows maps what this run stored and folds the
|
|
copies (`Store.rebuild_dupes`), so a closed kept role hands its group on
|
|
and the roles this run added join theirs.
|
|
"""
|
|
answered = searched = incomplete = closed = 0
|
|
unanswered: list[str] = []
|
|
partial: list[str] = []
|
|
many: list[tuple[str, int, int]] = []
|
|
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
|
failing = failing_boards(store)
|
|
run_marker = _run_of(tally, context)
|
|
open_before = _open_counts(store) if run_marker else {}
|
|
for source in SOURCES:
|
|
for sub in store.subscriptions(source=source):
|
|
if not (sub.subscribed and sub.fetch):
|
|
closed += store.close_subscription(sub.id)
|
|
if not run_marker:
|
|
continue
|
|
for sub in store.subscriptions(source=source, fetchable=True):
|
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
|
if not (stamp and stamp == run_marker):
|
|
unanswered.append(sub.target)
|
|
since = failing.get(sub.target, ("", ""))[0]
|
|
shut = store.close_subscription(sub.id) if since and since < stale else 0
|
|
elif source == "search":
|
|
# Never a company's whole list, complete or not: a search's
|
|
# roles close by age alone (`close_unseen`, after the loop).
|
|
searched += 1
|
|
continue
|
|
else:
|
|
answered += 1
|
|
read = _close_read(store, source, sub, stale)
|
|
if read is None:
|
|
incomplete += 1
|
|
if not store.state.get(f"ats:sweep:{sub.target}"):
|
|
partial.append(sub.target)
|
|
continue
|
|
shut = read
|
|
closed += shut
|
|
before = open_before.get(sub.id, 0)
|
|
if shut > CLOSE_NOTICE or (before >= CLOSE_SHARE_MIN and 2 * shut > before):
|
|
many.append((sub.target, shut, before))
|
|
if searched:
|
|
closed += store.close_unseen("search", stale)
|
|
if run_marker:
|
|
if unanswered or partial or many:
|
|
context.notice(settled(unanswered, partial, many, failing,
|
|
boards=answered, searches=searched, closed=closed))
|
|
release_run(store, run_marker)
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "searches": searched, "unanswered": len(unanswered),
|
|
"incomplete": incomplete, "roles": roles, "closed": closed}
|
|
|
|
|
|
def _close_read(store: Store, source: str, sub: Any, stale: str) -> int | None:
|
|
"""Close what one board's read of this run says has closed, and return
|
|
how many; None when the read was incomplete, which closes nothing. A
|
|
search's read never comes here (see `close_boards`).
|
|
|
|
A read narrowed by terms closes only the roles it has not listed for
|
|
`STALE_DAYS` (`_close_aged`). A read that finished a sweep closes what
|
|
two sweeps in a row missed (`_close_swept`); any other complete read
|
|
clears that record and closes what it did not list, all of it unfiltered
|
|
(`Store.mark_closed`), filtered sparing the roles acted on
|
|
(`_close_unlisted`). A filtered read that could not ask for remote roles
|
|
(`remote_unasked`) spares the remote ones as well, and closes those by
|
|
age instead."""
|
|
state, target = store.state, sub.target
|
|
if not state.get(f"{source}:complete:{target}", ""):
|
|
return None
|
|
seen_ids = json.loads(state.get(f"{source}:ids:{target}", "[]"))
|
|
if state.get(f"{source}:narrowed:{target}", ""):
|
|
return _close_aged(store, source, sub.id, seen_ids, stale)
|
|
is_filtered = bool(state.get(f"{source}:filtered:{target}", ""))
|
|
spare_remote = is_filtered and bool(state.get(f"{source}:remote_unasked:{target}", ""))
|
|
aged = (_close_aged(store, source, sub.id, seen_ids, stale, remote_only=True)
|
|
if spare_remote else 0)
|
|
if state.get(f"{source}:swept:{target}", ""):
|
|
return aged + _close_swept(store, source, sub, seen_ids, is_filtered,
|
|
spare_remote=spare_remote)
|
|
state.pop(f"{source}:missing:{target}", None)
|
|
if is_filtered:
|
|
return aged + _close_unlisted(store, source, seen_ids, sub.id,
|
|
spare_remote=spare_remote)
|
|
return store.mark_closed(source, seen_ids, sub.id)
|
|
|
|
|
|
def _open_counts(store: Store) -> dict[int, int]:
|
|
"""`subscription id -> open roles`, before the settle closes any, for the
|
|
share a board closed."""
|
|
with store.transaction() as conn:
|
|
return {int(row[0]): int(row[1]) for row in conn.execute(
|
|
"""SELECT subscription_id, count(*) FROM roles
|
|
WHERE closed_at IS NULL AND subscription_id IS NOT NULL
|
|
GROUP BY subscription_id""")}
|
|
|
|
|
|
def settled(unanswered: list[str], incomplete: list[str], many: list[tuple[str, int, int]],
|
|
failing: dict[str, tuple[str, str]], *, boards: int, searches: int,
|
|
closed: int) -> Notice:
|
|
"""The notice of a settle that has something to look at: the targets
|
|
that did not answer (with the reason their failing record gives), the
|
|
boards read incomplete, and the boards that closed many roles, each
|
|
as `closed of open`, with the totals. The text names the first
|
|
`NOTICE_NAMES` of each, the detail all of them, one a line."""
|
|
def named(target: str) -> str:
|
|
reason = failing.get(target, ("", ""))[1]
|
|
return f"{target} ({reason})" if reason else target
|
|
|
|
groups = [("Not answering", [named(t) for t in unanswered]),
|
|
("Read incomplete", list(incomplete)),
|
|
("Closed many", [f"{t} {shut} of {before}" for t, shut, before in many])]
|
|
text = [f"The pull read {boards} boards and {searches} searches and closed "
|
|
f"{closed:,} roles."]
|
|
detail: list[str] = []
|
|
for title, names in groups:
|
|
if not names:
|
|
continue
|
|
shown = ", ".join(names[:NOTICE_NAMES])
|
|
more = len(names) - NOTICE_NAMES
|
|
text.append(f"{title} ({len(names)}): {shown}" + (f" and {more} more." if more > 0
|
|
else "."))
|
|
detail += [f"{title}: {name}" for name in names]
|
|
return Notice(code="pull.settled", level="warning", kind="expected", key="pull",
|
|
text=" ".join(text), detail="\n".join(detail), where="app.jobs.pull")
|
|
|
|
|
|
#: The roles a partial read never closes by its list: one with a vote (any,
|
|
#: neutral included) or an application (which holds the notes).
|
|
_ACTED_ON = """ AND uid NOT IN (SELECT uid FROM role_votes)
|
|
AND uid NOT IN (SELECT role_uid FROM applications
|
|
WHERE role_uid IS NOT NULL)"""
|
|
|
|
#: The remote roles a filtered read that could not ask for them leaves alone.
|
|
_NOT_REMOTE = " AND remote != 'remote'"
|
|
|
|
|
|
def _close_unlisted(store: Store, source: str, seen_ids: list[str], subscription_id: int,
|
|
*, spare_remote: bool = False) -> int:
|
|
"""`Store.mark_closed` for a filtered read: close every open role of the
|
|
board it did not list, except a role someone acted on (`_ACTED_ON`). A
|
|
filtered read cannot tell a role gone from a role outside its filter, and
|
|
a role acted on stays until the board is read without the filter or
|
|
fails for `STALE_DAYS`. With `spare_remote` the remote roles stay as
|
|
well, since the read could not ask for them."""
|
|
with store.transaction() as conn:
|
|
return conn.execute(
|
|
f"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?)){_ACTED_ON}"""
|
|
+ (_NOT_REMOTE if spare_remote else ""),
|
|
(store.now(), source, subscription_id, json.dumps(list(seen_ids)))).rowcount
|
|
|
|
|
|
def _close_aged(store: Store, source: str, subscription_id: int, seen_ids: list[str],
|
|
stale: str, *, remote_only: bool = False) -> int:
|
|
"""Close the open roles of one board that its read did not list and that
|
|
no read has listed since `stale` (`roles.last_seen`, which `upsert_role`
|
|
writes on every listing): the rule for what a read cannot speak for. A
|
|
read narrowed by terms closes all such roles, as a search does; with
|
|
`remote_only`, a filtered read that could not ask for remote closes only
|
|
its remote roles so, sparing those acted on as its list does."""
|
|
where = " AND remote = 'remote'" + _ACTED_ON if remote_only else ""
|
|
with store.transaction() as conn:
|
|
return conn.execute(
|
|
f"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND last_seen < ?
|
|
AND source_id NOT IN (SELECT value FROM json_each(?)){where}""",
|
|
(store.now(), source, subscription_id, stale,
|
|
json.dumps(list(seen_ids)))).rowcount
|
|
|
|
|
|
def _close_swept(store: Store, source: str, sub: Any, seen_ids: list[str],
|
|
filtered: bool, *, spare_remote: bool = False) -> int:
|
|
"""The close of a board whose complete read finished a sweep across
|
|
pulls: its open roles missing from `seen_ids` that also are in
|
|
`<source>:missing:<target>`, the ones the previous finished sweep missed.
|
|
The ones missing now are kept there for the next sweep. A role one sweep
|
|
missed may only have shifted back past the cursor when roles closed
|
|
between pulls, so one sweep is not evidence that it is gone. A filtered
|
|
read keeps its exceptions as `_close_unlisted` does: a role acted on
|
|
stays open, and stays in `missing`, and so does a remote role with
|
|
`spare_remote`."""
|
|
key = f"{source}:missing:{sub.target}"
|
|
try:
|
|
earlier = {str(i) for i in json.loads(store.state.get(key) or "[]")}
|
|
except (TypeError, ValueError):
|
|
earlier = set()
|
|
acted_on = (_ACTED_ON if filtered else "") + (_NOT_REMOTE if spare_remote else "")
|
|
with store.transaction() as conn:
|
|
missed = [row[0] for row in conn.execute(
|
|
"""SELECT source_id FROM roles
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id NOT IN (SELECT value FROM json_each(?))""",
|
|
(source, sub.id, json.dumps(list(seen_ids))))]
|
|
closing = [row[0] for row in conn.execute(
|
|
f"""SELECT source_id FROM roles
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id IN (SELECT value FROM json_each(?))
|
|
AND source_id IN (SELECT value FROM json_each(?)){acted_on}""",
|
|
(source, sub.id, json.dumps(sorted(earlier)), json.dumps(missed)))]
|
|
count = conn.execute(
|
|
"""UPDATE roles SET closed_at = ?
|
|
WHERE source = ? AND subscription_id = ? AND closed_at IS NULL
|
|
AND source_id IN (SELECT value FROM json_each(?))""",
|
|
(store.now(), source, sub.id, json.dumps(closing))).rowcount
|
|
store.state[key] = json.dumps(sorted(set(missed) - set(closing)))
|
|
return count
|
|
|
|
|
|
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
|
"""`target -> (since, reason)` for every `ats` board and every search
|
|
whose last fetch failed: when the failures began, and what the latest
|
|
one said. No board host is also a search provider, so a board's target
|
|
(`host:slug`) and a search's (`provider:name`) never collide in one
|
|
dict."""
|
|
out: dict[str, tuple[str, str]] = {}
|
|
for prefix in (FAILING, SEARCH_FAILING):
|
|
for key, value in store.state_items(prefix).items():
|
|
try:
|
|
record = json.loads(value)
|
|
except ValueError:
|
|
continue
|
|
out[key[len(prefix):]] = (str(record.get("since") or ""),
|
|
str(record.get("reason") or ""))
|
|
return out
|
|
|
|
|
|
def search_target(provider: str, name: str) -> str:
|
|
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
|
The query itself lives in `params`, so editing it keeps the target."""
|
|
return f"{provider}:{company_key(name)}"
|
|
|
|
|
|
def seed_searches(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
|
and only while there are none: the same rule as `seed_agents`."""
|
|
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
|
return []
|
|
added = []
|
|
for seed in config.searches:
|
|
try:
|
|
store.add_subscription("search", search_target(seed.provider, seed.name),
|
|
label=seed.name, board=seed.provider, params=seed.params)
|
|
except ValueError as exc:
|
|
log.warning("searches: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEARCHES_SEEDED] = store.now()
|
|
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
|
return added
|
|
|
|
|
|
def pull_task(run: str = "") -> Task:
|
|
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
|
of the chain the `pull` job starts. `run` is a marker `claim_run` already
|
|
gave the caller; without one the finder claims its own."""
|
|
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
|
|
label="Reading every board", settle=SETTLE)
|