What a pull reads can now be changed on a page instead of only the command line: try a search and see its first ten results before saving it, edit or remove one, follow a company's ATS board, switch any board or search on or off, and preview and apply a prune by region. Each row shows its open roles and the countries they are in. The page and `app companies`/`app searches` call the same functions in app/jobs/sources.py. Switching a target off closes its open roles, and the pull's settle closes them again if a batch in flight reopened them. Roles pulled before `country` existed are backfilled at startup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
193 lines
8.1 KiB
Python
193 lines
8.1 KiB
Python
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
|
they list.
|
|
|
|
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
|
`http` resource pool, because a board is one HTTP round trip and that pool is
|
|
sized for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which each plugin itself stamps per target
|
|
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
|
under `search:`), rather than from what each `pull_batch` job happens to
|
|
return. A batch job that raises partway through still leaves the boards it had
|
|
already finished stamped, so one bad board in a batch of ten does not cost the
|
|
other nine their closure.
|
|
|
|
A search closes the same way when its results ran out before its page limit,
|
|
since it then listed everything that matches. One that hit the limit did not,
|
|
so its roles close only once no search has seen them for `STALE_DAYS`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, RunContext, Tally, Task, job
|
|
|
|
from app.config import Config
|
|
from app.models import company_key
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: As many boards in the air per job as there are `http` workers by default
|
|
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
|
#: holding all 190-odd boards' worth of retries against a single attempt.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "app.jobs.pull:close_boards"
|
|
|
|
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
|
SOURCES = ("ats", "search")
|
|
|
|
#: How long a search role stays open once no search lists it any more.
|
|
STALE_DAYS = 21
|
|
|
|
#: The `state` key that records the starter searches were imported.
|
|
SEARCHES_SEEDED = "searches:seeded"
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
|
source. Stamps the run marker `close_boards` will look for, before any
|
|
batch runs — finders run synchronously, in whoever starts the task, ahead
|
|
of every job it queues."""
|
|
seed_searches(config, store)
|
|
batches: list[dict[str, Any]] = []
|
|
for source in SOURCES:
|
|
subs = store.subscriptions(source=source, fetchable=True)
|
|
if not subs:
|
|
continue
|
|
store.state[f"{source}:run"] = _now()
|
|
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
|
|
for i in range(0, len(subs), BATCH_SIZE)]
|
|
return batches
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards or searches and `upsert_role` every item
|
|
they list.
|
|
|
|
`targets` is keyword-only without a default, which is what makes it a
|
|
required payload key to the queue: the Jobs tab offers no button for a
|
|
batch that could only fail without one.
|
|
|
|
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
|
`options[subscription_field]`, so handing it only this batch's entries is
|
|
what keeps one job from re-fetching every other job's boards.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
from plugins.search import SearchSource
|
|
|
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
|
# Sources tab since then is skipped.
|
|
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
|
if s.target in targets}
|
|
options = dict(config.sources.get(source) or {})
|
|
plugin: ATSSource | SearchSource
|
|
if source == "search":
|
|
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
|
for t in targets if t in subs]
|
|
plugin = SearchSource(options)
|
|
else:
|
|
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
|
for t in targets if t in subs]
|
|
plugin = ATSSource(options)
|
|
plugin.state = store.state
|
|
|
|
stored = 0
|
|
for item in plugin.fetch():
|
|
item.source = source
|
|
if source == "search":
|
|
target = str(item.raw.get("search", ""))
|
|
else:
|
|
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
|
store.upsert_role(item, subs.get(target))
|
|
stored += 1
|
|
return {"roles": stored}
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, and tally what happened.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed. A search closes by its
|
|
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
|
|
|
A board or search switched off while its batch was in flight had its
|
|
roles reopened by that batch, after the switch closed them, so every
|
|
switched-off one is closed again here.
|
|
"""
|
|
answered = searched = failed = closed = 0
|
|
for source in SOURCES:
|
|
for sub in store.subscriptions(source=source):
|
|
if not (sub.subscribed and sub.fetch):
|
|
closed += store.close_subscription(sub.id)
|
|
run_marker = store.state.get(f"{source}:run", "")
|
|
for sub in store.subscriptions(source=source, fetchable=True):
|
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
|
if not (stamp and stamp == run_marker):
|
|
failed += 1
|
|
continue
|
|
if source == "search":
|
|
searched += 1
|
|
if not store.state.get(f"search:complete:{sub.target}", ""):
|
|
continue
|
|
else:
|
|
answered += 1
|
|
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
|
closed += store.mark_closed(source, seen_ids, sub.id)
|
|
if searched:
|
|
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
|
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
|
"closed": closed}
|
|
|
|
|
|
def search_target(provider: str, name: str) -> str:
|
|
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
|
The query itself lives in `params`, so editing it keeps the target."""
|
|
return f"{provider}:{company_key(name)}"
|
|
|
|
|
|
def seed_searches(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
|
and only while there are none: the same rule as `seed_agents`."""
|
|
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
|
return []
|
|
added = []
|
|
for seed in config.searches:
|
|
try:
|
|
store.add_subscription("search", search_target(seed.provider, seed.name),
|
|
label=seed.name, board=seed.provider, params=seed.params)
|
|
except ValueError as exc:
|
|
log.warning("searches: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEARCHES_SEEDED] = store.now()
|
|
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
|
return added
|
|
|
|
|
|
def pull_task() -> Task:
|
|
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
|
of the chain the `pull` job starts."""
|
|
return Task(kind="pull_batch", finder=_boards, name="pull",
|
|
label="Reading every board", settle=SETTLE)
|