Files
jobboard/app/jobs/pull.py
T
WolfandClaude Opus 5.5 047f7cd8ad Find roles by search, not only by company: Jobindex and Adzuna searches
The ats boards only read companies someone listed, and the seed list is
US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs
saved queries on boards that span companies, pulled in the same task as the
boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU,
free key, built to its documented API). Jobnet and Jooble were checked and
left out.

`country` is now filled from the board's location line by a lookup, with EU
for Europe-wide roles, so country gates work without extract. `app companies
prune --in` unsubscribes boards with no open role in a region. The starter
agents and searches are Copenhagen and Europe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 19:55:50 +02:00

183 lines
7.6 KiB
Python

"""`pull`: fetch every active `ats` board and `search`, and store the roles
they list.
A task, not a single job: ten boards or searches per `pull_batch` job on the
`http` resource pool, because a board is one HTTP round trip and that pool is
sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
under `search:`), rather than from what each `pull_batch` job happens to
return. A batch job that raises partway through still leaves the boards it had
already finished stamped, so one bad board in a batch of ten does not cost the
other nine their closure.
A search closes the same way when its results ran out before its page limit,
since it then listed everything that matches. One that hit the limit did not,
so its roles close only once no search has seen them for `STALE_DAYS`.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime, timedelta
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.models import company_key
from app.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `config.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more.
STALE_DAYS = 21
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any
batch runs — finders run synchronously, in whoever starts the task, ahead
of every job it queues."""
seed_searches(config, store)
batches: list[dict[str, Any]] = []
for source in SOURCES:
subs = store.subscriptions(source=source, fetchable=True)
if not subs:
continue
store.state[f"{source}:run"] = _now()
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
for i in range(0, len(subs), BATCH_SIZE)]
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: the Jobs tab offers no button for a
batch that could only fail without one.
Builds the plugin scoped to just these targets — `Source.targets()` reads
`options[subscription_field]`, so handing it only this batch's entries is
what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
for t in targets if t in subs]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
for t in targets if t in subs]
plugin = ATSSource(options)
plugin.state = store.state
stored = 0
for item in plugin.fetch():
item.source = source
if source == "search":
target = str(item.raw.get("search", ""))
else:
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
"""
answered = searched = failed = closed = 0
for source in SOURCES:
run_marker = store.state.get(f"{source}:run", "")
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
continue
if source == "search":
searched += 1
if not store.state.get(f"search:complete:{sub.target}", ""):
continue
else:
answered += 1
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
"closed": closed}
def search_target(provider: str, name: str) -> str:
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
The query itself lives in `params`, so editing it keeps the target."""
return f"{provider}:{company_key(name)}"
def seed_searches(config: Config, store: Store) -> list[str]:
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
and only while there are none: the same rule as `seed_agents`."""
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
return []
added = []
for seed in config.searches:
try:
store.add_subscription("search", search_target(seed.provider, seed.name),
label=seed.name, board=seed.provider, params=seed.params)
except ValueError as exc:
log.warning("searches: %s", exc)
continue
added.append(seed.name)
if added:
store.state[SEARCHES_SEEDED] = store.now()
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
return added
def pull_task() -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)