Find roles by search, not only by company: Jobindex and Adzuna searches
The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfab68c878
commit
047f7cd8ad
17 files changed
+1293
-72
No files matched your search
+114
-46
@@ -1,28 +1,34 @@
|
||||
"""`pull`: fetch every active `ats` board and store the roles it lists.
|
||||
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
||||
they list.
|
||||
|
||||
A task, not a single job: ten boards per `pull_batch` job on the `http`
|
||||
resource pool, because a board is one HTTP round trip and that pool is sized
|
||||
for exactly this. The final job's `settle` then closes every board a
|
||||
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
||||
`http` resource pool, because a board is one HTTP round trip and that pool is
|
||||
sized for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which the `ats` plugin itself stamps per
|
||||
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
|
||||
than from what each `pull_batch` job happens to return. A batch job that
|
||||
raises partway through still leaves the boards it had already finished
|
||||
stamped, so one bad board in a batch of ten does not cost the other nine
|
||||
their closure.
|
||||
Closure is read from `self.state`, which each plugin itself stamps per target
|
||||
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
||||
under `search:`), rather than from what each `pull_batch` job happens to
|
||||
return. A batch job that raises partway through still leaves the boards it had
|
||||
already finished stamped, so one bad board in a batch of ten does not cost the
|
||||
other nine their closure.
|
||||
|
||||
A search closes the same way when its results ran out before its page limit,
|
||||
since it then listed everything that matches. One that hit the limit did not,
|
||||
so its roles close only once no search has seen them for `STALE_DAYS`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.models import company_key
|
||||
from app.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -34,49 +40,74 @@ BATCH_SIZE = 10
|
||||
|
||||
SETTLE = "app.jobs.pull:close_boards"
|
||||
|
||||
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
||||
SOURCES = ("ats", "search")
|
||||
|
||||
#: How long a search role stays open once no search lists it any more.
|
||||
STALE_DAYS = 21
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
|
||||
marker `close_boards` will look for, before any batch runs — finders run
|
||||
synchronously, in whoever starts the task, ahead of every job it queues."""
|
||||
subs = store.subscriptions(source="ats", fetchable=True)
|
||||
if not subs:
|
||||
return []
|
||||
store.state["ats:run"] = _now()
|
||||
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
||||
source. Stamps the run marker `close_boards` will look for, before any
|
||||
batch runs — finders run synchronously, in whoever starts the task, ahead
|
||||
of every job it queues."""
|
||||
seed_searches(config, store)
|
||||
batches: list[dict[str, Any]] = []
|
||||
for source in SOURCES:
|
||||
subs = store.subscriptions(source=source, fetchable=True)
|
||||
if not subs:
|
||||
continue
|
||||
store.state[f"{source}:run"] = _now()
|
||||
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
return batches
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str],
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards and `upsert_role` every item they list.
|
||||
"""Fetch this batch's boards or searches and `upsert_role` every item
|
||||
they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Jobs tab offers no button for a
|
||||
batch that could only fail without one.
|
||||
|
||||
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
||||
reads `options[subscription_field]`, so handing it only this batch's
|
||||
entries is what keeps one job from re-fetching every other job's boards.
|
||||
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
||||
`options[subscription_field]`, so handing it only this batch's entries is
|
||||
what keeps one job from re-fetching every other job's boards.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
|
||||
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
|
||||
options = dict(config.sources.get("ats") or {})
|
||||
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
|
||||
for target in targets if target in subs]
|
||||
source = ATSSource(options)
|
||||
source.state = store.state
|
||||
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
||||
for t in targets if t in subs]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
||||
for t in targets if t in subs]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in source.fetch():
|
||||
item.source = "ats"
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
if source == "search":
|
||||
target = str(item.raw.get("search", ""))
|
||||
else:
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
@@ -91,20 +122,57 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed.
|
||||
whole batch job never got to it — is never closed. A search closes by its
|
||||
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
||||
"""
|
||||
run_marker = store.state.get("ats:run", "")
|
||||
answered = failed = closed = 0
|
||||
for sub in store.subscriptions(source="ats", fetchable=True):
|
||||
stamp = store.state.get(f"ats:ok:{sub.target}", "")
|
||||
if stamp and stamp == run_marker:
|
||||
answered += 1
|
||||
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed("ats", seen_ids, sub.id)
|
||||
else:
|
||||
failed += 1
|
||||
answered = searched = failed = closed = 0
|
||||
for source in SOURCES:
|
||||
run_marker = store.state.get(f"{source}:run", "")
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
if not store.state.get(f"search:complete:{sub.target}", ""):
|
||||
continue
|
||||
else:
|
||||
answered += 1
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
||||
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
|
||||
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
||||
"closed": closed}
|
||||
|
||||
|
||||
def search_target(provider: str, name: str) -> str:
|
||||
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
||||
The query itself lives in `params`, so editing it keeps the target."""
|
||||
return f"{provider}:{company_key(name)}"
|
||||
|
||||
|
||||
def seed_searches(config: Config, store: Store) -> list[str]:
|
||||
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
||||
and only while there are none: the same rule as `seed_agents`."""
|
||||
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
||||
return []
|
||||
added = []
|
||||
for seed in config.searches:
|
||||
try:
|
||||
store.add_subscription("search", search_target(seed.provider, seed.name),
|
||||
label=seed.name, board=seed.provider, params=seed.params)
|
||||
except ValueError as exc:
|
||||
log.warning("searches: %s", exc)
|
||||
continue
|
||||
added.append(seed.name)
|
||||
if added:
|
||||
store.state[SEARCHES_SEEDED] = store.now()
|
||||
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
||||
return added
|
||||
|
||||
|
||||
def pull_task() -> Task:
|
||||
|
||||
Reference in new issue
Block a user