The top bar is Roles and Applications plus health's cog. Behind the cog,
/settings/{ranking,searches,companies,pulls,messages,account} share one row
of sub-tabs, an underlined row on a desktop and pills on a phone. The old
paths answer 301 with their query string.
Ranking edits the first enabled agent, or creates one, with the intent,
the weights and the gates under Advanced. The order menu on Roles reads
Newest and Best match with one enabled agent, and names them with several.
config.yaml keeps the Copenhagen starter; the other two are commented
examples in config.example.yaml.
roles.js is split into roles.js, applications.js, ranking.js and
companies.js; count, cell, debounce, the company list and the affinity
control live once in app.js. app.css gets the phone frame after health and
newsdesk: one-row bar, edge-to-edge panels, 44px targets and 16px fields on
a coarse pointer, hover only where there is one, safe areas. The app is
installable from a manifest and roledesk-coloured icons. Every lede touched
is written for the person using the app.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
193 lines
8.1 KiB
Python
193 lines
8.1 KiB
Python
"""`pull`: fetch every active `ats` board and `search`, and store the roles
|
|
they list.
|
|
|
|
A task, not a single job: ten boards or searches per `pull_batch` job on the
|
|
`http` resource pool, because a board is one HTTP round trip and that pool is
|
|
sized for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which each plugin itself stamps per target
|
|
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
|
under `search:`), rather than from what each `pull_batch` job happens to
|
|
return. A batch job that raises partway through still leaves the boards it had
|
|
already finished stamped, so one bad board in a batch of ten does not cost the
|
|
other nine their closure.
|
|
|
|
A search closes the same way when its results ran out before its page limit,
|
|
since it then listed everything that matches. One that hit the limit did not,
|
|
so its roles close only once no search has seen them for `STALE_DAYS`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, RunContext, Tally, Task, job
|
|
|
|
from app.config import Config
|
|
from app.models import company_key
|
|
from app.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: As many boards in the air per job as there are `http` workers by default
|
|
#: (see `config.yaml`), so one pull keeps the pool busy without one job
|
|
#: holding all 190-odd boards' worth of retries against a single attempt.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "app.jobs.pull:close_boards"
|
|
|
|
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
|
SOURCES = ("ats", "search")
|
|
|
|
#: How long a search role stays open once no search lists it any more.
|
|
STALE_DAYS = 21
|
|
|
|
#: The `state` key that records the starter searches were imported.
|
|
SEARCHES_SEEDED = "searches:seeded"
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
|
source. Stamps the run marker `close_boards` will look for, before any
|
|
batch runs — finders run synchronously, in whoever starts the task, ahead
|
|
of every job it queues."""
|
|
seed_searches(config, store)
|
|
batches: list[dict[str, Any]] = []
|
|
for source in SOURCES:
|
|
subs = store.subscriptions(source=source, fetchable=True)
|
|
if not subs:
|
|
continue
|
|
store.state[f"{source}:run"] = _now()
|
|
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
|
|
for i in range(0, len(subs), BATCH_SIZE)]
|
|
return batches
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards or searches and `upsert_role` every item
|
|
they list.
|
|
|
|
`targets` is keyword-only without a default, which is what makes it a
|
|
required payload key to the queue: the Pulls page offers no button for a
|
|
batch that could only fail without one.
|
|
|
|
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
|
`options[subscription_field]`, so handing it only this batch's entries is
|
|
what keeps one job from re-fetching every other job's boards.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
from plugins.search import SearchSource
|
|
|
|
# Fetchable now, not when the pull was queued: a board switched off on the
|
|
# Searches or Companies page since then is skipped.
|
|
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
|
if s.target in targets}
|
|
options = dict(config.sources.get(source) or {})
|
|
plugin: ATSSource | SearchSource
|
|
if source == "search":
|
|
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
|
for t in targets if t in subs]
|
|
plugin = SearchSource(options)
|
|
else:
|
|
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
|
for t in targets if t in subs]
|
|
plugin = ATSSource(options)
|
|
plugin.state = store.state
|
|
|
|
stored = 0
|
|
for item in plugin.fetch():
|
|
item.source = source
|
|
if source == "search":
|
|
target = str(item.raw.get("search", ""))
|
|
else:
|
|
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
|
store.upsert_role(item, subs.get(target))
|
|
stored += 1
|
|
return {"roles": stored}
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, and tally what happened.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed. A search closes by its
|
|
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
|
|
|
A board or search switched off while its batch was in flight had its
|
|
roles reopened by that batch, after the switch closed them, so every
|
|
switched-off one is closed again here.
|
|
"""
|
|
answered = searched = failed = closed = 0
|
|
for source in SOURCES:
|
|
for sub in store.subscriptions(source=source):
|
|
if not (sub.subscribed and sub.fetch):
|
|
closed += store.close_subscription(sub.id)
|
|
run_marker = store.state.get(f"{source}:run", "")
|
|
for sub in store.subscriptions(source=source, fetchable=True):
|
|
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
|
if not (stamp and stamp == run_marker):
|
|
failed += 1
|
|
continue
|
|
if source == "search":
|
|
searched += 1
|
|
if not store.state.get(f"search:complete:{sub.target}", ""):
|
|
continue
|
|
else:
|
|
answered += 1
|
|
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
|
closed += store.mark_closed(source, seen_ids, sub.id)
|
|
if searched:
|
|
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
|
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
|
"closed": closed}
|
|
|
|
|
|
def search_target(provider: str, name: str) -> str:
|
|
"""The subscription target of a saved search: `jobindex:copenhagen-backend`.
|
|
The query itself lives in `params`, so editing it keeps the target."""
|
|
return f"{provider}:{company_key(name)}"
|
|
|
|
|
|
def seed_searches(config: Config, store: Store) -> list[str]:
|
|
"""Import `config.yaml`'s `searches:` as `search` subscriptions, once,
|
|
and only while there are none: the same rule as `seed_agents`."""
|
|
if store.state.get(SEARCHES_SEEDED) or store.subscriptions(source="search"):
|
|
return []
|
|
added = []
|
|
for seed in config.searches:
|
|
try:
|
|
store.add_subscription("search", search_target(seed.provider, seed.name),
|
|
label=seed.name, board=seed.provider, params=seed.params)
|
|
except ValueError as exc:
|
|
log.warning("searches: %s", exc)
|
|
continue
|
|
added.append(seed.name)
|
|
if added:
|
|
store.state[SEARCHES_SEEDED] = store.now()
|
|
log.info("imported %d searches from %s", len(added), config.path or "the defaults")
|
|
return added
|
|
|
|
|
|
def pull_task() -> Task:
|
|
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
|
of the chain the `pull` job starts."""
|
|
return Task(kind="pull_batch", finder=_boards, name="pull",
|
|
label="Reading every board", settle=SETTLE)
|