Files
jobboard/app/jobs/sources.py
T
WolfandClaude Opus c7da12b93d Paged boards are read to their stated total, and a board failing for 21 days closes
Workable and SmartRecruiters page their lists, and both parsers read page one
only, so the settle closed every role past it (Delivery Hero: 980 stated, 100
open). `_paged` follows nextPage and offset, and the plugin stamps
`ats:complete` only when the distinct ids read equal the stated total; a 429
after page one keeps what was read and closes nothing. Adzuna completes on its
`count`, Jobindex counts raw items and raises on an answer with no channel.

A board whose fetch raises is recorded under `ats:failing:<target>` with the
first failure's time and the latest reason; after STALE_DAYS the settle closes
its roles, and its next answer reopens them. `/api/companies`, the Companies
list and `app companies check` show it.

Co-Authored-By: Claude Opus <noreply@anthropic.com>
2026-09-29 14:02:49 +02:00

162 lines
6.8 KiB
Python

"""What a pull reads, changed by hand: add a board or a search, switch one on
or off. `app companies`, `app searches` and the settings pages all call these,
so the command line and the pages cannot disagree about what "off" means.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
roles; switched back on, the next pull reopens every role it still lists.
"""
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any
from app.config import Config
from app.models import Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring."""
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
closed = 0 if fetch else store.close_subscription(sub_id)
return sub, closed
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
"""Follow one company's ATS board: fetch it first, and refuse if the host
is unknown, the company exists, the board does not answer or lists
nothing. Returns the subscription and how many roles the board lists."""
from plugins.ats import careers_url, fetch_board, is_known_target
board, name = board.strip(), name.strip()
if not name:
raise ValueError("a company needs a name")
if not is_known_target(board):
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
"e.g. greenhouse:acme or ashby:acme")
key = company_key(name)
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
try:
rows, _complete = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
if not rows:
raise ValueError(f"{board} answered but lists no roles; nothing added")
host = board.partition(":")[0]
store.add_company(name, key=key, careers_url=careers_url(board))
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
return sub, len(rows)
def add_discovered(store: Store, name: str, domain: str,
found: Discovery) -> tuple[str, int]:
"""Follow a company from what `app.jobs.discover` found: through
`add_board` when the board is readable, else as a row with its `ats` host
and careers page, so the company is kept as one the app cannot read yet.
Returns the company key and how many roles its board lists (0 with no
board). Refuses a company that exists, as `add_board` does."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
key = company_key(name)
if found.board:
_sub, roles = add_board(store, found.board, name)
store.update_company(key, domain=domain, ats=found.ats)
return key, roles
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
store.add_company(name, key=key, domain=domain, ats=found.ats,
careers_url=found.careers_url)
return key, 0
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
params: dict[str, Any] = {"q": q.strip()}
if provider == "jobindex" and area:
params["area"] = int(area)
if provider == "adzuna":
params["country"] = country.strip().lower()
if where.strip():
params["where"] = where.strip()
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
keys = (config.sources.get("search") or {}).get("adzuna") or {}
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
def try_search(config: Config, provider: str, name: str,
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""Run a search once, outside any pull. Raises ValueError with the
provider's own reason when it does not answer."""
from plugins.search import run_search
try:
return run_search(search_target(provider, name or "preview"), params,
dict(config.sources.get("search") or {}))
except ValueError:
raise
except Exception as exc:
raise ValueError(f"{provider} did not answer: {exc}") from exc
def save_search(config: Config, store: Store, provider: str, name: str,
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
"""Run a search once and save it only if it finds something, so a typo in a
query shows now and not as an empty tab later. A search of the same
provider and name is updated in place, removed or not: its roles keep
their subscription. Returns the subscription, how many roles it found, and
whether that was every match."""
name = name.strip()
if not name:
raise ValueError("a search needs a name")
rows, complete = try_search(config, provider, name, params)
if not rows:
raise ValueError("the search found nothing; not saved")
target = search_target(provider, name)
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
None)
if existing is None:
sub = store.add_subscription("search", target, label=name, board=provider,
params=params)
else:
sub = store.update_subscription(existing.id, label=name, params=params,
subscribed=True, fetch=True, show=True)
return sub, len(rows), complete
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
"""Stop running a search and hide it. Its open roles close; adding a search
of the same name again brings the row back."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "search":
raise KeyError(sub_id)
store.unsubscribe(sub_id)
return sub, store.close_subscription(sub_id)