Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
162 lines
6.8 KiB
Python
162 lines
6.8 KiB
Python
"""What a pull reads, changed by hand: add a board or a search, switch one on
|
|
or off. `app companies`, `app searches` and the settings pages all call these,
|
|
so the command line and the pages cannot disagree about what "off" means.
|
|
|
|
A board or search switched off has its open roles closed at once. Nothing
|
|
else would ever close them, since only a pull that reads a target closes its
|
|
roles; switched back on, the next pull reopens every role it still lists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from app.config import Config
|
|
from app.models import Subscription, company_key
|
|
from app.storage import Store
|
|
from app.jobs.pull import search_target
|
|
|
|
if TYPE_CHECKING:
|
|
from app.jobs.discover import Discovery
|
|
|
|
#: The providers `plugins.search` runs, and which of them needs a key.
|
|
PROVIDERS = ("jobindex", "adzuna")
|
|
|
|
|
|
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
|
"""Switch one board or search on or off. Off closes its open roles and
|
|
returns how many; see the module docstring."""
|
|
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
|
|
closed = 0 if fetch else store.close_subscription(sub_id)
|
|
return sub, closed
|
|
|
|
|
|
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
|
"""Follow one company's ATS board: fetch it first, and refuse if the host
|
|
is unknown, the company exists, the board does not answer or lists
|
|
nothing. Returns the subscription and how many roles the board lists."""
|
|
from plugins.ats import careers_url, fetch_board, is_known_target
|
|
|
|
board, name = board.strip(), name.strip()
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
if not is_known_target(board):
|
|
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
|
|
"e.g. greenhouse:acme or ashby:acme")
|
|
key = company_key(name)
|
|
if store.company(key) is not None:
|
|
raise ValueError(f"{name} is already followed")
|
|
try:
|
|
rows, _complete = fetch_board(board)
|
|
except Exception as exc:
|
|
raise ValueError(f"{board} did not answer: {exc}") from exc
|
|
if not rows:
|
|
raise ValueError(f"{board} answered but lists no roles; nothing added")
|
|
host = board.partition(":")[0]
|
|
store.add_company(name, key=key, careers_url=careers_url(board))
|
|
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
return sub, len(rows)
|
|
|
|
|
|
def add_discovered(store: Store, name: str, domain: str,
|
|
found: Discovery) -> tuple[str, int]:
|
|
"""Follow a company from what `app.jobs.discover` found: through
|
|
`add_board` when the board is readable, else as a row with its `ats` host
|
|
and careers page, so the company is kept as one the app cannot read yet.
|
|
Returns the company key and how many roles its board lists (0 with no
|
|
board). Refuses a company that exists, as `add_board` does."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
key = company_key(name)
|
|
if found.board:
|
|
_sub, roles = add_board(store, found.board, name)
|
|
store.update_company(key, domain=domain, ats=found.ats)
|
|
return key, roles
|
|
if store.company(key) is not None:
|
|
raise ValueError(f"{name} is already followed")
|
|
store.add_company(name, key=key, domain=domain, ats=found.ats,
|
|
careers_url=found.careers_url)
|
|
return key, 0
|
|
|
|
|
|
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
|
where: str = "", pages: int = 0) -> dict[str, Any]:
|
|
"""The `params` a search subscription stores, with only the keys its
|
|
provider reads."""
|
|
if provider not in PROVIDERS:
|
|
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
|
|
if not q.strip():
|
|
raise ValueError("a search needs a query")
|
|
params: dict[str, Any] = {"q": q.strip()}
|
|
if provider == "jobindex" and area:
|
|
params["area"] = int(area)
|
|
if provider == "adzuna":
|
|
params["country"] = country.strip().lower()
|
|
if where.strip():
|
|
params["where"] = where.strip()
|
|
if pages:
|
|
params["pages"] = int(pages)
|
|
return params
|
|
|
|
|
|
def has_key(config: Config, provider: str) -> bool:
|
|
"""Whether `provider` can run: jobindex always, adzuna with a key in the
|
|
config or the environment."""
|
|
if provider != "adzuna":
|
|
return True
|
|
keys = (config.sources.get("search") or {}).get("adzuna") or {}
|
|
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
|
|
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
|
|
|
|
|
|
def try_search(config: Config, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
|
"""Run a search once, outside any pull. Raises ValueError with the
|
|
provider's own reason when it does not answer."""
|
|
from plugins.search import run_search
|
|
|
|
try:
|
|
return run_search(search_target(provider, name or "preview"), params,
|
|
dict(config.sources.get("search") or {}))
|
|
except ValueError:
|
|
raise
|
|
except Exception as exc:
|
|
raise ValueError(f"{provider} did not answer: {exc}") from exc
|
|
|
|
|
|
def save_search(config: Config, store: Store, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
|
|
"""Run a search once and save it only if it finds something, so a typo in a
|
|
query shows now and not as an empty tab later. A search of the same
|
|
provider and name is updated in place, removed or not: its roles keep
|
|
their subscription. Returns the subscription, how many roles it found, and
|
|
whether that was every match."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a search needs a name")
|
|
rows, complete = try_search(config, provider, name, params)
|
|
if not rows:
|
|
raise ValueError("the search found nothing; not saved")
|
|
target = search_target(provider, name)
|
|
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
|
|
None)
|
|
if existing is None:
|
|
sub = store.add_subscription("search", target, label=name, board=provider,
|
|
params=params)
|
|
else:
|
|
sub = store.update_subscription(existing.id, label=name, params=params,
|
|
subscribed=True, fetch=True, show=True)
|
|
return sub, len(rows), complete
|
|
|
|
|
|
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
|
|
"""Stop running a search and hide it. Its open roles close; adding a search
|
|
of the same name again brings the row back."""
|
|
sub = store.subscription(sub_id)
|
|
if sub is None or sub.source != "search":
|
|
raise KeyError(sub_id)
|
|
store.unsubscribe(sub_id)
|
|
return sub, store.close_subscription(sub_id)
|