What a pull reads can now be changed on a page instead of only the command line: try a search and see its first ten results before saving it, edit or remove one, follow a company's ATS board, switch any board or search on or off, and preview and apply a prune by region. Each row shows its open roles and the countries they are in. The page and `app companies`/`app searches` call the same functions in app/jobs/sources.py. Switching a target off closes its open roles, and the pull's settle closes them again if a batch in flight reopened them. Roles pulled before `country` existed are backfilled at startup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
172 lines
7.4 KiB
Python
172 lines
7.4 KiB
Python
"""What a pull reads, changed by hand: add a board or a search, switch one on
|
|
or off, prune boards by region. `app companies`, `app searches` and the
|
|
Sources tab all call these, so the command line and the page cannot disagree
|
|
about what "off" means.
|
|
|
|
A board or search switched off has its open roles closed at once. Nothing
|
|
else would ever close them, since only a pull that reads a target closes its
|
|
roles; switched back on, the next pull reopens every role it still lists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any
|
|
|
|
from app.config import Config
|
|
from app.models import EUROPE, Subscription, company_key
|
|
from app.storage import Store
|
|
from app.jobs.pull import search_target
|
|
|
|
#: The providers `plugins.search` runs, and which of them needs a key.
|
|
PROVIDERS = ("jobindex", "adzuna")
|
|
|
|
|
|
def region_codes(region: str) -> set[str]:
|
|
"""`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of
|
|
codes. `europe` may be one entry among codes: `europe,US`."""
|
|
codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()}
|
|
if "EUROPE" in codes:
|
|
codes = (codes - {"EUROPE"}) | set(EUROPE)
|
|
if not codes:
|
|
raise ValueError("name a region: europe, or country codes such as DK,SE,EU")
|
|
return codes
|
|
|
|
|
|
def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]:
|
|
"""Every fetched board none of whose open roles is in the region, with its
|
|
open role count, largest first; and how many roles had `country` filled
|
|
from `place` on the way. A board whose roles all name no country is kept:
|
|
it says nothing about where that company hires."""
|
|
wanted = region_codes(region)
|
|
filled = store.backfill_country()
|
|
countries = store.open_countries_by_subscription()
|
|
plan = [(sub, sum(countries[sub.id].values()))
|
|
for sub in store.subscriptions(source="ats", fetchable=True)
|
|
if set(countries.get(sub.id, {})) - {""}
|
|
and not wanted.intersection(countries[sub.id])]
|
|
return filled, sorted(plan, key=lambda p: -p[1])
|
|
|
|
|
|
def prune(store: Store, region: str) -> dict[str, Any]:
|
|
"""Switch off every board `prune_plan` names."""
|
|
filled, plan = prune_plan(store, region)
|
|
closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan)
|
|
return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed,
|
|
"boards_left": len(store.subscriptions(source="ats", fetchable=True))}
|
|
|
|
|
|
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
|
"""Switch one board or search on or off. Off closes its open roles and
|
|
returns how many; see the module docstring."""
|
|
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
|
|
closed = 0 if fetch else store.close_subscription(sub_id)
|
|
return sub, closed
|
|
|
|
|
|
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
|
"""Follow one company's ATS board: fetch it first, and refuse if the host
|
|
is unknown, the company exists, the board does not answer or lists
|
|
nothing. Returns the subscription and how many roles the board lists."""
|
|
from plugins.ats import careers_url, fetch_board, is_known_target
|
|
|
|
board, name = board.strip(), name.strip()
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
if not is_known_target(board):
|
|
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
|
|
"e.g. greenhouse:acme or ashby:acme")
|
|
key = company_key(name)
|
|
if store.company(key) is not None:
|
|
raise ValueError(f"{name} is already followed")
|
|
try:
|
|
rows = fetch_board(board)
|
|
except Exception as exc:
|
|
raise ValueError(f"{board} did not answer: {exc}") from exc
|
|
if not rows:
|
|
raise ValueError(f"{board} answered but lists no roles; nothing added")
|
|
host = board.partition(":")[0]
|
|
store.add_company(name, key=key, careers_url=careers_url(board))
|
|
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
|
|
return sub, len(rows)
|
|
|
|
|
|
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
|
where: str = "", pages: int = 0) -> dict[str, Any]:
|
|
"""The `params` a search subscription stores, with only the keys its
|
|
provider reads."""
|
|
if provider not in PROVIDERS:
|
|
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
|
|
if not q.strip():
|
|
raise ValueError("a search needs a query")
|
|
params: dict[str, Any] = {"q": q.strip()}
|
|
if provider == "jobindex" and area:
|
|
params["area"] = int(area)
|
|
if provider == "adzuna":
|
|
params["country"] = country.strip().lower()
|
|
if where.strip():
|
|
params["where"] = where.strip()
|
|
if pages:
|
|
params["pages"] = int(pages)
|
|
return params
|
|
|
|
|
|
def has_key(config: Config, provider: str) -> bool:
|
|
"""Whether `provider` can run: jobindex always, adzuna with a key in the
|
|
config or the environment."""
|
|
if provider != "adzuna":
|
|
return True
|
|
keys = (config.sources.get("search") or {}).get("adzuna") or {}
|
|
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
|
|
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
|
|
|
|
|
|
def try_search(config: Config, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
|
"""Run a search once, outside any pull. Raises ValueError with the
|
|
provider's own reason when it does not answer."""
|
|
from plugins.search import run_search
|
|
|
|
try:
|
|
return run_search(search_target(provider, name or "preview"), params,
|
|
dict(config.sources.get("search") or {}))
|
|
except ValueError:
|
|
raise
|
|
except Exception as exc:
|
|
raise ValueError(f"{provider} did not answer: {exc}") from exc
|
|
|
|
|
|
def save_search(config: Config, store: Store, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
|
|
"""Run a search once and save it only if it finds something, so a typo in a
|
|
query shows now and not as an empty tab later. A search of the same
|
|
provider and name is updated in place, removed or not: its roles keep
|
|
their subscription. Returns the subscription, how many roles it found, and
|
|
whether that was every match."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a search needs a name")
|
|
rows, complete = try_search(config, provider, name, params)
|
|
if not rows:
|
|
raise ValueError("the search found nothing; not saved")
|
|
target = search_target(provider, name)
|
|
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
|
|
None)
|
|
if existing is None:
|
|
sub = store.add_subscription("search", target, label=name, board=provider,
|
|
params=params)
|
|
else:
|
|
sub = store.update_subscription(existing.id, label=name, params=params,
|
|
subscribed=True, fetch=True, show=True)
|
|
return sub, len(rows), complete
|
|
|
|
|
|
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
|
|
"""Stop running a search and hide it. Its open roles close; adding a search
|
|
of the same name again brings the row back."""
|
|
sub = store.subscription(sub_id)
|
|
if sub is None or sub.source != "search":
|
|
raise KeyError(sub_id)
|
|
store.unsubscribe(sub_id)
|
|
return sub, store.close_subscription(sub_id)
|