"""What a pull reads, changed by hand: add a board or a search, switch one on or off, attach, replace or detach a company's board, import the seed. `app companies`, `app searches` and the settings pages all call these, so the command line and the pages cannot disagree about what "off" means. Every way a company gets a board is `set_board`. A board or search switched off has its open roles closed at once. Nothing else would ever close them, since only a pull that reads a target closes its roles; switched back on, the next pull reopens every role it still lists. """ from __future__ import annotations import json import os from pathlib import Path from typing import TYPE_CHECKING, Any from app.config import Config from app.models import Subscription, company_key from app.storage import Store from app.jobs.pull import FAILING, search_target if TYPE_CHECKING: from app.jobs.discover import Discovery #: The providers `plugins.search` runs, and which of them needs a key. PROVIDERS = ("jobindex", "adzuna") def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: """Switch one board or search on or off. Off closes its open roles and returns how many; see the module docstring.""" sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True) closed = 0 if fetch else store.close_subscription(sub_id) return sub, closed def set_board(store: Store, key: str, board: str, *, name: str = "", check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]: """Read company `key` from `board` (`host:slug`): attach it to a company that has none, replace the one it has, or create the company from `name` when there is no row. The board is fetched first and refused if its host is unknown or it does not answer; a board listing zero roles is kept, since a company between openings is still one to follow. `check=False` skips the fetch, for the seed, whose boards were read once when it was written. The row's `ats` and `ats_slug` become the board's host and slug, and its `careers_url` the board's page where it was empty or the old board's. `fields` (domain, country, city, ...) fill only columns that are empty, so nothing a person or the seed wrote is overwritten. Row and subscription change in one transaction (`Store.set_company_board`), and replacing a board closes the old one's open roles. Returns the subscription, how many roles the board lists (None unchecked) and how many roles closed.""" from plugins.ats import careers_url, fetch_board, is_known_target board, name = board.strip(), name.strip() key = key.strip() or company_key(name) if not is_known_target(board): raise ValueError(f"{board!r} is not host:slug for a known ATS host, " "e.g. greenhouse:acme or ashby:acme") company = store.company(key) if company is None: if not name: raise ValueError(f"no company {key!r}; a new company needs a name") if company_key(name) != key: raise ValueError(f"{name!r} is not the company {key!r}") rows: list[Any] | None = None if check: try: rows, _complete = fetch_board(board) except Exception as exc: raise ValueError(f"{board} did not answer: {exc}") from exc host, _, slug = board.partition(":") writes: dict[str, str] = {k: v for k, v in fields.items() if v} if company is not None: writes = {k: v for k, v in writes.items() if not getattr(company, k)} old = store.company_subscription(key) if not company.careers_url or (old is not None and company.careers_url == careers_url(old.target)): writes["careers_url"] = careers_url(board) else: writes.setdefault("careers_url", careers_url(board)) writes.update(ats=host, ats_slug=slug) before = store.company_subscription(key) _company, sub, closed = store.set_company_board(key, board, host=host, name=name, fields=writes) assert sub is not None if before is not None and before.target != board: _forget_failing(store, before.target) return sub, None if rows is None else len(rows), closed def detach_board(store: Store, key: str) -> int: """Stop reading company `key`'s board: its open roles close and the row stays, with its `ats` and `ats_slug`, as one the app does not read. Returns how many roles closed. Raises KeyError for an unknown company and ValueError for one with no board.""" if store.company(key) is None: raise KeyError(key) sub = store.company_subscription(key) if sub is None or not sub.show: raise ValueError(f"{key} has no board to detach") _company, _sub, closed = store.set_company_board(key, "") _forget_failing(store, sub.target) return closed def _forget_failing(store: Store, target: str) -> None: """Drop what the pull recorded about a board nobody reads any more: a record of `target` failing would otherwise outlive the board, and count in `app companies check` for good.""" store.state.pop(FAILING + target, None) def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]: """Follow one company by its ATS board: `set_board` keyed by `name`, so a company that exists gets the board attached or replaced. Returns the subscription and how many roles the board lists.""" name = name.strip() if not name: raise ValueError("a company needs a name") sub, roles, _closed = set_board(store, company_key(name), board, name=name) return sub, roles or 0 def add_discovered(store: Store, name: str, domain: str, found: Discovery) -> tuple[str, int]: """Follow a company from what `app.jobs.discover` found. A readable board goes through `set_board`, which attaches it, replaces the one the company has, or creates the company. Otherwise the row records `ats`, `ats_slug` and the careers page, as one the app cannot read yet; a company that already has a board keeps it and this raises, since discovery finding nothing readable is weaker evidence than a board that answers. Returns the company key and how many roles its board lists (0 with no board).""" name = name.strip() if not name: raise ValueError("a company needs a name") key = company_key(name) if found.board: _sub, roles, _closed = set_board(store, key, found.board, name=name, domain=domain) return key, roles or 0 company = store.company(key) if company is None: store.add_company(name, key=key, domain=domain, ats=found.ats, ats_slug=found.ats_slug, careers_url=found.careers_url) return key, 0 sub = store.company_subscription(key) if sub is not None and sub.show: raise ValueError(f"{name} is read from {sub.target} and discovery found no " "readable board; nothing changed") store.update_company(key, ats=found.ats, ats_slug=found.ats_slug, careers_url=found.careers_url or company.careers_url, **({"domain": domain} if not company.domain else {})) return key, 0 def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: """Import `seed/companies.json`: one `companies` row per entry, and its `board` through `set_board` where the entry names one. An entry without a board is the point of the list as much as one with: a company whose careers site the app cannot read yet keeps its `ats` host, `ats_slug` and `careers_url`, and the Companies section says so. Re-running adds what is missing: a new entry becomes a row, and an existing row that never had a board gets the one the seed now names, so a seed update that gains boards reaches a database made before it. A row's board, once it has one (detached counts), and its affinity, notes and status are never touched. The seed's boards are not fetched again: they were read once when the seed was written, and `app companies check` finds the dead ones. Returns counts, and `problems`, one line per entry that could not land whole.""" from plugins.ats import is_known_target data = json.loads(Path(path).read_text(encoding="utf-8")) entries = data.get("companies", []) added = boards = skipped = 0 problems: list[str] = [] for row in entries: name = row["name"].strip() key = row.get("key") or company_key(name) board = (row.get("board") or "").strip() if board and not is_known_target(board): problems.append(f"{key}: {board!r} is not a known board; kept without it") board = "" facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city", "careers_url", "ats", "ats_slug")} exists = store.company(key) is not None if exists and (not board or store.company_subscription(key) is not None): skipped += 1 continue if board: try: set_board(store, key, board, name=name, check=False, **{f: v for f, v in facts.items() if f not in ("ats", "ats_slug")}) boards += 1 added += not exists continue except ValueError as exc: problems.append(f"{key}: {exc}") if exists: skipped += 1 continue # No board, or one that clashed: the company is still a row, and a # clashing board's host and slug are what discovery found. if board: facts["ats"], _, facts["ats_slug"] = board.partition(":") store.add_company(name, key=key, **facts) added += 1 return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries), "problems": problems} def search_params(provider: str, q: str, *, area: int = 0, country: str = "", where: str = "", pages: int = 0) -> dict[str, Any]: """The `params` a search subscription stores, with only the keys its provider reads.""" if provider not in PROVIDERS: raise ValueError(f"provider is one of {', '.join(PROVIDERS)}") if not q.strip(): raise ValueError("a search needs a query") params: dict[str, Any] = {"q": q.strip()} if provider == "jobindex" and area: params["area"] = int(area) if provider == "adzuna": params["country"] = country.strip().lower() if where.strip(): params["where"] = where.strip() if pages: params["pages"] = int(pages) return params def has_key(config: Config, provider: str) -> bool: """Whether `provider` can run: jobindex always, adzuna with a key in the config or the environment.""" if provider != "adzuna": return True keys = (config.sources.get("search") or {}).get("adzuna") or {} return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID")) and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY"))) def try_search(config: Config, provider: str, name: str, params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: """Run a search once, outside any pull. Raises ValueError with the provider's own reason when it does not answer.""" from plugins.search import run_search try: return run_search(search_target(provider, name or "preview"), params, dict(config.sources.get("search") or {})) except ValueError: raise except Exception as exc: raise ValueError(f"{provider} did not answer: {exc}") from exc def save_search(config: Config, store: Store, provider: str, name: str, params: dict[str, Any]) -> tuple[Subscription, int, bool]: """Run a search once and save it only if it finds something, so a typo in a query shows now and not as an empty tab later. A search of the same provider and name is updated in place, removed or not: its roles keep their subscription. Returns the subscription, how many roles it found, and whether that was every match.""" name = name.strip() if not name: raise ValueError("a search needs a name") rows, complete = try_search(config, provider, name, params) if not rows: raise ValueError("the search found nothing; not saved") target = search_target(provider, name) existing = next((s for s in store.subscriptions(source="search") if s.target == target), None) if existing is None: sub = store.add_subscription("search", target, label=name, board=provider, params=params) else: sub = store.update_subscription(existing.id, label=name, params=params, subscribed=True, fetch=True, show=True) return sub, len(rows), complete def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]: """Stop running a search and hide it. Its open roles close; adding a search of the same name again brings the row back.""" sub = store.subscription(sub_id) if sub is None or sub.source != "search": raise KeyError(sub_id) store.unsubscribe(sub_id) return sub, store.close_subscription(sub_id)