Files
jobboard/app/jobs/sources.py
T
WolfandClaude Opus 5.5 3b209d628a Every way a company gets a board is set_board, and an unreadable host keeps its slug
`app.jobs.sources.set_board` attaches a board to an existing company,
replaces the one it has (closing the old board's open roles) or creates
the company, with row and subscription changed in one transaction by
`Store.set_company_board`. A board that lists zero roles is kept. `add`,
`discover --add`, the seed importer (moved out of app/cli.py) and the new
`POST|DELETE /api/companies/{key}/board` all go through it; a seed re-run
attaches a board to a row that never had one. A company's subscription is
rewritten in place, and detaching hides it.

`companies.ats_slug` (migration 9) keeps the slug of a host the app
cannot read yet, and discovery returns such a host before probing slugs.
The Companies rows get Set board / Change, the host select is rendered
from `plugins.ats.HOSTS`, and a row with nothing found reads "board not
found" and is in the "Not readable yet" filter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 14:04:52 +02:00

282 lines
13 KiB
Python

"""What a pull reads, changed by hand: add a board or a search, switch one on
or off, attach, replace or detach a company's board, import the seed.
`app companies`, `app searches` and the settings pages all call these, so the
command line and the pages cannot disagree about what "off" means. Every way
a company gets a board is `set_board`.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
roles; switched back on, the next pull reopens every role it still lists.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app.config import Config
from app.models import Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring."""
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
closed = 0 if fetch else store.close_subscription(sub_id)
return sub, closed
def set_board(store: Store, key: str, board: str, *, name: str = "",
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
"""Read company `key` from `board` (`host:slug`): attach it to a company
that has none, replace the one it has, or create the company from `name`
when there is no row. The board is fetched first and refused if its host
is unknown or it does not answer; a board listing zero roles is kept, since
a company between openings is still one to follow. `check=False` skips the
fetch, for the seed, whose boards were read once when it was written.
The row's `ats` and `ats_slug` become the board's host and slug, and its
`careers_url` the board's page where it was empty or the old board's.
`fields` (domain, country, city, ...) fill only columns that are empty, so
nothing a person or the seed wrote is overwritten. Row and subscription
change in one transaction (`Store.set_company_board`), and replacing a
board closes the old one's open roles. Returns the subscription, how many
roles the board lists (None unchecked) and how many roles closed."""
from plugins.ats import careers_url, fetch_board, is_known_target
board, name = board.strip(), name.strip()
key = key.strip() or company_key(name)
if not is_known_target(board):
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
"e.g. greenhouse:acme or ashby:acme")
company = store.company(key)
if company is None:
if not name:
raise ValueError(f"no company {key!r}; a new company needs a name")
if company_key(name) != key:
raise ValueError(f"{name!r} is not the company {key!r}")
rows: list[Any] | None = None
if check:
try:
rows = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
writes: dict[str, str] = {k: v for k, v in fields.items() if v}
if company is not None:
writes = {k: v for k, v in writes.items() if not getattr(company, k)}
old = store.company_subscription(key)
if not company.careers_url or (old is not None
and company.careers_url == careers_url(old.target)):
writes["careers_url"] = careers_url(board)
else:
writes.setdefault("careers_url", careers_url(board))
writes.update(ats=host, ats_slug=slug)
_company, sub, closed = store.set_company_board(key, board, host=host, name=name,
fields=writes)
assert sub is not None
return sub, None if rows is None else len(rows), closed
def detach_board(store: Store, key: str) -> int:
"""Stop reading company `key`'s board: its open roles close and the row
stays, with its `ats` and `ats_slug`, as one the app does not read.
Returns how many roles closed. Raises KeyError for an unknown company and
ValueError for one with no board."""
if store.company(key) is None:
raise KeyError(key)
sub = store.company_subscription(key)
if sub is None or not sub.show:
raise ValueError(f"{key} has no board to detach")
_company, _sub, closed = store.set_company_board(key, "")
return closed
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
"""Follow one company by its ATS board: `set_board` keyed by `name`, so
a company that exists gets the board attached or replaced. Returns the
subscription and how many roles the board lists."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
sub, roles, _closed = set_board(store, company_key(name), board, name=name)
return sub, roles or 0
def add_discovered(store: Store, name: str, domain: str,
found: Discovery) -> tuple[str, int]:
"""Follow a company from what `app.jobs.discover` found. A readable
board goes through `set_board`, which attaches it, replaces the one the
company has, or creates the company. Otherwise the row records `ats`,
`ats_slug` and the careers page, as one the app cannot read yet; a
company that already has a board keeps it and this raises, since
discovery finding nothing readable is weaker evidence than a board that
answers. Returns the company key and how many roles its board lists (0
with no board)."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
key = company_key(name)
if found.board:
_sub, roles, _closed = set_board(store, key, found.board, name=name, domain=domain)
return key, roles or 0
company = store.company(key)
if company is None:
store.add_company(name, key=key, domain=domain, ats=found.ats,
ats_slug=found.ats_slug, careers_url=found.careers_url)
return key, 0
sub = store.company_subscription(key)
if sub is not None and sub.show:
raise ValueError(f"{name} is read from {sub.target} and discovery found no "
"readable board; nothing changed")
store.update_company(key, ats=found.ats, ats_slug=found.ats_slug,
careers_url=found.careers_url or company.careers_url,
**({"domain": domain} if not company.domain else {}))
return key, 0
def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
"""Import `seed/companies.json`: one `companies` row per entry, and its
`board` through `set_board` where the entry names one. An entry without
a board is the point of the list as much as one with: a company whose
careers site the app cannot read yet keeps its `ats` host, `ats_slug`
and `careers_url`, and the Companies section says so.
Re-running adds what is missing: a new entry becomes a row, and an
existing row that never had a board gets the one the seed now names, so a
seed update that gains boards reaches a database made before it. A row's
board, once it has one (detached counts), and its affinity, notes and
status are never touched. The seed's boards are not fetched again: they
were read once when the seed was written, and `app companies check`
finds the dead ones. Returns counts, and `problems`, one line per entry
that could not land whole."""
from plugins.ats import is_known_target
data = json.loads(Path(path).read_text(encoding="utf-8"))
entries = data.get("companies", [])
added = boards = skipped = 0
problems: list[str] = []
for row in entries:
name = row["name"].strip()
key = row.get("key") or company_key(name)
board = (row.get("board") or "").strip()
if board and not is_known_target(board):
problems.append(f"{key}: {board!r} is not a known board; kept without it")
board = ""
facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city",
"careers_url", "ats", "ats_slug")}
exists = store.company(key) is not None
if exists and (not board or store.company_subscription(key) is not None):
skipped += 1
continue
if board:
try:
set_board(store, key, board, name=name, check=False,
**{f: v for f, v in facts.items() if f not in ("ats", "ats_slug")})
boards += 1
added += not exists
continue
except ValueError as exc:
problems.append(f"{key}: {exc}")
if exists:
skipped += 1
continue
# No board, or one that clashed: the company is still a row, and a
# clashing board's host and slug are what discovery found.
if board:
facts["ats"], _, facts["ats_slug"] = board.partition(":")
store.add_company(name, key=key, **facts)
added += 1
return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries),
"problems": problems}
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
params: dict[str, Any] = {"q": q.strip()}
if provider == "jobindex" and area:
params["area"] = int(area)
if provider == "adzuna":
params["country"] = country.strip().lower()
if where.strip():
params["where"] = where.strip()
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
keys = (config.sources.get("search") or {}).get("adzuna") or {}
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
def try_search(config: Config, provider: str, name: str,
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""Run a search once, outside any pull. Raises ValueError with the
provider's own reason when it does not answer."""
from plugins.search import run_search
try:
return run_search(search_target(provider, name or "preview"), params,
dict(config.sources.get("search") or {}))
except ValueError:
raise
except Exception as exc:
raise ValueError(f"{provider} did not answer: {exc}") from exc
def save_search(config: Config, store: Store, provider: str, name: str,
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
"""Run a search once and save it only if it finds something, so a typo in a
query shows now and not as an empty tab later. A search of the same
provider and name is updated in place, removed or not: its roles keep
their subscription. Returns the subscription, how many roles it found, and
whether that was every match."""
name = name.strip()
if not name:
raise ValueError("a search needs a name")
rows, complete = try_search(config, provider, name, params)
if not rows:
raise ValueError("the search found nothing; not saved")
target = search_target(provider, name)
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
None)
if existing is None:
sub = store.add_subscription("search", target, label=name, board=provider,
params=params)
else:
sub = store.update_subscription(existing.id, label=name, params=params,
subscribed=True, fetch=True, show=True)
return sub, len(rows), complete
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
"""Stop running a search and hide it. Its open roles close; adding a search
of the same name again brings the row back."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "search":
raise KeyError(sub_id)
store.unsubscribe(sub_id)
return sub, store.close_subscription(sub_id)