Files
jobboard/app/jobs/sources.py
T
WolfandClaude Opus 5.5 b78c9dfc67 Search providers as reader modules: jobindex, adzuna, platsbanken
Each declares READS with scope=search, keeps every place and hands the excerpt over as teaser. search_params and PROVIDERS follow the declarations.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:08:25 +02:00

424 lines
20 KiB
Python

"""What a pull reads, changed by hand: add a board or a search, switch one on
or off, attach, replace or detach a company's board, import the seed.
`app companies`, `app searches` and the settings pages all call these, so the
command line and the pages cannot disagree about what "off" means. Every way
a company gets a board is `set_board`.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
roles; switched back on, the next pull reopens every role it still lists.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import TYPE_CHECKING, Any
from urllib.parse import urlsplit
from app.config import Config
from app.models import HAND_REASONS, Company, Subscription, company_key
from app.storage import Store
from app.jobs.pull import FAILING, search_target
from plugins import readers
if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers, from the reader modules that declare `scope="search"`.
PROVIDERS = tuple(readers.PROVIDERS)
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring. A close regroups the copies
(`Store.rebuild_dupes`), so a copy of a role that closed here shows now
rather than at the next pull."""
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
closed = 0 if fetch else store.close_subscription(sub_id)
if closed:
store.rebuild_dupes()
return sub, closed
def set_board(store: Store, key: str, board: str, *, name: str = "",
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
"""Read company `key` from `board` (`host:slug`): attach it to a company
that has none, replace the one it has, or create the company from `name`
when there is no row. The board is fetched first and refused if its host
is unknown or it does not answer; a board listing zero roles is kept, since
a company between openings is still one to follow. `check=False` skips the
fetch, for the seed, whose boards were read once when it was written.
The row's `ats` and `ats_slug` become the board's host and slug, its
`careers_url` the board's page where it was empty or the old board's, and
its `hand_reason` empty, since a company that is read is not read by hand.
`fields` (domain, country, city, ...) fill only columns that are empty, so
nothing a person or the seed wrote is overwritten. Row and subscription
change in one transaction (`Store.set_company_board`), and replacing a
board closes the old one's open roles. Returns the subscription, how many
roles the board lists (None unchecked) and how many roles closed."""
from plugins.ats import careers_url, fetch_board, is_known_target
board, name = board.strip(), name.strip()
key = key.strip() or company_key(name)
if not is_known_target(board):
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
"e.g. greenhouse:acme or ashby:acme")
company = store.company(key)
if company is None:
if not name:
raise ValueError(f"no company {key!r}; a new company needs a name")
if company_key(name) != key:
raise ValueError(f"{name!r} is not the company {key!r}")
rows: list[Any] | None = None
if check:
try:
rows, _complete = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
writes: dict[str, str] = {k: v for k, v in fields.items() if v}
if company is not None:
writes = {k: v for k, v in writes.items() if not getattr(company, k)}
old = store.company_subscription(key)
if not company.careers_url or (old is not None
and company.careers_url == careers_url(old.target)):
writes["careers_url"] = careers_url(board)
else:
writes.setdefault("careers_url", careers_url(board))
writes.update(ats=host, ats_slug=slug, hand_reason="")
before = store.company_subscription(key)
_company, sub, closed = store.set_company_board(key, board, host=host, name=name,
fields=writes)
assert sub is not None
if before is not None and before.target != board:
_forget_failing(store, before.target)
return sub, None if rows is None else len(rows), closed
def detach_board(store: Store, key: str) -> int:
"""Stop reading company `key`'s board: its open roles close and the row
stays, with its `ats` and `ats_slug`, as one the app does not read.
Returns how many roles closed. Raises KeyError for an unknown company and
ValueError for one with no board."""
if store.company(key) is None:
raise KeyError(key)
sub = store.company_subscription(key)
if sub is None or not sub.show:
raise ValueError(f"{key} has no board to detach")
_company, _sub, closed = store.set_company_board(key, "")
_forget_failing(store, sub.target)
return closed
def _forget_failing(store: Store, target: str) -> None:
"""Drop what the pull recorded about a board nobody reads any more: a
record of `target` failing would otherwise outlive the board, and count
in `app companies check` for good. The `ats` plugin's robots.txt verdict
and its lists of roles read and failed in detail go with it, since they
are about that board alone and would only grow stale."""
store.state.pop(FAILING + target, None)
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
store.state.pop(key + target, None)
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
"""Follow one company by its ATS board: `set_board` keyed by `name`, so
a company that exists gets the board attached or replaced. Returns the
subscription and how many roles the board lists."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
sub, roles, _closed = set_board(store, company_key(name), board, name=name)
return sub, roles or 0
def add_discovered(store: Store, name: str, domain: str,
found: Discovery) -> tuple[str, int]:
"""Follow a company from what `app.jobs.discover` found. A readable
board goes through `set_board`, which attaches it, replaces the one the
company has, or creates the company. Otherwise the row records `ats`,
`ats_slug` and the careers page, as one the app cannot read yet; a
company that already has a board keeps it and this raises, since
discovery finding nothing readable is weaker evidence than a board that
answers. Returns the company key and how many roles its board lists (0
with no board)."""
name = name.strip()
if not name:
raise ValueError("a company needs a name")
key = company_key(name)
if found.board:
_sub, roles, _closed = set_board(store, key, found.board, name=name, domain=domain)
return key, roles or 0
company = store.company(key)
if company is None:
store.add_company(name, key=key, domain=domain, ats=found.ats,
ats_slug=found.ats_slug, careers_url=found.careers_url)
return key, 0
sub = store.company_subscription(key)
if sub is not None and sub.show:
raise ValueError(f"{name} is read from {sub.target} and discovery found no "
"readable board; nothing changed")
# A link somebody typed is kept: discovery's is only a guess at the page.
store.update_company(key, ats=found.ats, ats_slug=found.ats_slug,
careers_url=company.careers_url or found.careers_url,
**({"domain": domain} if not company.domain else {}))
return key, 0
def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool:
"""A row that exists only because a search listed its roles and somebody
judged it (`Store.set_affinity`): no board, not even a detached one
(`ever_boarded`, the keys of every `ats` subscription), and none of the
facts every way of following a company or listing it by hand writes."""
return not (company.key in ever_boarded or company.ats or company.careers_url
or company.domain or company.hand_reason)
def by_hand(store: Store) -> list[Company]:
"""The list checked by hand: companies with no board shown that are not
known from a search only. Never opened first, then the longest ago
(`careers_opened_at`, the Watchlist's stamp); the name breaks a tie."""
boards = store.company_boards()
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
rows = [c for c in store.companies()
if c.key not in boards and not known_from_search_only(c, ever_boarded)]
return sorted(rows, key=lambda c: (c.careers_opened_at, c.name.casefold()))
class Conflict(ValueError):
"""A change refused because of what the company already is, not because of
what was asked for: the pages answer it with 409, not 422."""
def check_careers_url(url: str) -> str:
"""`url` stripped; ValueError unless it is empty or an http(s) link. The
page puts it in an `href`, where `javascript:` would run."""
url = url.strip()
if url:
parts = urlsplit(url)
if parts.scheme not in ("http", "https") or not parts.netloc:
raise ValueError("The careers link must start with http:// or https://.")
return url
def add_by_hand(store: Store, name: str, *, careers_url: str = "", notes: str = "",
hand_reason: str = "added") -> Company:
"""A company with no board, put on the list checked by hand. Raises
Conflict when the key its name folds to is taken, ValueError for a name
with no letters, a link that is no http(s) URL or a reason outside
`HAND_REASONS`."""
link = check_careers_url(careers_url)
key = company_key(name)
if key and store.company(key) is not None:
raise Conflict(f"{name.strip()} is on the list already as {key}.")
return store.add_company(name, key=key, careers_url=link, notes=notes,
hand_reason=hand_reason)
def edit_company(store: Store, key: str, **fields: str) -> Company:
"""Change a company's `name`, `careers_url`, `notes` or `hand_reason`; the
key never changes. A new name is Conflict for a company with a board
subscription, shown or not, since the subscription is found by the key
its company name folds to and would be lost. Raises KeyError for an
unknown company and ValueError for a blank name, a link that is no http(s)
URL or a reason outside `HAND_REASONS`."""
unknown = set(fields) - {"name", "careers_url", "notes", "hand_reason"}
if unknown:
raise ValueError(f"not a field to edit: {', '.join(sorted(unknown))}")
company = store.company(key)
if company is None:
raise KeyError(key)
if "careers_url" in fields:
fields["careers_url"] = check_careers_url(fields["careers_url"])
if "name" in fields:
fields["name"] = fields["name"].strip()
if not fields["name"]:
raise ValueError("A company needs a name.")
if fields["name"] != company.name and store.company_subscription(key) is not None:
raise Conflict(f"{company.name} has a board, which is found by its name; "
"detach the board first.")
return store.update_company(key, **fields)
def remove_company(store: Store, key: str) -> None:
"""Delete company `key` and every agent's stance on it. Raises KeyError
for an unknown company and Conflict for one still read: a company with
a board on (`show`, as `detach_board` decides) or with open roles, which
would otherwise vanish from the list with nothing said. A detached board
is no board, since nothing reads it and its roles are closed; its
subscription row stays, and follows the company if it comes back."""
if store.company(key) is None:
raise KeyError(key)
sub = store.company_subscription(key)
if sub is not None and sub.show:
raise Conflict(f"{key} is read from {sub.target}; detach the board first")
if store.open_role_counts().get(key):
raise Conflict(f"{key} has open roles; detach its board first")
store.remove_company(key)
def _fill_empty(store: Store, key: str, **facts: str) -> None:
"""Write each of `facts` to company `key` where the stored value is
empty: what a seed may add to a row that exists, since a person's link or
reason is never overwritten."""
company = store.company(key)
assert company is not None
fill = {f: v for f, v in facts.items() if v and not getattr(company, f)}
if fill:
store.update_company(key, **fill)
def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
"""Import `seed/companies.json`: one `companies` row per entry, and its
`board` through `set_board` where the entry names one. An entry without
a board is the point of the list as much as one with: a company whose
careers site the app cannot read yet keeps its `ats` host, `ats_slug`
and `careers_url`, and the Companies section says so.
Re-running adds what is missing: a new entry becomes a row, and an
existing row that never had a board gets the one the seed now names, so a
seed update that gains boards reaches a database made before it. A row's
board, once it has one (detached counts), and its affinity, notes and
status are never touched; its `careers_url` and `hand_reason` are filled
where empty, so the seed can hand a row its link and its reason but never
overwrite the one a person set. The seed's boards are not fetched again:
they were read once when the seed was written, and `app companies check`
finds the dead ones. Returns counts, and `problems`, one line per entry
that could not land whole."""
from plugins.ats import is_known_target
data = json.loads(Path(path).read_text(encoding="utf-8"))
entries = data.get("companies", [])
added = boards = skipped = 0
problems: list[str] = []
for row in entries:
name = row["name"].strip()
key = row.get("key") or company_key(name)
board = (row.get("board") or "").strip()
if board and not is_known_target(board):
problems.append(f"{key}: {board!r} is not a known board; kept without it")
board = ""
facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city",
"careers_url", "ats", "ats_slug")}
reason = str(row.get("hand_reason") or "")
if reason not in HAND_REASONS:
problems.append(f"{key}: {reason!r} is not a hand_reason; kept without it")
reason = ""
exists = store.company(key) is not None
if exists:
_fill_empty(store, key, hand_reason=reason)
if exists and (not board or store.company_subscription(key) is not None):
_fill_empty(store, key, careers_url=facts["careers_url"])
skipped += 1
continue
if board:
try:
set_board(store, key, board, name=name, check=False,
**{f: v for f, v in facts.items() if f not in ("ats", "ats_slug")})
boards += 1
added += not exists
continue
except ValueError as exc:
problems.append(f"{key}: {exc}")
if exists:
_fill_empty(store, key, careers_url=facts["careers_url"])
skipped += 1
continue
# No board, or one that clashed: the company is still a row, and a
# clashing board's host and slug are what discovery found.
if board:
facts["ats"], _, facts["ats_slug"] = board.partition(":")
store.add_company(name, key=key, hand_reason=reason, **facts)
added += 1
return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries),
"problems": problems}
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads: those of its module's `PARAMS` whose filter dimension
`READS.filters` declares, cleaned by the module's `clean_params`, which
refuses a value the provider would not accept."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
module = readers.PROVIDERS[provider]
declared = module.READS.filters.declared()
given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country,
"where": where, "region": region}
wanted = {name: given[name] for name, dimension in module.PARAMS.items()
if dimension in declared}
params = module.clean_params(wanted)
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
keys = (config.sources.get("search") or {}).get("adzuna") or {}
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
def try_search(config: Config, provider: str, name: str,
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""Run a search once, outside any pull. Raises ValueError with the
provider's own reason when it does not answer."""
from plugins.search import run_search
try:
return run_search(search_target(provider, name or "preview"), params,
dict(config.sources.get("search") or {}))
except ValueError:
raise
except Exception as exc:
raise ValueError(f"{provider} did not answer: {exc}") from exc
def save_search(config: Config, store: Store, provider: str, name: str,
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
"""Run a search once and save it only if it finds something, so a typo in a
query shows now and not as an empty tab later. A search of the same
provider and name is updated in place, removed or not: its roles keep
their subscription. Returns the subscription, how many roles it found, and
whether that was every match."""
name = name.strip()
if not name:
raise ValueError("a search needs a name")
rows, complete = try_search(config, provider, name, params)
if not rows:
raise ValueError("the search found nothing; not saved")
target = search_target(provider, name)
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
None)
if existing is None:
sub = store.add_subscription("search", target, label=name, board=provider,
params=params)
else:
sub = store.update_subscription(existing.id, label=name, params=params,
subscribed=True, fetch=True, show=True)
return sub, len(rows), complete
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
"""Stop running a search and hide it. Its open roles close; adding a search
of the same name again brings the row back, and the copies regroup as in
`set_fetch`."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "search":
raise KeyError(sub_id)
store.unsubscribe(sub_id)
closed = store.close_subscription(sub_id)
if closed:
store.rebuild_dupes()
return sub, closed