420 lines
20 KiB
Python
420 lines
20 KiB
Python
"""What a pull reads, changed by hand: add a board or a search, switch one on
|
|
or off, attach, replace or detach a company's board, import the seed.
|
|
`app companies`, `app searches` and the settings pages all call these, so the
|
|
command line and the pages cannot disagree about what "off" means. Every way
|
|
a company gets a board is `set_board`.
|
|
|
|
A board or search switched off has its open roles closed at once. Nothing
|
|
else would ever close them, since only a pull that reads a target closes its
|
|
roles; switched back on, the next pull reopens every role it still lists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
from urllib.parse import urlsplit
|
|
|
|
from app.config import Config
|
|
from app.models import HAND_REASONS, Company, Subscription, company_key
|
|
from app.storage import Store
|
|
from app.jobs.pull import FAILING, search_target
|
|
|
|
if TYPE_CHECKING:
|
|
from app.jobs.discover import Discovery
|
|
|
|
#: The providers `plugins.search` runs, and which of them needs a key.
|
|
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
|
|
|
|
|
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
|
"""Switch one board or search on or off. Off closes its open roles and
|
|
returns how many; see the module docstring."""
|
|
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
|
|
closed = 0 if fetch else store.close_subscription(sub_id)
|
|
return sub, closed
|
|
|
|
|
|
def set_board(store: Store, key: str, board: str, *, name: str = "",
|
|
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
|
|
"""Read company `key` from `board` (`host:slug`): attach it to a company
|
|
that has none, replace the one it has, or create the company from `name`
|
|
when there is no row. The board is fetched first and refused if its host
|
|
is unknown or it does not answer; a board listing zero roles is kept, since
|
|
a company between openings is still one to follow. `check=False` skips the
|
|
fetch, for the seed, whose boards were read once when it was written.
|
|
|
|
The row's `ats` and `ats_slug` become the board's host and slug, its
|
|
`careers_url` the board's page where it was empty or the old board's, and
|
|
its `hand_reason` empty, since a company that is read is not read by hand.
|
|
`fields` (domain, country, city, ...) fill only columns that are empty, so
|
|
nothing a person or the seed wrote is overwritten. Row and subscription
|
|
change in one transaction (`Store.set_company_board`), and replacing a
|
|
board closes the old one's open roles. Returns the subscription, how many
|
|
roles the board lists (None unchecked) and how many roles closed."""
|
|
from plugins.ats import careers_url, fetch_board, is_known_target
|
|
|
|
board, name = board.strip(), name.strip()
|
|
key = key.strip() or company_key(name)
|
|
if not is_known_target(board):
|
|
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
|
|
"e.g. greenhouse:acme or ashby:acme")
|
|
company = store.company(key)
|
|
if company is None:
|
|
if not name:
|
|
raise ValueError(f"no company {key!r}; a new company needs a name")
|
|
if company_key(name) != key:
|
|
raise ValueError(f"{name!r} is not the company {key!r}")
|
|
rows: list[Any] | None = None
|
|
if check:
|
|
try:
|
|
rows, _complete = fetch_board(board)
|
|
except Exception as exc:
|
|
raise ValueError(f"{board} did not answer: {exc}") from exc
|
|
host, _, slug = board.partition(":")
|
|
writes: dict[str, str] = {k: v for k, v in fields.items() if v}
|
|
if company is not None:
|
|
writes = {k: v for k, v in writes.items() if not getattr(company, k)}
|
|
old = store.company_subscription(key)
|
|
if not company.careers_url or (old is not None
|
|
and company.careers_url == careers_url(old.target)):
|
|
writes["careers_url"] = careers_url(board)
|
|
else:
|
|
writes.setdefault("careers_url", careers_url(board))
|
|
writes.update(ats=host, ats_slug=slug, hand_reason="")
|
|
before = store.company_subscription(key)
|
|
_company, sub, closed = store.set_company_board(key, board, host=host, name=name,
|
|
fields=writes)
|
|
assert sub is not None
|
|
if before is not None and before.target != board:
|
|
_forget_failing(store, before.target)
|
|
return sub, None if rows is None else len(rows), closed
|
|
|
|
|
|
def detach_board(store: Store, key: str) -> int:
|
|
"""Stop reading company `key`'s board: its open roles close and the row
|
|
stays, with its `ats` and `ats_slug`, as one the app does not read.
|
|
Returns how many roles closed. Raises KeyError for an unknown company and
|
|
ValueError for one with no board."""
|
|
if store.company(key) is None:
|
|
raise KeyError(key)
|
|
sub = store.company_subscription(key)
|
|
if sub is None or not sub.show:
|
|
raise ValueError(f"{key} has no board to detach")
|
|
_company, _sub, closed = store.set_company_board(key, "")
|
|
_forget_failing(store, sub.target)
|
|
return closed
|
|
|
|
|
|
def _forget_failing(store: Store, target: str) -> None:
|
|
"""Drop what the pull recorded about a board nobody reads any more: a
|
|
record of `target` failing would otherwise outlive the board, and count
|
|
in `app companies check` for good. The `ats` plugin's robots.txt verdict
|
|
and its lists of roles read and failed in detail go with it, since they
|
|
are about that board alone and would only grow stale."""
|
|
store.state.pop(FAILING + target, None)
|
|
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
|
|
store.state.pop(key + target, None)
|
|
|
|
|
|
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
|
|
"""Follow one company by its ATS board: `set_board` keyed by `name`, so
|
|
a company that exists gets the board attached or replaced. Returns the
|
|
subscription and how many roles the board lists."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
sub, roles, _closed = set_board(store, company_key(name), board, name=name)
|
|
return sub, roles or 0
|
|
|
|
|
|
def add_discovered(store: Store, name: str, domain: str,
|
|
found: Discovery) -> tuple[str, int]:
|
|
"""Follow a company from what `app.jobs.discover` found. A readable
|
|
board goes through `set_board`, which attaches it, replaces the one the
|
|
company has, or creates the company. Otherwise the row records `ats`,
|
|
`ats_slug` and the careers page, as one the app cannot read yet; a
|
|
company that already has a board keeps it and this raises, since
|
|
discovery finding nothing readable is weaker evidence than a board that
|
|
answers. Returns the company key and how many roles its board lists (0
|
|
with no board)."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a company needs a name")
|
|
key = company_key(name)
|
|
if found.board:
|
|
_sub, roles, _closed = set_board(store, key, found.board, name=name, domain=domain)
|
|
return key, roles or 0
|
|
company = store.company(key)
|
|
if company is None:
|
|
store.add_company(name, key=key, domain=domain, ats=found.ats,
|
|
ats_slug=found.ats_slug, careers_url=found.careers_url)
|
|
return key, 0
|
|
sub = store.company_subscription(key)
|
|
if sub is not None and sub.show:
|
|
raise ValueError(f"{name} is read from {sub.target} and discovery found no "
|
|
"readable board; nothing changed")
|
|
# A link somebody typed is kept: discovery's is only a guess at the page.
|
|
store.update_company(key, ats=found.ats, ats_slug=found.ats_slug,
|
|
careers_url=company.careers_url or found.careers_url,
|
|
**({"domain": domain} if not company.domain else {}))
|
|
return key, 0
|
|
|
|
|
|
def known_from_search_only(company: Company, ever_boarded: set[str]) -> bool:
|
|
"""A row that exists only because a search listed its roles and somebody
|
|
judged it (`Store.set_affinity`): no board, not even a detached one
|
|
(`ever_boarded`, the keys of every `ats` subscription), and none of the
|
|
facts every way of following a company or listing it by hand writes."""
|
|
return not (company.key in ever_boarded or company.ats or company.careers_url
|
|
or company.domain or company.hand_reason)
|
|
|
|
|
|
def by_hand(store: Store) -> list[Company]:
|
|
"""The list checked by hand: companies with no board shown that are not
|
|
known from a search only. Never opened first, then the longest ago
|
|
(`careers_opened_at`, the Watchlist's stamp); the name breaks a tie."""
|
|
boards = store.company_boards()
|
|
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
|
|
rows = [c for c in store.companies()
|
|
if c.key not in boards and not known_from_search_only(c, ever_boarded)]
|
|
return sorted(rows, key=lambda c: (c.careers_opened_at, c.name.casefold()))
|
|
|
|
|
|
class Conflict(ValueError):
|
|
"""A change refused because of what the company already is, not because of
|
|
what was asked for: the pages answer it with 409, not 422."""
|
|
|
|
|
|
def check_careers_url(url: str) -> str:
|
|
"""`url` stripped; ValueError unless it is empty or an http(s) link. The
|
|
page puts it in an `href`, where `javascript:` would run."""
|
|
url = url.strip()
|
|
if url:
|
|
parts = urlsplit(url)
|
|
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
raise ValueError("The careers link must start with http:// or https://.")
|
|
return url
|
|
|
|
|
|
def add_by_hand(store: Store, name: str, *, careers_url: str = "", notes: str = "",
|
|
hand_reason: str = "added") -> Company:
|
|
"""A company with no board, put on the list checked by hand. Raises
|
|
Conflict when the key its name folds to is taken, ValueError for a name
|
|
with no letters, a link that is no http(s) URL or a reason outside
|
|
`HAND_REASONS`."""
|
|
link = check_careers_url(careers_url)
|
|
key = company_key(name)
|
|
if key and store.company(key) is not None:
|
|
raise Conflict(f"{name.strip()} is on the list already as {key}.")
|
|
return store.add_company(name, key=key, careers_url=link, notes=notes,
|
|
hand_reason=hand_reason)
|
|
|
|
|
|
def edit_company(store: Store, key: str, **fields: str) -> Company:
|
|
"""Change a company's `name`, `careers_url`, `notes` or `hand_reason`; the
|
|
key never changes. A new name is Conflict for a company with a board
|
|
subscription, shown or not, since the subscription is found by the key
|
|
its company name folds to and would be lost. Raises KeyError for an
|
|
unknown company and ValueError for a blank name, a link that is no http(s)
|
|
URL or a reason outside `HAND_REASONS`."""
|
|
unknown = set(fields) - {"name", "careers_url", "notes", "hand_reason"}
|
|
if unknown:
|
|
raise ValueError(f"not a field to edit: {', '.join(sorted(unknown))}")
|
|
company = store.company(key)
|
|
if company is None:
|
|
raise KeyError(key)
|
|
if "careers_url" in fields:
|
|
fields["careers_url"] = check_careers_url(fields["careers_url"])
|
|
if "name" in fields:
|
|
fields["name"] = fields["name"].strip()
|
|
if not fields["name"]:
|
|
raise ValueError("A company needs a name.")
|
|
if fields["name"] != company.name and store.company_subscription(key) is not None:
|
|
raise Conflict(f"{company.name} has a board, which is found by its name; "
|
|
"detach the board first.")
|
|
return store.update_company(key, **fields)
|
|
|
|
|
|
def remove_company(store: Store, key: str) -> None:
|
|
"""Delete company `key` and every agent's stance on it. Raises KeyError
|
|
for an unknown company and Conflict for one still read: a company with
|
|
a board on (`show`, as `detach_board` decides) or with open roles, which
|
|
would otherwise vanish from the list with nothing said. A detached board
|
|
is no board, since nothing reads it and its roles are closed; its
|
|
subscription row stays, and follows the company if it comes back."""
|
|
if store.company(key) is None:
|
|
raise KeyError(key)
|
|
sub = store.company_subscription(key)
|
|
if sub is not None and sub.show:
|
|
raise Conflict(f"{key} is read from {sub.target}; detach the board first")
|
|
if store.open_role_counts().get(key):
|
|
raise Conflict(f"{key} has open roles; detach its board first")
|
|
store.remove_company(key)
|
|
|
|
|
|
def _fill_empty(store: Store, key: str, **facts: str) -> None:
|
|
"""Write each of `facts` to company `key` where the stored value is
|
|
empty: what a seed may add to a row that exists, since a person's link or
|
|
reason is never overwritten."""
|
|
company = store.company(key)
|
|
assert company is not None
|
|
fill = {f: v for f, v in facts.items() if v and not getattr(company, f)}
|
|
if fill:
|
|
store.update_company(key, **fill)
|
|
|
|
|
|
def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
|
|
"""Import `seed/companies.json`: one `companies` row per entry, and its
|
|
`board` through `set_board` where the entry names one. An entry without
|
|
a board is the point of the list as much as one with: a company whose
|
|
careers site the app cannot read yet keeps its `ats` host, `ats_slug`
|
|
and `careers_url`, and the Companies section says so.
|
|
|
|
Re-running adds what is missing: a new entry becomes a row, and an
|
|
existing row that never had a board gets the one the seed now names, so a
|
|
seed update that gains boards reaches a database made before it. A row's
|
|
board, once it has one (detached counts), and its affinity, notes and
|
|
status are never touched; its `careers_url` and `hand_reason` are filled
|
|
where empty, so the seed can hand a row its link and its reason but never
|
|
overwrite the one a person set. The seed's boards are not fetched again:
|
|
they were read once when the seed was written, and `app companies check`
|
|
finds the dead ones. Returns counts, and `problems`, one line per entry
|
|
that could not land whole."""
|
|
from plugins.ats import is_known_target
|
|
|
|
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
entries = data.get("companies", [])
|
|
added = boards = skipped = 0
|
|
problems: list[str] = []
|
|
for row in entries:
|
|
name = row["name"].strip()
|
|
key = row.get("key") or company_key(name)
|
|
board = (row.get("board") or "").strip()
|
|
if board and not is_known_target(board):
|
|
problems.append(f"{key}: {board!r} is not a known board; kept without it")
|
|
board = ""
|
|
facts = {f: str(row.get(f) or "") for f in ("domain", "country", "city",
|
|
"careers_url", "ats", "ats_slug")}
|
|
reason = str(row.get("hand_reason") or "")
|
|
if reason not in HAND_REASONS:
|
|
problems.append(f"{key}: {reason!r} is not a hand_reason; kept without it")
|
|
reason = ""
|
|
exists = store.company(key) is not None
|
|
if exists:
|
|
_fill_empty(store, key, hand_reason=reason)
|
|
if exists and (not board or store.company_subscription(key) is not None):
|
|
_fill_empty(store, key, careers_url=facts["careers_url"])
|
|
skipped += 1
|
|
continue
|
|
if board:
|
|
try:
|
|
set_board(store, key, board, name=name, check=False,
|
|
**{f: v for f, v in facts.items() if f not in ("ats", "ats_slug")})
|
|
boards += 1
|
|
added += not exists
|
|
continue
|
|
except ValueError as exc:
|
|
problems.append(f"{key}: {exc}")
|
|
if exists:
|
|
_fill_empty(store, key, careers_url=facts["careers_url"])
|
|
skipped += 1
|
|
continue
|
|
# No board, or one that clashed: the company is still a row, and a
|
|
# clashing board's host and slug are what discovery found.
|
|
if board:
|
|
facts["ats"], _, facts["ats_slug"] = board.partition(":")
|
|
store.add_company(name, key=key, hand_reason=reason, **facts)
|
|
added += 1
|
|
return {"added": added, "boards": boards, "skipped": skipped, "total": len(entries),
|
|
"problems": problems}
|
|
|
|
|
|
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
|
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
|
|
"""The `params` a search subscription stores, with only the keys its
|
|
provider reads."""
|
|
if provider not in PROVIDERS:
|
|
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
|
|
if not q.strip():
|
|
raise ValueError("a search needs a query")
|
|
params: dict[str, Any] = {"q": q.strip()}
|
|
if provider == "jobindex" and area:
|
|
params["area"] = int(area)
|
|
if provider == "adzuna":
|
|
params["country"] = country.strip().lower()
|
|
if where.strip():
|
|
params["where"] = where.strip()
|
|
if provider == "platsbanken":
|
|
from plugins.search import PLATSBANKEN_REGIONS
|
|
|
|
region = region.strip()
|
|
if region and region not in PLATSBANKEN_REGIONS:
|
|
raise ValueError("region is empty or the id of a Swedish län")
|
|
params["region"] = region
|
|
if pages:
|
|
params["pages"] = int(pages)
|
|
return params
|
|
|
|
|
|
def has_key(config: Config, provider: str) -> bool:
|
|
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
|
|
config or the environment."""
|
|
if provider != "adzuna":
|
|
return True
|
|
keys = (config.sources.get("search") or {}).get("adzuna") or {}
|
|
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
|
|
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
|
|
|
|
|
|
def try_search(config: Config, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
|
|
"""Run a search once, outside any pull. Raises ValueError with the
|
|
provider's own reason when it does not answer."""
|
|
from plugins.search import run_search
|
|
|
|
try:
|
|
return run_search(search_target(provider, name or "preview"), params,
|
|
dict(config.sources.get("search") or {}))
|
|
except ValueError:
|
|
raise
|
|
except Exception as exc:
|
|
raise ValueError(f"{provider} did not answer: {exc}") from exc
|
|
|
|
|
|
def save_search(config: Config, store: Store, provider: str, name: str,
|
|
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
|
|
"""Run a search once and save it only if it finds something, so a typo in a
|
|
query shows now and not as an empty tab later. A search of the same
|
|
provider and name is updated in place, removed or not: its roles keep
|
|
their subscription. Returns the subscription, how many roles it found, and
|
|
whether that was every match."""
|
|
name = name.strip()
|
|
if not name:
|
|
raise ValueError("a search needs a name")
|
|
rows, complete = try_search(config, provider, name, params)
|
|
if not rows:
|
|
raise ValueError("the search found nothing; not saved")
|
|
target = search_target(provider, name)
|
|
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
|
|
None)
|
|
if existing is None:
|
|
sub = store.add_subscription("search", target, label=name, board=provider,
|
|
params=params)
|
|
else:
|
|
sub = store.update_subscription(existing.id, label=name, params=params,
|
|
subscribed=True, fetch=True, show=True)
|
|
return sub, len(rows), complete
|
|
|
|
|
|
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
|
|
"""Stop running a search and hide it. Its open roles close; adding a search
|
|
of the same name again brings the row back."""
|
|
sub = store.subscription(sub_id)
|
|
if sub is None or sub.source != "search":
|
|
raise KeyError(sub_id)
|
|
store.unsubscribe(sub_id)
|
|
return sub, store.close_subscription(sub_id)
|