Files
jobboard/plugins/search.py
T

187 lines
7.9 KiB
Python

"""Job searches: a query against a job board that spans companies.
`ats` reads the boards of companies someone already chose. This plugin finds
roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question.
The providers are the readers of `scope="search"` in `plugins/readers/`
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
reader: the search's params go in `Ask.params`, a declared key is resolved
from `sources.search.<name>` or the environment and refused when missing,
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
`MAX_PAGES`).
A search result list is never a board's complete list, so a role missing from
one run is not closed, also when the search read everything that matches
(`complete="stated"`, as many distinct results as the stated total): its
roles close once no search has seen them for `STALE_DAYS` (see
`app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the runner drops a row without an id, a title or a
link, the 10% guards hold it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's.
"""
from __future__ import annotations
import logging
import sys
from collections.abc import Iterator, Mapping, MutableMapping
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
)
try:
from plugins import readers
except ModuleNotFoundError:
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
from plugins.readers import _runner
from plugins.readers._shared import Row
log = logging.getLogger(__name__)
#: Pages read per search unless its params say otherwise, and the most a
#: search may ask for. A search that needs more than ten pages is too broad to
#: read anyway.
DEFAULT_PAGES = 5
MAX_PAGES = 10
#: The pause between two pages of one search.
PAGE_DELAY = _runner.PAGE_DELAY
def _pages(params: Mapping[str, Any]) -> int:
try:
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
except (TypeError, ValueError):
return DEFAULT_PAGES
def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
stale_days=stale_days)
def provider_of(target: str) -> str:
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
provider, sep, name = target.partition(":")
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any
request. With `state` (a pull) it is `_runner.read` under `search`, the
10% guard included and never swept; without (a search tried before it is
saved) a bare `read_listing`, since there is nothing to compare with."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
f"{', '.join(readers.PROVIDERS)}")
module = readers.PROVIDERS[provider]
slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider)
limits = _limits(params, stale_days)
if state is None:
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
return module, _runner.read(state, "search", target, module, slug, get, ask,
limits=limits, can_sweep=False)
def run_search(target: str, params: dict[str, Any],
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
"""One search, outside any pull: for `app searches add`, which runs a
search once before saving it. `keys` is the config's `sources.search`.
Each row also carries `place`, the display line of its places."""
with httpx.Client() as client:
_module, listing = _read(client, target, params, keys)
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
return rows, listing.complete
# -------------------------------------------------------------- the plugin
class SearchSource(Source):
name = "search"
description = "Saved searches on cross-company job boards: " + ", ".join(
module.LABEL for module in readers.PROVIDERS.values()) + "."
#: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Job searches",
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# The pull run's marker, handed in by its batch as for `ats`.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
yield from self._search(client, target, label, run_marker)
def _search(self, client: httpx.Client, target: str, label: str,
run_marker: str) -> Iterator[Item]:
"""One search: read, its rows out as items, then stamped. A search
that fails, a suspect answer included, is recorded under
`search:failing:<target>` and stamps nothing; the record goes once a
read is stamped, after every row went out, as for a board."""
failing = f"search:failing:{target}"
provider = provider_of(target)
stale_days = self.options.get("stale_days")
try:
_module, listing = _read(client, target, self.params_for(target), self.options,
self.state, int(stale_days) if stale_days else None)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
now = datetime.now(UTC)
for row in listing.rows:
description = row.get("description") or ""
yield Item(
source_id=f"{provider}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or now,
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
_runner.stamp(self.state, "search", target, run_marker, listing,
[f"{provider}:{i}" for i in listing.ids])
self.state.pop(failing, None)