"""Job searches: a query against a job board that spans companies. `ats` reads the boards of companies someone already chose. This plugin finds roles at companies nobody has listed yet, by asking a cross-company board a saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. A subscription is one saved search; its `params` are the question. The providers are the readers of `scope="search"` in `plugins/readers/` (`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other reader: the search's params go in `Ask.params`, a declared key is resolved from `sources.search.` or the environment and refused when missing, and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most `MAX_PAGES`). A search result list is never a board's complete list, so a role missing from one run is not closed here. A search that read everything that matches was complete (`complete="stated"`, as many distinct results as the stated total), and the pull closes what it stopped listing; the rest close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). In a pull a search is read by `_runner.read` under the prefix `search`, as a board is under `ats`: the 10% guard holds it, `_runner.stamp` records it, and a search that fails or is suspect is kept as failing under `search:failing:` (`_runner.record_failure`) until it answers, which the settle reads as it reads a failing board's. """ from __future__ import annotations import logging import sys from collections.abc import Iterator, Mapping, MutableMapping from datetime import UTC, datetime from pathlib import Path from types import ModuleType from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, ) try: from plugins import readers except ModuleNotFoundError: # Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from plugins import readers from plugins.readers import _runner from plugins.readers._shared import Row log = logging.getLogger(__name__) #: Pages read per search unless its params say otherwise, and the most a #: search may ask for. A search that needs more than ten pages is too broad to #: read anyway. DEFAULT_PAGES = 5 MAX_PAGES = 10 #: The pause between two pages of one search. PAGE_DELAY = _runner.PAGE_DELAY def _pages(params: Mapping[str, Any]) -> int: try: return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES)) except (TypeError, ValueError): return DEFAULT_PAGES def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits: return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY, stale_days=stale_days) def provider_of(target: str) -> str: """`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one.""" provider, sep, name = target.partition(":") return provider if sep and name.strip() and provider in readers.PROVIDERS else "" def _read(client: httpx.Client, target: str, params: Mapping[str, Any], keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None, stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]: """One search read through the runner: its provider and its listing. Raises ValueError for an unknown provider or a missing key, before any request. With `state` (a pull) it is `_runner.read` under `search`, the 10% guard included and never swept; without (a search tried before it is saved) a bare `read_listing`, since there is nothing to compare with.""" provider = provider_of(target) if not provider: raise ValueError(f"{target!r} is not provider:name with provider one of " f"{', '.join(readers.PROVIDERS)}") module = readers.PROVIDERS[provider] slug = target.partition(":")[2] ask = _runner.ask_for(module, params, keys=keys) get = _runner.getter(client, module.allowed(slug), provider) limits = _limits(params, stale_days) if state is None: return module, _runner.read_listing(module, slug, get, ask, limits=limits) return module, _runner.read(state, "search", target, module, slug, get, ask, limits=limits, can_sweep=False) def run_search(target: str, params: dict[str, Any], keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]: """One search, outside any pull: for `app searches add`, which runs a search once before saving it. `keys` is the config's `sources.search`. Each row also carries `place`, the display line of its places.""" with httpx.Client() as client: _module, listing = _read(client, target, params, keys) rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows] return rows, listing.complete # -------------------------------------------------------------- the plugin class SearchSource(Source): name = "search" description = "Saved searches on cross-company job boards: " + ", ".join( module.LABEL for module in readers.PROVIDERS.values()) + "." #: Entries are `{"url": target, "label": name, "params": {...}}`. subscription_field = "searches" spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="Job searches", targets=TargetSpec(mode="editable", kind="search", defaults=[]), profiles={"default": Profile( kind="role", tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy(enabled=False), ), presentation=ARTICLE_READER, )}, ) def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # The pull run's marker, handed in by its batch as for `ats`. run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): yield from self._search(client, target, label, run_marker) def _search(self, client: httpx.Client, target: str, label: str, run_marker: str) -> Iterator[Item]: """One search: read, its rows out as items, then stamped. A search that fails, a suspect answer included, is recorded under `search:failing:` and stamps nothing; the record goes once a read is stamped, after every row went out, as for a board.""" failing = f"search:failing:{target}" provider = provider_of(target) stale_days = self.options.get("stale_days") try: _module, listing = _read(client, target, self.params_for(target), self.options, self.state, int(stale_days) if stale_days else None) except Exception as exc: log.warning("search: %s failed (%s)", target, exc) _runner.record_failure(self.state, failing, exc) return now = datetime.now(UTC) for row in listing.rows: description = row.get("description") or "" yield Item( source_id=f"{provider}:{row['source_id']}", location=row["url"], title=row["title"], excerpt=description[:300], body=description, published_at=row.get("posted") or now, subscription=label, kind="role", raw=_runner.raw_of(row) | { "company": row.get("company") or "", "provider": provider, "search": target, }, ) _runner.stamp(self.state, "search", target, run_marker, listing, [f"{provider}:{i}" for i in listing.ids]) self.state.pop(failing, None)