Files
jobboard/plugins/search.py
T
WolfandClaude Opus 5.5 902089a5e2 Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole,
paged, one series per filter value unioned by id), completeness from READS,
the sweep of a board over the page cap across pulls, the 10% guard on a
filtered or empty="suspect" read, facets kept a day, the detail reads, the
raw an Item carries and the failure bookkeeping out of plugins/ats.py.
Ask comes from a subscription's params (ask_for, open for the reach), with
a declared key resolved into Ask.params["key"] and refused when missing.

The runtime robots.txt check goes (decided at build time, READS.robots).
plugins/search.py reads the providers of readers.PROVIDERS through the
runner and applies the saved pages param. A filtered board's close keeps
roles with a vote or an application. DETAIL_FIELDS gain places, teaser,
country and employment.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 09:28:33 +02:00

167 lines
6.7 KiB
Python

"""Job searches: a query against a job board that spans companies.
`ats` reads the boards of companies someone already chose. This plugin finds
roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question.
The providers are the readers of `scope="search"` in `plugins/readers/`
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
reader: the search's params go in `Ask.params`, a declared key is resolved
from `sources.search.<name>` or the environment and refused when missing,
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
`MAX_PAGES`).
A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that read everything that matches was
complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
"""
from __future__ import annotations
import json
import logging
import sys
from collections.abc import Iterator, Mapping
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
)
try:
from plugins import readers
except ModuleNotFoundError:
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
from plugins.readers import _runner
from plugins.readers._shared import Row
log = logging.getLogger(__name__)
#: Pages read per search unless its params say otherwise, and the most a
#: search may ask for. A search that needs more than ten pages is too broad to
#: read anyway.
DEFAULT_PAGES = 5
MAX_PAGES = 10
#: The pause between two pages of one search.
PAGE_DELAY = _runner.PAGE_DELAY
def _pages(params: Mapping[str, Any]) -> int:
try:
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
except (TypeError, ValueError):
return DEFAULT_PAGES
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
def provider_of(target: str) -> str:
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
provider, sep, name = target.partition(":")
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any
request."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
f"{', '.join(readers.PROVIDERS)}")
module = readers.PROVIDERS[provider]
slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider)
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
def run_search(target: str, params: dict[str, Any],
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
"""One search, outside any pull: for `app searches add`, which runs a
search once before saving it. `keys` is the config's `sources.search`.
Each row also carries `place`, the display line of its places."""
with httpx.Client() as client:
_module, listing = _read(client, target, params, keys)
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
return rows, listing.complete
# -------------------------------------------------------------- the plugin
class SearchSource(Source):
name = "search"
description = "Saved searches on cross-company job boards: " + ", ".join(
module.LABEL for module in readers.PROVIDERS.values()) + "."
#: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Job searches",
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# The pull run's marker, handed in by its batch as for `ats`.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
provider = provider_of(target)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
_module, listing = _read(client, target, self.params_for(target),
self.options)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
seen_ids = []
for row in listing.rows:
source_id = f"{provider}:{row['source_id']}"
seen_ids.append(source_id)
description = row.get("description") or ""
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)