plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
167 lines
6.7 KiB
Python
167 lines
6.7 KiB
Python
"""Job searches: a query against a job board that spans companies.
|
|
|
|
`ats` reads the boards of companies someone already chose. This plugin finds
|
|
roles at companies nobody has listed yet, by asking a cross-company board a
|
|
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
|
A subscription is one saved search; its `params` are the question.
|
|
|
|
The providers are the readers of `scope="search"` in `plugins/readers/`
|
|
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
|
|
reader: the search's params go in `Ask.params`, a declared key is resolved
|
|
from `sources.search.<name>` or the environment and refused when missing,
|
|
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
|
|
`MAX_PAGES`).
|
|
|
|
A search result list is never a board's complete list, so a role missing from
|
|
one run is not closed here. A search that read everything that matches was
|
|
complete (`complete="stated"`, as many distinct results as the stated total),
|
|
and the pull closes what it stopped listing; the rest close once no search
|
|
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sys
|
|
from collections.abc import Iterator, Mapping
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
|
|
)
|
|
|
|
try:
|
|
from plugins import readers
|
|
except ModuleNotFoundError:
|
|
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
from plugins.readers import _runner
|
|
from plugins.readers._shared import Row
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Pages read per search unless its params say otherwise, and the most a
|
|
#: search may ask for. A search that needs more than ten pages is too broad to
|
|
#: read anyway.
|
|
DEFAULT_PAGES = 5
|
|
MAX_PAGES = 10
|
|
|
|
#: The pause between two pages of one search.
|
|
PAGE_DELAY = _runner.PAGE_DELAY
|
|
|
|
|
|
def _pages(params: Mapping[str, Any]) -> int:
|
|
try:
|
|
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
|
except (TypeError, ValueError):
|
|
return DEFAULT_PAGES
|
|
|
|
|
|
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
|
|
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
|
|
|
|
|
|
def provider_of(target: str) -> str:
|
|
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
|
provider, sep, name = target.partition(":")
|
|
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
|
|
|
|
|
|
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
|
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
|
|
"""One search read through the runner: its provider and its listing.
|
|
Raises ValueError for an unknown provider or a missing key, before any
|
|
request."""
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
|
f"{', '.join(readers.PROVIDERS)}")
|
|
module = readers.PROVIDERS[provider]
|
|
slug = target.partition(":")[2]
|
|
ask = _runner.ask_for(module, params, keys=keys)
|
|
get = _runner.getter(client, module.allowed(slug), provider)
|
|
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
|
|
|
|
|
|
def run_search(target: str, params: dict[str, Any],
|
|
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
|
"""One search, outside any pull: for `app searches add`, which runs a
|
|
search once before saving it. `keys` is the config's `sources.search`.
|
|
Each row also carries `place`, the display line of its places."""
|
|
with httpx.Client() as client:
|
|
_module, listing = _read(client, target, params, keys)
|
|
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
|
|
return rows, listing.complete
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class SearchSource(Source):
|
|
name = "search"
|
|
description = "Saved searches on cross-company job boards: " + ", ".join(
|
|
module.LABEL for module in readers.PROVIDERS.values()) + "."
|
|
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
|
subscription_field = "searches"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="Job searches",
|
|
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# The pull run's marker, handed in by its batch as for `ats`.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
log.warning("search: %r is not provider:name; skipped", target)
|
|
continue
|
|
try:
|
|
_module, listing = _read(client, target, self.params_for(target),
|
|
self.options)
|
|
except Exception as exc:
|
|
log.warning("search: %s failed (%s)", target, exc)
|
|
continue
|
|
|
|
seen_ids = []
|
|
for row in listing.rows:
|
|
source_id = f"{provider}:{row['source_id']}"
|
|
seen_ids.append(source_id)
|
|
description = row.get("description") or ""
|
|
yield Item(
|
|
source_id=source_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted"),
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {
|
|
"company": row.get("company") or "",
|
|
"provider": provider,
|
|
"search": target,
|
|
},
|
|
)
|
|
self.state[f"search:ok:{target}"] = run_marker
|
|
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
|
|
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|