186 lines
7.9 KiB
Python
186 lines
7.9 KiB
Python
"""Job searches: a query against a job board that spans companies.
|
|
|
|
`ats` reads the boards of companies someone already chose. This plugin finds
|
|
roles at companies nobody has listed yet, by asking a cross-company board a
|
|
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
|
A subscription is one saved search; its `params` are the question.
|
|
|
|
The providers are the readers of `scope="search"` in `plugins/readers/`
|
|
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
|
|
reader: the search's params go in `Ask.params`, a declared key is resolved
|
|
from `sources.search.<name>` or the environment and refused when missing,
|
|
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
|
|
`MAX_PAGES`).
|
|
|
|
A search result list is never a board's complete list, so a role missing from
|
|
one run is not closed here. A search that read everything that matches was
|
|
complete (`complete="stated"`, as many distinct results as the stated total),
|
|
and the pull closes what it stopped listing; the rest close once no search
|
|
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
|
|
|
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
|
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
|
and a search that fails or is suspect is kept as failing under
|
|
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
|
the settle reads as it reads a failing board's.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import sys
|
|
from collections.abc import Iterator, Mapping, MutableMapping
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from types import ModuleType
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
|
|
)
|
|
|
|
try:
|
|
from plugins import readers
|
|
except ModuleNotFoundError:
|
|
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
from plugins.readers import _runner
|
|
from plugins.readers._shared import Row
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: Pages read per search unless its params say otherwise, and the most a
|
|
#: search may ask for. A search that needs more than ten pages is too broad to
|
|
#: read anyway.
|
|
DEFAULT_PAGES = 5
|
|
MAX_PAGES = 10
|
|
|
|
#: The pause between two pages of one search.
|
|
PAGE_DELAY = _runner.PAGE_DELAY
|
|
|
|
|
|
def _pages(params: Mapping[str, Any]) -> int:
|
|
try:
|
|
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
|
except (TypeError, ValueError):
|
|
return DEFAULT_PAGES
|
|
|
|
|
|
def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
|
|
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
|
|
stale_days=stale_days)
|
|
|
|
|
|
def provider_of(target: str) -> str:
|
|
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
|
provider, sep, name = target.partition(":")
|
|
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
|
|
|
|
|
|
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
|
keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
|
|
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
|
|
"""One search read through the runner: its provider and its listing.
|
|
Raises ValueError for an unknown provider or a missing key, before any
|
|
request. With `state` (a pull) it is `_runner.read` under `search`, the
|
|
10% guard included and never swept; without (a search tried before it is
|
|
saved) a bare `read_listing`, since there is nothing to compare with."""
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
|
f"{', '.join(readers.PROVIDERS)}")
|
|
module = readers.PROVIDERS[provider]
|
|
slug = target.partition(":")[2]
|
|
ask = _runner.ask_for(module, params, keys=keys)
|
|
get = _runner.getter(client, module.allowed(slug), provider)
|
|
limits = _limits(params, stale_days)
|
|
if state is None:
|
|
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
|
|
return module, _runner.read(state, "search", target, module, slug, get, ask,
|
|
limits=limits, can_sweep=False)
|
|
|
|
|
|
def run_search(target: str, params: dict[str, Any],
|
|
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
|
"""One search, outside any pull: for `app searches add`, which runs a
|
|
search once before saving it. `keys` is the config's `sources.search`.
|
|
Each row also carries `place`, the display line of its places."""
|
|
with httpx.Client() as client:
|
|
_module, listing = _read(client, target, params, keys)
|
|
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
|
|
return rows, listing.complete
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class SearchSource(Source):
|
|
name = "search"
|
|
description = "Saved searches on cross-company job boards: " + ", ".join(
|
|
module.LABEL for module in readers.PROVIDERS.values()) + "."
|
|
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
|
subscription_field = "searches"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="Job searches",
|
|
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# The pull run's marker, handed in by its batch as for `ats`.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
yield from self._search(client, target, label, run_marker)
|
|
|
|
def _search(self, client: httpx.Client, target: str, label: str,
|
|
run_marker: str) -> Iterator[Item]:
|
|
"""One search: read, its rows out as items, then stamped. A search
|
|
that fails, a suspect answer included, is recorded under
|
|
`search:failing:<target>` and stamps nothing; the record goes once a
|
|
read is stamped, after every row went out, as for a board."""
|
|
failing = f"search:failing:{target}"
|
|
provider = provider_of(target)
|
|
stale_days = self.options.get("stale_days")
|
|
try:
|
|
_module, listing = _read(client, target, self.params_for(target), self.options,
|
|
self.state, int(stale_days) if stale_days else None)
|
|
except Exception as exc:
|
|
log.warning("search: %s failed (%s)", target, exc)
|
|
_runner.record_failure(self.state, failing, exc)
|
|
return
|
|
|
|
now = datetime.now(UTC)
|
|
for row in listing.rows:
|
|
description = row.get("description") or ""
|
|
yield Item(
|
|
source_id=f"{provider}:{row['source_id']}",
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted") or now,
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {
|
|
"company": row.get("company") or "",
|
|
"provider": provider,
|
|
"search": target,
|
|
},
|
|
)
|
|
_runner.stamp(self.state, "search", target, run_marker, listing,
|
|
[f"{provider}:{i}" for i in listing.ids])
|
|
self.state.pop(failing, None)
|