Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
+64
-381
@@ -5,421 +5,109 @@ roles at companies nobody has listed yet, by asking a cross-company board a
|
||||
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
||||
A subscription is one saved search; its `params` are the question.
|
||||
|
||||
Three boards, checked live on 2026-09-23 and, for Platsbanken, 2026-09-29:
|
||||
|
||||
jobindex Denmark's largest job board. Every search result page has an RSS
|
||||
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
||||
own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
||||
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
|
||||
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
|
||||
20 results a page.
|
||||
adzuna An aggregator with a documented API for most of Europe, the UK
|
||||
and the US, but not Denmark (adzuna.dk does not exist). Needs a
|
||||
free key from developer.adzuna.com, as `sources.search.adzuna:
|
||||
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
||||
against Adzuna's documented response and not yet run live.
|
||||
platsbanken Sweden's public employment service, through the JobTech search
|
||||
API at jobsearch.api.jobtechdev.se, no key needed. `region` is
|
||||
the id of a län in the JobTech taxonomy (Skåne län is
|
||||
`CaRE_1nn_cSU`), empty for all of Sweden. 100 results a page,
|
||||
and an `offset` above 2000 is refused. It states `total`. It has
|
||||
no remote flag worth reading (`workplace_model` is mostly
|
||||
empty), so `remote` comes from the headline.
|
||||
|
||||
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
||||
browser session of its own site) and Jooble (a key needs a manual
|
||||
application, and its API host answered a Cloudflare challenge).
|
||||
The providers are the readers of `scope="search"` in `plugins/readers/`
|
||||
(`readers.PROVIDERS`), read by `plugins/readers/_runner.py` like any other
|
||||
reader: the search's params go in `Ask.params`, a declared key is resolved
|
||||
from `sources.search.<name>` or the environment and refused when missing,
|
||||
and the saved `pages` param bounds the pages read (`DEFAULT_PAGES`, at most
|
||||
`MAX_PAGES`).
|
||||
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that read everything that matches was
|
||||
complete, and the pull closes what it stopped listing; the rest close once no
|
||||
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna and
|
||||
Platsbanken state how many results match, so their searches are complete when
|
||||
they read that many; Jobindex states no count, so a Jobindex search is
|
||||
complete when a page within the page limit came back short.
|
||||
complete (`complete="stated"`, as many distinct results as the stated total),
|
||||
and the pull closes what it stopped listing; the rest close once no search
|
||||
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
import sys
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks,
|
||||
)
|
||||
|
||||
try:
|
||||
from plugins import readers
|
||||
except ModuleNotFoundError:
|
||||
# Loaded by path by `desk.sources.registry`; see the same lines in `ats.py`.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import readers
|
||||
from plugins.readers import _runner
|
||||
from plugins.readers._shared import Row
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT = 15.0
|
||||
|
||||
#: Pages fetched per search unless its params say otherwise, and the most a
|
||||
#: Pages read per search unless its params say otherwise, and the most a
|
||||
#: search may ask for. A search that needs more than ten pages is too broad to
|
||||
#: read anyway.
|
||||
DEFAULT_PAGES = 5
|
||||
MAX_PAGES = 10
|
||||
|
||||
PROVIDERS = ("jobindex", "adzuna", "platsbanken")
|
||||
|
||||
#: What every provider turns a result into.
|
||||
Row = dict[str, Any]
|
||||
#: The pause between two pages of one search.
|
||||
PAGE_DELAY = _runner.PAGE_DELAY
|
||||
|
||||
|
||||
def _web_url(url: str) -> bool:
|
||||
parsed = httpx.URL((url or "").strip())
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def _pages(params: dict[str, Any]) -> int:
|
||||
def _pages(params: Mapping[str, Any]) -> int:
|
||||
try:
|
||||
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_PAGES
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
|
||||
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- jobindex
|
||||
|
||||
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
||||
JOBINDEX_PAGE = 20
|
||||
|
||||
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
||||
#: against the titles its feeds answer with. 0 is all of Denmark.
|
||||
JOBINDEX_AREAS: dict[int, str] = {
|
||||
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
||||
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
||||
8: "Aarhus", 7: "Aalborg",
|
||||
}
|
||||
|
||||
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
||||
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
||||
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
||||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
|
||||
|
||||
#: What a Danish ad's town line says about where the work is done:
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde|distansarbete|på distans", re.IGNORECASE)
|
||||
|
||||
|
||||
def _remote(text: str) -> str | None:
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||||
zero-width space, which would otherwise end up in `title_key`."""
|
||||
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
|
||||
|
||||
|
||||
def parse_jobindex(body: bytes) -> list[Row]:
|
||||
return _jobindex_page(body)[0]
|
||||
|
||||
|
||||
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
||||
"""One page of a Jobindex RSS search, and how many items it had before
|
||||
any was dropped. An item is `<title>` "Role, Company",
|
||||
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
||||
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
||||
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's
|
||||
last comma, or '' when there is none. A body with no `<channel>` is not
|
||||
a search result and raises, since an empty list would read as a search
|
||||
that ran out and close everything it listed before."""
|
||||
root = ET.fromstring(body)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("jobindex answered without an RSS channel")
|
||||
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
search_area = area_match.group(1).strip() if area_match else ""
|
||||
rows = []
|
||||
items = channel.findall("item")
|
||||
for item in items:
|
||||
link = (item.findtext("link") or "").strip()
|
||||
full_title = (item.findtext("title") or "").strip()
|
||||
if not full_title or not _web_url(link):
|
||||
continue
|
||||
description = item.findtext("description") or ""
|
||||
h4 = _H4.search(description)
|
||||
title = _text(h4.group(1)) if h4 else full_title
|
||||
alt = _ALT.search(description)
|
||||
company = html.unescape(alt.group(1)).strip() if alt else ""
|
||||
if not company and full_title.startswith(title + ", "):
|
||||
company = full_title[len(title) + 2:].strip()
|
||||
area = _AREA.search(description)
|
||||
posted = None
|
||||
if pub := item.findtext("pubDate"):
|
||||
try:
|
||||
posted = parsedate_to_datetime(pub)
|
||||
except (TypeError, ValueError):
|
||||
posted = None
|
||||
place = _text(area.group(1)) if area else search_area
|
||||
rows.append({
|
||||
"id": link.rstrip("/").rsplit("/", 1)[-1],
|
||||
"title": title or full_title,
|
||||
"company": company,
|
||||
"place": place,
|
||||
"country": "DK",
|
||||
"remote": _remote(f"{place} {title}"),
|
||||
"url": link,
|
||||
"posted": posted,
|
||||
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows, len(items)
|
||||
|
||||
|
||||
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page or the page
|
||||
limit. Returns the rows and whether the results ran out: a page with
|
||||
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
||||
a dropped item does not pass for the end. Jobindex states no total, so a
|
||||
search that read the page limit full is never complete."""
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
||||
if params.get("area"):
|
||||
query["geoareaid"] = params["area"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
rows += got
|
||||
if items < JOBINDEX_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ adzuna
|
||||
|
||||
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
||||
ADZUNA_PAGE = 50
|
||||
|
||||
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
||||
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
||||
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
||||
|
||||
|
||||
def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
||||
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
||||
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
||||
return _adzuna_rows(json.loads(body), country)
|
||||
|
||||
|
||||
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
||||
rows = []
|
||||
for job in data.get("results") or []:
|
||||
title = _text(job.get("title") or "")
|
||||
url = (job.get("redirect_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||
rows.append({
|
||||
"id": str(job.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
||||
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
|
||||
"country": country.upper(),
|
||||
"remote": _remote(title),
|
||||
"url": url,
|
||||
"posted": _parse_iso(job.get("created")),
|
||||
"description": _text(job.get("description") or ""),
|
||||
"salary_min": None if predicted else job.get("salary_min"),
|
||||
"salary_max": None if predicted else job.get("salary_max"),
|
||||
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, until a short page, the stated `count` or
|
||||
the page limit. Returns the rows and whether they are all `count` says
|
||||
match, or, on an answer with no `count`, whether a page came back short."""
|
||||
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
||||
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
||||
if not (app_id and app_key):
|
||||
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
|
||||
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
|
||||
country = str(params.get("country") or "").lower()
|
||||
if country not in ADZUNA_COUNTRIES:
|
||||
raise ValueError(f"adzuna country {country!r} is not one of "
|
||||
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
|
||||
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
|
||||
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
|
||||
"what": str(params.get("q") or "")}
|
||||
if params.get("where"):
|
||||
query["where"] = params["where"]
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
count: Any = None
|
||||
ran_out = False
|
||||
for page in range(1, _pages(params) + 1):
|
||||
url = ADZUNA_SEARCH.format(country=country, page=page)
|
||||
data = json.loads(_get(client, url, query))
|
||||
results = data.get("results") or []
|
||||
count = data.get("count", count)
|
||||
seen |= {str(job.get("id") or "") for job in results}
|
||||
rows += _adzuna_rows(data, country)
|
||||
if len(results) < ADZUNA_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(count, int) and len(seen) >= count:
|
||||
break
|
||||
# Adzuna states how many results match; the search is complete when it
|
||||
# read that many distinct ones. Without a count, a short page is the end.
|
||||
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
||||
|
||||
|
||||
PLATSBANKEN_SEARCH = "https://jobsearch.api.jobtechdev.se/search"
|
||||
PLATSBANKEN_PAGE = 100
|
||||
#: The API refuses an `offset` above 2000, which no page limit reaches.
|
||||
PLATSBANKEN_MAX_OFFSET = 2000
|
||||
|
||||
#: The 21 Swedish län as concept ids of the JobTech taxonomy, from its `region`
|
||||
#: concepts (which also list regions outside Sweden). "" is all of Sweden and
|
||||
#: is not a key here.
|
||||
PLATSBANKEN_REGIONS: dict[str, str] = {
|
||||
"DQZd_uYs_oKb": "Blekinge län", "oDpK_oZ2_WYt": "Dalarnas län",
|
||||
"K8iD_VQv_2BA": "Gotlands län", "zupA_8Nt_xcD": "Gävleborgs län",
|
||||
"wjee_qH2_yb6": "Hallands län", "65Ms_7r1_RTG": "Jämtlands län",
|
||||
"MtbE_xWT_eMi": "Jönköpings län", "9QUH_2bb_6Np": "Kalmar län",
|
||||
"tF3y_MF9_h5G": "Kronobergs län", "9hXe_F4g_eTG": "Norrbottens län",
|
||||
"CaRE_1nn_cSU": "Skåne län", "CifL_Rzy_Mku": "Stockholms län",
|
||||
"s93u_BEb_sx2": "Södermanlands län", "zBon_eET_fFU": "Uppsala län",
|
||||
"EVVp_h6U_GSZ": "Värmlands län", "g5Tt_CAV_zBd": "Västerbottens län",
|
||||
"NvUF_SP1_1zo": "Västernorrlands län", "G6DV_fKE_Viz": "Västmanlands län",
|
||||
"zdoY_6u5_Krt": "Västra Götalands län", "xTCk_nT5_Zjm": "Örebro län",
|
||||
"oLT3_Q9p_3nn": "Östergötlands län",
|
||||
}
|
||||
|
||||
|
||||
def parse_platsbanken(body: bytes) -> list[Row]:
|
||||
"""One page of JobTech's `/search`: `{"total": {"value"}, "hits": [...]}`.
|
||||
A hit's `webpage_url` is Platsbanken's own page for the ad. `remote` is
|
||||
read from the headline and from `workplace_model`, which the ads mostly
|
||||
leave empty."""
|
||||
return _platsbanken_rows(json.loads(body))
|
||||
|
||||
|
||||
def _platsbanken_rows(data: dict[str, Any]) -> list[Row]:
|
||||
rows = []
|
||||
for hit in data.get("hits") or []:
|
||||
title = _text(hit.get("headline") or "")
|
||||
url = (hit.get("webpage_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
address = hit.get("workplace_address") or {}
|
||||
model = (hit.get("workplace_model") or {}).get("label") or ""
|
||||
rows.append({
|
||||
"id": str(hit.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((hit.get("employer") or {}).get("name") or "").strip(),
|
||||
"place": (address.get("municipality") or address.get("region") or "").strip(),
|
||||
"country": "SE",
|
||||
"remote": _remote(f"{title} {model}"),
|
||||
"url": url,
|
||||
"posted": _parse_iso(hit.get("publication_date")),
|
||||
"description": _text((hit.get("description") or {}).get("text") or ""),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def search_platsbanken(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page, the stated
|
||||
total or the page limit. Returns the rows and whether they are all the
|
||||
`total` says match, as for Adzuna. `region` is a key of
|
||||
`PLATSBANKEN_REGIONS`, or empty for all of Sweden."""
|
||||
region = str(params.get("region") or "")
|
||||
if region and region not in PLATSBANKEN_REGIONS:
|
||||
raise ValueError(f"platsbanken region {region!r} is not one of the ids in "
|
||||
"PLATSBANKEN_REGIONS")
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "pubdate-desc",
|
||||
"limit": PLATSBANKEN_PAGE}
|
||||
if region:
|
||||
query["region"] = region
|
||||
rows: list[Row] = []
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
ran_out = False
|
||||
for page in range(_pages(params)):
|
||||
offset = page * PLATSBANKEN_PAGE
|
||||
if offset > PLATSBANKEN_MAX_OFFSET:
|
||||
break
|
||||
data = json.loads(_get(client, PLATSBANKEN_SEARCH, query | {"offset": offset}))
|
||||
hits = data.get("hits") or []
|
||||
total = (data.get("total") or {}).get("value", total)
|
||||
seen |= {str(hit.get("id") or "") for hit in hits}
|
||||
rows += _platsbanken_rows(data)
|
||||
if len(hits) < PLATSBANKEN_PAGE:
|
||||
ran_out = True
|
||||
break
|
||||
if isinstance(total, int) and len(seen) >= total:
|
||||
break
|
||||
return rows, (len(seen) == total) if isinstance(total, int) else ran_out
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
tuple[list[Row], bool]]] = {
|
||||
"jobindex": search_jobindex,
|
||||
"adzuna": search_adzuna,
|
||||
"platsbanken": search_platsbanken,
|
||||
}
|
||||
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
|
||||
|
||||
|
||||
def provider_of(target: str) -> str:
|
||||
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
||||
provider, sep, name = target.partition(":")
|
||||
return provider if sep and name.strip() and provider in _SEARCHES else ""
|
||||
return provider if sep and name.strip() and provider in readers.PROVIDERS else ""
|
||||
|
||||
|
||||
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
||||
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
|
||||
"""One search read through the runner: its provider and its listing.
|
||||
Raises ValueError for an unknown provider or a missing key, before any
|
||||
request."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
f"{', '.join(readers.PROVIDERS)}")
|
||||
module = readers.PROVIDERS[provider]
|
||||
slug = target.partition(":")[2]
|
||||
ask = _runner.ask_for(module, params, keys=keys)
|
||||
get = _runner.getter(client, module.allowed(slug), provider)
|
||||
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
|
||||
|
||||
|
||||
def run_search(target: str, params: dict[str, Any],
|
||||
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
||||
"""One search, outside any pull: for `app searches add`, which runs a
|
||||
search once before saving it."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
f"{', '.join(PROVIDERS)}")
|
||||
search once before saving it. `keys` is the config's `sources.search`.
|
||||
Each row also carries `place`, the display line of its places."""
|
||||
with httpx.Client() as client:
|
||||
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
|
||||
_module, listing = _read(client, target, params, keys)
|
||||
rows = [row | {"place": _runner.raw_of(row)["place"]} for row in listing.rows]
|
||||
return rows, listing.complete
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class SearchSource(Source):
|
||||
name = "search"
|
||||
description = "Saved searches on cross-company job boards: Jobindex, Adzuna, Platsbanken."
|
||||
description = "Saved searches on cross-company job boards: " + ", ".join(
|
||||
module.LABEL for module in readers.PROVIDERS.values()) + "."
|
||||
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
||||
subscription_field = "searches"
|
||||
|
||||
@@ -447,37 +135,32 @@ class SearchSource(Source):
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
_module, listing = _read(client, target, self.params_for(target),
|
||||
self.options)
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
for row in listing.rows:
|
||||
source_id = f"{provider}:{row['source_id']}"
|
||||
seen_ids.append(source_id)
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": row.get("company") or "",
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
Reference in new issue
Block a user