Workable and SmartRecruiters page their lists, and both parsers read page one only, so the settle closed every role past it (Delivery Hero: 980 stated, 100 open). `_paged` follows nextPage and offset, and the plugin stamps `ats:complete` only when the distinct ids read equal the stated total; a 429 after page one keeps what was read and closes nothing. Adzuna completes on its `count`, Jobindex counts raw items and raises on an answer with no channel. A board whose fetch raises is recorded under `ats:failing:<target>` with the first failure's time and the latest reason; after STALE_DAYS the settle closes its roles, and its next answer reopens them. `/api/companies`, the Companies list and `app companies check` show it. Co-Authored-By: Claude Opus <noreply@anthropic.com>
385 lines
16 KiB
Python
385 lines
16 KiB
Python
"""Job searches: a query against a job board that spans companies.
|
|
|
|
`ats` reads the boards of companies someone already chose. This plugin finds
|
|
roles at companies nobody has listed yet, by asking a cross-company board a
|
|
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
|
A subscription is one saved search; its `params` are the question.
|
|
|
|
Two boards, both checked live on 2026-09-23:
|
|
|
|
jobindex Denmark's largest job board. Every search result page has an RSS
|
|
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
|
own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
|
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
|
|
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
|
|
20 results a page.
|
|
adzuna An aggregator with a documented API for most of Europe, the UK
|
|
and the US, but not Denmark (adzuna.dk does not exist). Needs a
|
|
free key from developer.adzuna.com, as `sources.search.adzuna:
|
|
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
|
against Adzuna's documented response and not yet run live.
|
|
|
|
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
|
browser session of its own site) and Jooble (a key needs a manual
|
|
application, and its API host answered a Cloudflare challenge).
|
|
|
|
A search result list is never a board's complete list, so a role missing from
|
|
one run is not closed here. A search that read everything that matches was
|
|
complete, and the pull closes what it stopped listing; the rest close once no
|
|
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
|
|
how many results match, so an Adzuna search is complete when it read that
|
|
many; Jobindex states no count, so a Jobindex search is complete when a page
|
|
within the page limit came back short.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
TIMEOUT = 15.0
|
|
|
|
#: Pages fetched per search unless its params say otherwise, and the most a
|
|
#: search may ask for. A search that needs more than ten pages is too broad to
|
|
#: read anyway.
|
|
DEFAULT_PAGES = 5
|
|
MAX_PAGES = 10
|
|
|
|
PROVIDERS = ("jobindex", "adzuna")
|
|
|
|
#: What every provider turns a result into.
|
|
Row = dict[str, Any]
|
|
|
|
|
|
def _web_url(url: str) -> bool:
|
|
parsed = httpx.URL((url or "").strip())
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
def _pages(params: dict[str, Any]) -> int:
|
|
try:
|
|
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
|
except (TypeError, ValueError):
|
|
return DEFAULT_PAGES
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
|
|
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT})
|
|
response.raise_for_status()
|
|
return response.content
|
|
|
|
|
|
# ---------------------------------------------------------------- jobindex
|
|
|
|
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
|
JOBINDEX_PAGE = 20
|
|
|
|
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
|
#: against the titles its feeds answer with. 0 is all of Denmark.
|
|
JOBINDEX_AREAS: dict[int, str] = {
|
|
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
|
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
|
8: "Aarhus", 7: "Aalborg",
|
|
}
|
|
|
|
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
|
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
|
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
|
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
|
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
|
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
|
|
|
|
|
#: What a Danish ad's town line says about where the work is done:
|
|
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
|
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
|
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
|
|
|
|
|
|
def _remote(text: str) -> str | None:
|
|
if _REMOTE.search(text):
|
|
return "remote"
|
|
return "hybrid" if _HYBRID.search(text) else None
|
|
|
|
|
|
def _text(fragment: str) -> str:
|
|
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
|
zero-width space, which would otherwise end up in `title_key`."""
|
|
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
|
|
|
|
|
|
def parse_jobindex(body: bytes) -> list[Row]:
|
|
return _jobindex_page(body)[0]
|
|
|
|
|
|
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
|
"""One page of a Jobindex RSS search, and how many items it had before
|
|
any was dropped. An item is `<title>` "Role, Company",
|
|
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
|
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
|
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
|
("robot job") has no logo, so its company is what follows the title's
|
|
last comma, or '' when there is none. A body with no `<channel>` is not
|
|
a search result and raises, since an empty list would read as a search
|
|
that ran out and close everything it listed before."""
|
|
root = ET.fromstring(body)
|
|
channel = root.find("channel")
|
|
if channel is None:
|
|
raise ValueError("jobindex answered without an RSS channel")
|
|
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
|
search_area = area_match.group(1).strip() if area_match else ""
|
|
rows = []
|
|
items = channel.findall("item")
|
|
for item in items:
|
|
link = (item.findtext("link") or "").strip()
|
|
full_title = (item.findtext("title") or "").strip()
|
|
if not full_title or not _web_url(link):
|
|
continue
|
|
description = item.findtext("description") or ""
|
|
h4 = _H4.search(description)
|
|
title = _text(h4.group(1)) if h4 else full_title
|
|
alt = _ALT.search(description)
|
|
company = html.unescape(alt.group(1)).strip() if alt else ""
|
|
if not company and full_title.startswith(title + ", "):
|
|
company = full_title[len(title) + 2:].strip()
|
|
area = _AREA.search(description)
|
|
posted = None
|
|
if pub := item.findtext("pubDate"):
|
|
try:
|
|
posted = parsedate_to_datetime(pub)
|
|
except (TypeError, ValueError):
|
|
posted = None
|
|
place = _text(area.group(1)) if area else search_area
|
|
rows.append({
|
|
"id": link.rstrip("/").rsplit("/", 1)[-1],
|
|
"title": title or full_title,
|
|
"company": company,
|
|
"place": place,
|
|
"country": "DK",
|
|
"remote": _remote(f"{place} {title}"),
|
|
"url": link,
|
|
"posted": posted,
|
|
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
|
"salary_min": None, "salary_max": None, "currency": "",
|
|
})
|
|
return rows, len(items)
|
|
|
|
|
|
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
|
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
|
"""Every page of one search, newest first, until a short page or the page
|
|
limit. Returns the rows and whether the results ran out: a page with
|
|
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
|
a dropped item does not pass for the end. Jobindex states no total, so a
|
|
search that read the page limit full is never complete."""
|
|
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
|
if params.get("area"):
|
|
query["geoareaid"] = params["area"]
|
|
rows: list[Row] = []
|
|
for page in range(1, _pages(params) + 1):
|
|
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
|
rows += got
|
|
if items < JOBINDEX_PAGE:
|
|
return rows, True
|
|
return rows, False
|
|
|
|
|
|
# ------------------------------------------------------------------ adzuna
|
|
|
|
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
|
ADZUNA_PAGE = 50
|
|
|
|
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
|
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
|
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
|
|
|
|
|
def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
|
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
|
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
|
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
|
return _adzuna_rows(json.loads(body), country)
|
|
|
|
|
|
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
|
rows = []
|
|
for job in data.get("results") or []:
|
|
title = _text(job.get("title") or "")
|
|
url = (job.get("redirect_url") or "").strip()
|
|
if not title or not _web_url(url):
|
|
continue
|
|
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
|
rows.append({
|
|
"id": str(job.get("id") or ""),
|
|
"title": title,
|
|
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
|
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
|
|
"country": country.upper(),
|
|
"remote": _remote(title),
|
|
"url": url,
|
|
"posted": _parse_iso(job.get("created")),
|
|
"description": _text(job.get("description") or ""),
|
|
"salary_min": None if predicted else job.get("salary_min"),
|
|
"salary_max": None if predicted else job.get("salary_max"),
|
|
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_iso(value: Any) -> datetime | None:
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
|
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
|
"""Every page of one search, until a short page, the stated `count` or
|
|
the page limit. Returns the rows and whether they are all `count` says
|
|
match, or, on an answer with no `count`, whether a page came back short."""
|
|
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
|
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
|
if not (app_id and app_key):
|
|
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
|
|
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
|
|
country = str(params.get("country") or "").lower()
|
|
if country not in ADZUNA_COUNTRIES:
|
|
raise ValueError(f"adzuna country {country!r} is not one of "
|
|
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
|
|
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
|
|
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
|
|
"what": str(params.get("q") or "")}
|
|
if params.get("where"):
|
|
query["where"] = params["where"]
|
|
rows: list[Row] = []
|
|
seen: set[str] = set()
|
|
count: Any = None
|
|
ran_out = False
|
|
for page in range(1, _pages(params) + 1):
|
|
url = ADZUNA_SEARCH.format(country=country, page=page)
|
|
data = json.loads(_get(client, url, query))
|
|
results = data.get("results") or []
|
|
count = data.get("count", count)
|
|
seen |= {str(job.get("id") or "") for job in results}
|
|
rows += _adzuna_rows(data, country)
|
|
if len(results) < ADZUNA_PAGE:
|
|
ran_out = True
|
|
break
|
|
if isinstance(count, int) and len(seen) >= count:
|
|
break
|
|
# Adzuna states how many results match; the search is complete when it
|
|
# read that many distinct ones. Without a count, a short page is the end.
|
|
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
|
|
|
|
|
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
|
tuple[list[Row], bool]]] = {
|
|
"jobindex": search_jobindex,
|
|
"adzuna": search_adzuna,
|
|
}
|
|
|
|
|
|
def provider_of(target: str) -> str:
|
|
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
|
provider, sep, name = target.partition(":")
|
|
return provider if sep and name.strip() and provider in _SEARCHES else ""
|
|
|
|
|
|
def run_search(target: str, params: dict[str, Any],
|
|
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
|
"""One search, outside any pull: for `app searches add`, which runs a
|
|
search once before saving it."""
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
|
f"{', '.join(PROVIDERS)}")
|
|
with httpx.Client() as client:
|
|
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class SearchSource(Source):
|
|
name = "search"
|
|
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
|
|
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
|
subscription_field = "searches"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="Job searches",
|
|
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
log.warning("search: %r is not provider:name; skipped", target)
|
|
continue
|
|
try:
|
|
rows, complete = _SEARCHES[provider](
|
|
client, self.params_for(target), self.options.get(provider) or {})
|
|
except Exception as exc:
|
|
log.warning("search: %s failed (%s)", target, exc)
|
|
continue
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
source_id = f"{provider}:{row['id']}"
|
|
seen_ids.append(source_id)
|
|
yield Item(
|
|
source_id=source_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": row["company"],
|
|
"place": row["place"],
|
|
"country": row["country"],
|
|
"remote": row["remote"],
|
|
"salary_min": row["salary_min"],
|
|
"salary_max": row["salary_max"],
|
|
"currency": row["currency"],
|
|
"provider": provider,
|
|
"search": target,
|
|
},
|
|
)
|
|
self.state[f"search:ok:{target}"] = run_marker
|
|
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
|
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|