"""Job searches: a query against a job board that spans companies.
`ats` reads the boards of companies someone already chose. This plugin finds
roles at companies nobody has listed yet, by asking a cross-company board a
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
A subscription is one saved search; its `params` are the question.
Two boards, both checked live on 2026-09-23:
jobindex Denmark's largest job board. Every search result page has an RSS
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
own syntax: words are OR, `"a b"` a phrase, `+word` required.
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
20 results a page.
adzuna An aggregator with a documented API for most of Europe, the UK
and the US, but not Denmark (adzuna.dk does not exist). Needs a
free key from developer.adzuna.com, as `sources.search.adzuna:
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
against Adzuna's documented response and not yet run live.
Considered and left out: Jobnet (its search endpoint answers 401 outside the
browser session of its own site) and Jooble (a key needs a manual
application, and its API host answered a Cloudflare challenge).
A search result list is never a board's complete list, so a role missing from
one run is not closed here. A search that ran out of results before its page
limit was complete, and the pull closes what it stopped listing; the rest
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
"""
from __future__ import annotations
import html
import json
import logging
import os
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from typing import Any
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
)
log = logging.getLogger(__name__)
TIMEOUT = 15.0
#: Pages fetched per search unless its params say otherwise, and the most a
#: search may ask for. A search that needs more than ten pages is too broad to
#: read anyway.
DEFAULT_PAGES = 5
MAX_PAGES = 10
PROVIDERS = ("jobindex", "adzuna")
#: What every provider turns a result into.
Row = dict[str, Any]
def _web_url(url: str) -> bool:
parsed = httpx.URL((url or "").strip())
return parsed.scheme in ("http", "https") and bool(parsed.host)
def _pages(params: dict[str, Any]) -> int:
try:
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
except (TypeError, ValueError):
return DEFAULT_PAGES
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
headers={"User-Agent": USER_AGENT})
response.raise_for_status()
return response.content
# ---------------------------------------------------------------- jobindex
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
JOBINDEX_PAGE = 20
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
#: against the titles its feeds answer with. 0 is all of Denmark.
JOBINDEX_AREAS: dict[int, str] = {
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
8: "Aarhus", 7: "Aalborg",
}
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
_H4 = re.compile(r"
", re.IGNORECASE | re.DOTALL)
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
#: What a Danish ad's town line says about where the work is done:
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
def _remote(text: str) -> str | None:
if _REMOTE.search(text):
return "remote"
return "hybrid" if _HYBRID.search(text) else None
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
def parse_jobindex(body: bytes) -> list[Row]:
"""One page of a Jobindex RSS search. An item is `` "Role, Company",
a `` to Jobindex's own page for it, and a `` of HTML
whose `
` is the title alone, whose logo's `alt` is the company on a
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
("robot job") has no logo, so its company is what follows the title's
last comma, or '' when there is none."""
root = ET.fromstring(body)
channel = root.find("channel")
if channel is None:
return []
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
search_area = area_match.group(1).strip() if area_match else ""
rows = []
for item in channel.findall("item"):
link = (item.findtext("link") or "").strip()
full_title = (item.findtext("title") or "").strip()
if not full_title or not _web_url(link):
continue
description = item.findtext("description") or ""
h4 = _H4.search(description)
title = _text(h4.group(1)) if h4 else full_title
alt = _ALT.search(description)
company = html.unescape(alt.group(1)).strip() if alt else ""
if not company and full_title.startswith(title + ", "):
company = full_title[len(title) + 2:].strip()
area = _AREA.search(description)
posted = None
if pub := item.findtext("pubDate"):
try:
posted = parsedate_to_datetime(pub)
except (TypeError, ValueError):
posted = None
place = _text(area.group(1)) if area else search_area
rows.append({
"id": link.rstrip("/").rsplit("/", 1)[-1],
"title": title or full_title,
"company": company,
"place": place,
"country": "DK",
"remote": _remote(f"{place} {title}"),
"url": link,
"posted": posted,
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
"salary_min": None, "salary_max": None, "currency": "",
})
return rows
def search_jobindex(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
"""Every page of one search, newest first, until a short page or the page
limit. Returns the rows and whether the results ran out before the limit."""
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
if params.get("area"):
query["geoareaid"] = params["area"]
rows: list[Row] = []
for page in range(1, _pages(params) + 1):
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page}))
rows += got
if len(got) < JOBINDEX_PAGE:
return rows, True
return rows, False
# ------------------------------------------------------------------ adzuna
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
ADZUNA_PAGE = 50
#: Adzuna's countries in Europe, and the currency its salaries are in.
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
def parse_adzuna(body: bytes, country: str) -> list[Row]:
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
and snippets carry `` highlights. A salary Adzuna estimated
(`salary_is_predicted`) is not the posting's, so it is dropped."""
data = json.loads(body)
rows = []
for job in data.get("results") or []:
title = _text(job.get("title") or "")
url = (job.get("redirect_url") or "").strip()
if not title or not _web_url(url):
continue
predicted = str(job.get("salary_is_predicted") or "0") == "1"
rows.append({
"id": str(job.get("id") or ""),
"title": title,
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
"country": country.upper(),
"remote": _remote(title),
"url": url,
"posted": _parse_iso(job.get("created")),
"description": _text(job.get("description") or ""),
"salary_min": None if predicted else job.get("salary_min"),
"salary_max": None if predicted else job.get("salary_max"),
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
})
return rows
def _parse_iso(value: Any) -> datetime | None:
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except ValueError:
return None
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
def search_adzuna(client: httpx.Client, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[Row], bool]:
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
if not (app_id and app_key):
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
country = str(params.get("country") or "").lower()
if country not in ADZUNA_COUNTRIES:
raise ValueError(f"adzuna country {country!r} is not one of "
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
"what": str(params.get("q") or "")}
if params.get("where"):
query["where"] = params["where"]
rows: list[Row] = []
for page in range(1, _pages(params) + 1):
url = ADZUNA_SEARCH.format(country=country, page=page)
got = parse_adzuna(_get(client, url, query), country)
rows += got
if len(got) < ADZUNA_PAGE:
return rows, True
return rows, False
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
tuple[list[Row], bool]]] = {
"jobindex": search_jobindex,
"adzuna": search_adzuna,
}
def provider_of(target: str) -> str:
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
provider, sep, name = target.partition(":")
return provider if sep and name.strip() and provider in _SEARCHES else ""
def run_search(target: str, params: dict[str, Any],
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
"""One search, outside any pull: for `app searches add`, which runs a
search once before saving it."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
f"{', '.join(PROVIDERS)}")
with httpx.Client() as client:
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
# -------------------------------------------------------------- the plugin
class SearchSource(Source):
name = "search"
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
#: Entries are `{"url": target, "label": name, "params": {...}}`.
subscription_field = "searches"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="Job searches",
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
provider = provider_of(target)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
rows, complete = _SEARCHES[provider](
client, self.params_for(target), self.options.get(provider) or {})
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
seen_ids = []
for row in rows:
source_id = f"{provider}:{row['id']}"
seen_ids.append(source_id)
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": row["company"],
"place": row["place"],
"country": row["country"],
"remote": row["remote"],
"salary_min": row["salary_min"],
"salary_max": row["salary_max"],
"currency": row["currency"],
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)