"""Job searches: a query against a job board that spans companies. `ats` reads the boards of companies someone already chose. This plugin finds roles at companies nobody has listed yet, by asking a cross-company board a saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. A subscription is one saved search; its `params` are the question. Two boards, both checked live on 2026-09-23: jobindex Denmark's largest job board. Every search result page has an RSS twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required. `area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page. adzuna An aggregator with a documented API for most of Europe, the UK and the US, but not Denmark (adzuna.dk does not exist). Needs a free key from developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built against Adzuna's documented response and not yet run live. Considered and left out: Jobnet (its search endpoint answers 401 outside the browser session of its own site) and Jooble (a key needs a manual application, and its API host answered a Cloudflare challenge). A search result list is never a board's complete list, so a role missing from one run is not closed here. A search that ran out of results before its page limit was complete, and the pull closes what it stopped listing; the rest close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). """ from __future__ import annotations import html import json import logging import os import re import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator from datetime import UTC, datetime from email.utils import parsedate_to_datetime from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, ) log = logging.getLogger(__name__) TIMEOUT = 15.0 #: Pages fetched per search unless its params say otherwise, and the most a #: search may ask for. A search that needs more than ten pages is too broad to #: read anyway. DEFAULT_PAGES = 5 MAX_PAGES = 10 PROVIDERS = ("jobindex", "adzuna") #: What every provider turns a result into. Row = dict[str, Any] def _web_url(url: str) -> bool: parsed = httpx.URL((url or "").strip()) return parsed.scheme in ("http", "https") and bool(parsed.host) def _pages(params: dict[str, Any]) -> int: try: return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES)) except (TypeError, ValueError): return DEFAULT_PAGES def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes: response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT}) response.raise_for_status() return response.content # ---------------------------------------------------------------- jobindex JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss" JOBINDEX_PAGE = 20 _ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) _H4 = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) _AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | re.DOTALL) _PARAGRAPHS = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) #: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") #: What a Danish ad's town line says about where the work is done: #: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". _HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) _REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE) def _remote(text: str) -> str | None: if _REMOTE.search(text): return "remote" return "hybrid" if _HYBRID.search(text) else None def _text(fragment: str) -> str: """HTML to plain text. Jobindex breaks titles after a slash with a zero-width space, which would otherwise end up in `title_key`.""" return html.unescape(strip_html(fragment)).replace("​", "").strip() def parse_jobindex(body: bytes) -> list[Row]: """One page of a Jobindex RSS search. An item is `` "Role, Company", a `<link>` to Jobindex's own page for it, and a `<description>` of HTML whose `<h4>` is the title alone, whose logo's `alt` is the company on a paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad ("robot job") has no logo, so its company is what follows the title's last comma, or '' when there is none.""" root = ET.fromstring(body) channel = root.find("channel") if channel is None: return [] area_match = _SEARCH_AREA.search(channel.findtext("title") or "") search_area = area_match.group(1).strip() if area_match else "" rows = [] for item in channel.findall("item"): link = (item.findtext("link") or "").strip() full_title = (item.findtext("title") or "").strip() if not full_title or not _web_url(link): continue description = item.findtext("description") or "" h4 = _H4.search(description) title = _text(h4.group(1)) if h4 else full_title alt = _ALT.search(description) company = html.unescape(alt.group(1)).strip() if alt else "" if not company and full_title.startswith(title + ", "): company = full_title[len(title) + 2:].strip() area = _AREA.search(description) posted = None if pub := item.findtext("pubDate"): try: posted = parsedate_to_datetime(pub) except (TypeError, ValueError): posted = None place = _text(area.group(1)) if area else search_area rows.append({ "id": link.rstrip("/").rsplit("/", 1)[-1], "title": title or full_title, "company": company, "place": place, "country": "DK", "remote": _remote(f"{place} {title}"), "url": link, "posted": posted, "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), "salary_min": None, "salary_max": None, "currency": "", }) return rows def search_jobindex(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: """Every page of one search, newest first, until a short page or the page limit. Returns the rows and whether the results ran out before the limit.""" query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} if params.get("area"): query["geoareaid"] = params["area"] rows: list[Row] = [] for page in range(1, _pages(params) + 1): got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page})) rows += got if len(got) < JOBINDEX_PAGE: return rows, True return rows, False # ------------------------------------------------------------------ adzuna ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}" ADZUNA_PAGE = 50 #: Adzuna's countries in Europe, and the currency its salaries are in. ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR", "be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"} def parse_adzuna(body: bytes, country: str) -> list[Row]: """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles and snippets carry `<strong>` highlights. A salary Adzuna estimated (`salary_is_predicted`) is not the posting's, so it is dropped.""" data = json.loads(body) rows = [] for job in data.get("results") or []: title = _text(job.get("title") or "") url = (job.get("redirect_url") or "").strip() if not title or not _web_url(url): continue predicted = str(job.get("salary_is_predicted") or "0") == "1" rows.append({ "id": str(job.get("id") or ""), "title": title, "company": ((job.get("company") or {}).get("display_name") or "").strip(), "place": ((job.get("location") or {}).get("display_name") or "").strip(), "country": country.upper(), "remote": _remote(title), "url": url, "posted": _parse_iso(job.get("created")), "description": _text(job.get("description") or ""), "salary_min": None if predicted else job.get("salary_min"), "salary_max": None if predicted else job.get("salary_max"), "currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""), }) return rows def _parse_iso(value: Any) -> datetime | None: try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except ValueError: return None return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) def search_adzuna(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") if not (app_id and app_key): raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, " "or ADZUNA_APP_ID and ADZUNA_APP_KEY") country = str(params.get("country") or "").lower() if country not in ADZUNA_COUNTRIES: raise ValueError(f"adzuna country {country!r} is not one of " f"{', '.join(sorted(ADZUNA_COUNTRIES))}") query: dict[str, Any] = {"app_id": app_id, "app_key": app_key, "results_per_page": ADZUNA_PAGE, "sort_by": "date", "what": str(params.get("q") or "")} if params.get("where"): query["where"] = params["where"] rows: list[Row] = [] for page in range(1, _pages(params) + 1): url = ADZUNA_SEARCH.format(country=country, page=page) got = parse_adzuna(_get(client, url, query), country) rows += got if len(got) < ADZUNA_PAGE: return rows, True return rows, False _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], tuple[list[Row], bool]]] = { "jobindex": search_jobindex, "adzuna": search_adzuna, } def provider_of(target: str) -> str: """`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one.""" provider, sep, name = target.partition(":") return provider if sep and name.strip() and provider in _SEARCHES else "" def run_search(target: str, params: dict[str, Any], keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]: """One search, outside any pull: for `app searches add`, which runs a search once before saving it.""" provider = provider_of(target) if not provider: raise ValueError(f"{target!r} is not provider:name with provider one of " f"{', '.join(PROVIDERS)}") with httpx.Client() as client: return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {}) # -------------------------------------------------------------- the plugin class SearchSource(Source): name = "search" description = "Saved searches on cross-company job boards: Jobindex, Adzuna." #: Entries are `{"url": target, "label": name, "params": {...}}`. subscription_field = "searches" spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="Job searches", targets=TargetSpec(mode="editable", kind="search", defaults=[]), profiles={"default": Profile( kind="role", tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy(enabled=False), ), presentation=ARTICLE_READER, )}, ) def fetch(self) -> Iterator[Item]: client = httpx.Client() run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): provider = provider_of(target) if not provider: log.warning("search: %r is not provider:name; skipped", target) continue try: rows, complete = _SEARCHES[provider]( client, self.params_for(target), self.options.get(provider) or {}) except Exception as exc: log.warning("search: %s failed (%s)", target, exc) continue seen_ids = [] for row in rows: source_id = f"{provider}:{row['id']}" seen_ids.append(source_id) yield Item( source_id=source_id, location=row["url"], title=row["title"], excerpt=(row["description"] or "")[:300], body=row["description"], published_at=row["posted"], subscription=label, kind="role", raw={ "company": row["company"], "place": row["place"], "country": row["country"], "remote": row["remote"], "salary_min": row["salary_min"], "salary_max": row["salary_max"], "currency": row["currency"], "provider": provider, "search": target, }, ) self.state[f"search:ok:{target}"] = run_marker self.state[f"search:complete:{target}"] = "1" if complete else "" self.state[f"search:ids:{target}"] = json.dumps(seen_ids)