"""Job searches: a query against a job board that spans companies. `ats` reads the boards of companies someone already chose. This plugin finds roles at companies nobody has listed yet, by asking a cross-company board a saved question: "backend developer" in Region Hovedstaden, "python" in Berlin. A subscription is one saved search; its `params` are the question. Two boards, both checked live on 2026-09-23: jobindex Denmark's largest job board. Every search result page has an RSS twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's own syntax: words are OR, `"a b"` a phrase, `+word` required. `area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus. 20 results a page. adzuna An aggregator with a documented API for most of Europe, the UK and the US, but not Denmark (adzuna.dk does not exist). Needs a free key from developer.adzuna.com, as `sources.search.adzuna: {app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built against Adzuna's documented response and not yet run live. Considered and left out: Jobnet (its search endpoint answers 401 outside the browser session of its own site) and Jooble (a key needs a manual application, and its API host answered a Cloudflare challenge). A search result list is never a board's complete list, so a role missing from one run is not closed here. A search that read everything that matches was complete, and the pull closes what it stopped listing; the rest close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states how many results match, so an Adzuna search is complete when it read that many; Jobindex states no count, so a Jobindex search is complete when a page within the page limit came back short. """ from __future__ import annotations import html import json import logging import os import re import xml.etree.ElementTree as ET from collections.abc import Callable, Iterator from datetime import UTC, datetime from email.utils import parsedate_to_datetime from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html, ) log = logging.getLogger(__name__) TIMEOUT = 15.0 #: Pages fetched per search unless its params say otherwise, and the most a #: search may ask for. A search that needs more than ten pages is too broad to #: read anyway. DEFAULT_PAGES = 5 MAX_PAGES = 10 PROVIDERS = ("jobindex", "adzuna") #: What every provider turns a result into. Row = dict[str, Any] def _web_url(url: str) -> bool: parsed = httpx.URL((url or "").strip()) return parsed.scheme in ("http", "https") and bool(parsed.host) def _pages(params: dict[str, Any]) -> int: try: return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES)) except (TypeError, ValueError): return DEFAULT_PAGES def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes: response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT}) response.raise_for_status() return response.content # ---------------------------------------------------------------- jobindex JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss" JOBINDEX_PAGE = 20 #: Jobindex's `geoareaid` for the regions and the biggest towns, as checked #: against the titles its feeds answer with. 0 is all of Denmark. JOBINDEX_AREAS: dict[int, str] = { 0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland", 2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland", 8: "Aarhus", 7: "Aalborg", } _ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) _H4 = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) _AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | re.DOTALL) _PARAGRAPHS = re.compile(r"

(.*?)

", re.IGNORECASE | re.DOTALL) #: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden". _SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende") #: What a Danish ad's town line says about where the work is done: #: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde". _HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE) _REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE) def _remote(text: str) -> str | None: if _REMOTE.search(text): return "remote" return "hybrid" if _HYBRID.search(text) else None def _text(fragment: str) -> str: """HTML to plain text. Jobindex breaks titles after a slash with a zero-width space, which would otherwise end up in `title_key`.""" return html.unescape(strip_html(fragment)).replace("\u200b", "").strip() def parse_jobindex(body: bytes) -> list[Row]: return _jobindex_page(body)[0] def _jobindex_page(body: bytes) -> tuple[list[Row], int]: """One page of a Jobindex RSS search, and how many items it had before any was dropped. An item is `` "Role, Company", a `<link>` to Jobindex's own page for it, and a `<description>` of HTML whose `<h4>` is the title alone, whose logo's `alt` is the company on a paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad ("robot job") has no logo, so its company is what follows the title's last comma, or '' when there is none. A body with no `<channel>` is not a search result and raises, since an empty list would read as a search that ran out and close everything it listed before.""" root = ET.fromstring(body) channel = root.find("channel") if channel is None: raise ValueError("jobindex answered without an RSS channel") area_match = _SEARCH_AREA.search(channel.findtext("title") or "") search_area = area_match.group(1).strip() if area_match else "" rows = [] items = channel.findall("item") for item in items: link = (item.findtext("link") or "").strip() full_title = (item.findtext("title") or "").strip() if not full_title or not _web_url(link): continue description = item.findtext("description") or "" h4 = _H4.search(description) title = _text(h4.group(1)) if h4 else full_title alt = _ALT.search(description) company = html.unescape(alt.group(1)).strip() if alt else "" if not company and full_title.startswith(title + ", "): company = full_title[len(title) + 2:].strip() area = _AREA.search(description) posted = None if pub := item.findtext("pubDate"): try: posted = parsedate_to_datetime(pub) except (TypeError, ValueError): posted = None place = _text(area.group(1)) if area else search_area rows.append({ "id": link.rstrip("/").rsplit("/", 1)[-1], "title": title or full_title, "company": company, "place": place, "country": "DK", "remote": _remote(f"{place} {title}"), "url": link, "posted": posted, "description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)), "salary_min": None, "salary_max": None, "currency": "", }) return rows, len(items) def search_jobindex(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: """Every page of one search, newest first, until a short page or the page limit. Returns the rows and whether the results ran out: a page with fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so a dropped item does not pass for the end. Jobindex states no total, so a search that read the page limit full is never complete.""" query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"} if params.get("area"): query["geoareaid"] = params["area"] rows: list[Row] = [] for page in range(1, _pages(params) + 1): got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page})) rows += got if items < JOBINDEX_PAGE: return rows, True return rows, False # ------------------------------------------------------------------ adzuna ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}" ADZUNA_PAGE = 50 #: Adzuna's countries in Europe, and the currency its salaries are in. ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR", "be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"} def parse_adzuna(body: bytes, country: str) -> list[Row]: """One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles and snippets carry `<strong>` highlights. A salary Adzuna estimated (`salary_is_predicted`) is not the posting's, so it is dropped.""" return _adzuna_rows(json.loads(body), country) def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]: rows = [] for job in data.get("results") or []: title = _text(job.get("title") or "") url = (job.get("redirect_url") or "").strip() if not title or not _web_url(url): continue predicted = str(job.get("salary_is_predicted") or "0") == "1" rows.append({ "id": str(job.get("id") or ""), "title": title, "company": ((job.get("company") or {}).get("display_name") or "").strip(), "place": ((job.get("location") or {}).get("display_name") or "").strip(), "country": country.upper(), "remote": _remote(title), "url": url, "posted": _parse_iso(job.get("created")), "description": _text(job.get("description") or ""), "salary_min": None if predicted else job.get("salary_min"), "salary_max": None if predicted else job.get("salary_max"), "currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""), }) return rows def _parse_iso(value: Any) -> datetime | None: try: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) except ValueError: return None return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) def search_adzuna(client: httpx.Client, params: dict[str, Any], keys: dict[str, Any]) -> tuple[list[Row], bool]: """Every page of one search, until a short page, the stated `count` or the page limit. Returns the rows and whether they are all `count` says match, or, on an answer with no `count`, whether a page came back short.""" app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "") app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "") if not (app_id and app_key): raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, " "or ADZUNA_APP_ID and ADZUNA_APP_KEY") country = str(params.get("country") or "").lower() if country not in ADZUNA_COUNTRIES: raise ValueError(f"adzuna country {country!r} is not one of " f"{', '.join(sorted(ADZUNA_COUNTRIES))}") query: dict[str, Any] = {"app_id": app_id, "app_key": app_key, "results_per_page": ADZUNA_PAGE, "sort_by": "date", "what": str(params.get("q") or "")} if params.get("where"): query["where"] = params["where"] rows: list[Row] = [] seen: set[str] = set() count: Any = None ran_out = False for page in range(1, _pages(params) + 1): url = ADZUNA_SEARCH.format(country=country, page=page) data = json.loads(_get(client, url, query)) results = data.get("results") or [] count = data.get("count", count) seen |= {str(job.get("id") or "") for job in results} rows += _adzuna_rows(data, country) if len(results) < ADZUNA_PAGE: ran_out = True break if isinstance(count, int) and len(seen) >= count: break # Adzuna states how many results match; the search is complete when it # read that many distinct ones. Without a count, a short page is the end. return rows, (len(seen) == count) if isinstance(count, int) else ran_out _SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]], tuple[list[Row], bool]]] = { "jobindex": search_jobindex, "adzuna": search_adzuna, } def provider_of(target: str) -> str: """`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one.""" provider, sep, name = target.partition(":") return provider if sep and name.strip() and provider in _SEARCHES else "" def run_search(target: str, params: dict[str, Any], keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]: """One search, outside any pull: for `app searches add`, which runs a search once before saving it.""" provider = provider_of(target) if not provider: raise ValueError(f"{target!r} is not provider:name with provider one of " f"{', '.join(PROVIDERS)}") with httpx.Client() as client: return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {}) # -------------------------------------------------------------- the plugin class SearchSource(Source): name = "search" description = "Saved searches on cross-company job boards: Jobindex, Adzuna." #: Entries are `{"url": target, "label": name, "params": {...}}`. subscription_field = "searches" spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="Job searches", targets=TargetSpec(mode="editable", kind="search", defaults=[]), profiles={"default": Profile( kind="role", tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy(enabled=False), ), presentation=ARTICLE_READER, )}, ) def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # The pull run's marker, handed in by its batch as for `ats`. run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): provider = provider_of(target) if not provider: log.warning("search: %r is not provider:name; skipped", target) continue try: rows, complete = _SEARCHES[provider]( client, self.params_for(target), self.options.get(provider) or {}) except Exception as exc: log.warning("search: %s failed (%s)", target, exc) continue seen_ids = [] for row in rows: source_id = f"{provider}:{row['id']}" seen_ids.append(source_id) yield Item( source_id=source_id, location=row["url"], title=row["title"], excerpt=(row["description"] or "")[:300], body=row["description"], published_at=row["posted"], subscription=label, kind="role", raw={ "company": row["company"], "place": row["place"], "country": row["country"], "remote": row["remote"], "salary_min": row["salary_min"], "salary_max": row["salary_max"], "currency": row["currency"], "provider": provider, "search": target, }, ) self.state[f"search:ok:{target}"] = run_marker self.state[f"search:complete:{target}"] = "1" if complete else "" self.state[f"search:ids:{target}"] = json.dumps(seen_ids)