Find roles by search, not only by company: Jobindex and Adzuna searches
The ats boards only read companies someone listed, and the seed list is US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs saved queries on boards that span companies, pulled in the same task as the boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU, free key, built to its documented API). Jobnet and Jooble were checked and left out. `country` is now filled from the board's location line by a lookup, with EU for Europe-wide roles, so country gates work without extract. `app companies prune --in` unsubscribes boards with no open role in a region. The starter agents and searches are Copenhagen and Europe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bfab68c878
commit
047f7cd8ad
17 files changed
+1293
-72
No files matched your search
@@ -0,0 +1,345 @@
|
||||
"""Job searches: a query against a job board that spans companies.
|
||||
|
||||
`ats` reads the boards of companies someone already chose. This plugin finds
|
||||
roles at companies nobody has listed yet, by asking a cross-company board a
|
||||
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
||||
A subscription is one saved search; its `params` are the question.
|
||||
|
||||
Two boards, both checked live on 2026-09-23:
|
||||
|
||||
jobindex Denmark's largest job board. Every search result page has an RSS
|
||||
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
||||
own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
||||
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
|
||||
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
|
||||
20 results a page.
|
||||
adzuna An aggregator with a documented API for most of Europe, the UK
|
||||
and the US, but not Denmark (adzuna.dk does not exist). Needs a
|
||||
free key from developer.adzuna.com, as `sources.search.adzuna:
|
||||
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
||||
against Adzuna's documented response and not yet run live.
|
||||
|
||||
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
||||
browser session of its own site) and Jooble (a key needs a manual
|
||||
application, and its API host answered a Cloudflare challenge).
|
||||
|
||||
A search result list is never a board's complete list, so a role missing from
|
||||
one run is not closed here. A search that ran out of results before its page
|
||||
limit was complete, and the pull closes what it stopped listing; the rest
|
||||
close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TIMEOUT = 15.0
|
||||
|
||||
#: Pages fetched per search unless its params say otherwise, and the most a
|
||||
#: search may ask for. A search that needs more than ten pages is too broad to
|
||||
#: read anyway.
|
||||
DEFAULT_PAGES = 5
|
||||
MAX_PAGES = 10
|
||||
|
||||
PROVIDERS = ("jobindex", "adzuna")
|
||||
|
||||
#: What every provider turns a result into.
|
||||
Row = dict[str, Any]
|
||||
|
||||
|
||||
def _web_url(url: str) -> bool:
|
||||
parsed = httpx.URL((url or "").strip())
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def _pages(params: dict[str, Any]) -> int:
|
||||
try:
|
||||
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_PAGES
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
|
||||
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- jobindex
|
||||
|
||||
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
||||
JOBINDEX_PAGE = 20
|
||||
|
||||
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
||||
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
||||
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
||||
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
||||
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
||||
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
||||
|
||||
|
||||
#: What a Danish ad's town line says about where the work is done:
|
||||
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
||||
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
||||
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
|
||||
|
||||
|
||||
def _remote(text: str) -> str | None:
|
||||
if _REMOTE.search(text):
|
||||
return "remote"
|
||||
return "hybrid" if _HYBRID.search(text) else None
|
||||
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
||||
zero-width space, which would otherwise end up in `title_key`."""
|
||||
return html.unescape(strip_html(fragment)).replace("", "").strip()
|
||||
|
||||
|
||||
def parse_jobindex(body: bytes) -> list[Row]:
|
||||
"""One page of a Jobindex RSS search. An item is `<title>` "Role, Company",
|
||||
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
||||
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
||||
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
||||
("robot job") has no logo, so its company is what follows the title's
|
||||
last comma, or '' when there is none."""
|
||||
root = ET.fromstring(body)
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
return []
|
||||
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
||||
search_area = area_match.group(1).strip() if area_match else ""
|
||||
rows = []
|
||||
for item in channel.findall("item"):
|
||||
link = (item.findtext("link") or "").strip()
|
||||
full_title = (item.findtext("title") or "").strip()
|
||||
if not full_title or not _web_url(link):
|
||||
continue
|
||||
description = item.findtext("description") or ""
|
||||
h4 = _H4.search(description)
|
||||
title = _text(h4.group(1)) if h4 else full_title
|
||||
alt = _ALT.search(description)
|
||||
company = html.unescape(alt.group(1)).strip() if alt else ""
|
||||
if not company and full_title.startswith(title + ", "):
|
||||
company = full_title[len(title) + 2:].strip()
|
||||
area = _AREA.search(description)
|
||||
posted = None
|
||||
if pub := item.findtext("pubDate"):
|
||||
try:
|
||||
posted = parsedate_to_datetime(pub)
|
||||
except (TypeError, ValueError):
|
||||
posted = None
|
||||
place = _text(area.group(1)) if area else search_area
|
||||
rows.append({
|
||||
"id": link.rstrip("/").rsplit("/", 1)[-1],
|
||||
"title": title or full_title,
|
||||
"company": company,
|
||||
"place": place,
|
||||
"country": "DK",
|
||||
"remote": _remote(f"{place} {title}"),
|
||||
"url": link,
|
||||
"posted": posted,
|
||||
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
||||
"salary_min": None, "salary_max": None, "currency": "",
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
"""Every page of one search, newest first, until a short page or the page
|
||||
limit. Returns the rows and whether the results ran out before the limit."""
|
||||
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
||||
if params.get("area"):
|
||||
query["geoareaid"] = params["area"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
got = parse_jobindex(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
||||
rows += got
|
||||
if len(got) < JOBINDEX_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ adzuna
|
||||
|
||||
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
||||
ADZUNA_PAGE = 50
|
||||
|
||||
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
||||
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
||||
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
||||
|
||||
|
||||
def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
||||
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
||||
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
||||
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for job in data.get("results") or []:
|
||||
title = _text(job.get("title") or "")
|
||||
url = (job.get("redirect_url") or "").strip()
|
||||
if not title or not _web_url(url):
|
||||
continue
|
||||
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
||||
rows.append({
|
||||
"id": str(job.get("id") or ""),
|
||||
"title": title,
|
||||
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
||||
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
|
||||
"country": country.upper(),
|
||||
"remote": _remote(title),
|
||||
"url": url,
|
||||
"posted": _parse_iso(job.get("created")),
|
||||
"description": _text(job.get("description") or ""),
|
||||
"salary_min": None if predicted else job.get("salary_min"),
|
||||
"salary_max": None if predicted else job.get("salary_max"),
|
||||
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def _parse_iso(value: Any) -> datetime | None:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
||||
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
||||
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
||||
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
||||
if not (app_id and app_key):
|
||||
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
|
||||
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
|
||||
country = str(params.get("country") or "").lower()
|
||||
if country not in ADZUNA_COUNTRIES:
|
||||
raise ValueError(f"adzuna country {country!r} is not one of "
|
||||
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
|
||||
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
|
||||
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
|
||||
"what": str(params.get("q") or "")}
|
||||
if params.get("where"):
|
||||
query["where"] = params["where"]
|
||||
rows: list[Row] = []
|
||||
for page in range(1, _pages(params) + 1):
|
||||
url = ADZUNA_SEARCH.format(country=country, page=page)
|
||||
got = parse_adzuna(_get(client, url, query), country)
|
||||
rows += got
|
||||
if len(got) < ADZUNA_PAGE:
|
||||
return rows, True
|
||||
return rows, False
|
||||
|
||||
|
||||
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
||||
tuple[list[Row], bool]]] = {
|
||||
"jobindex": search_jobindex,
|
||||
"adzuna": search_adzuna,
|
||||
}
|
||||
|
||||
|
||||
def provider_of(target: str) -> str:
|
||||
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
||||
provider, sep, name = target.partition(":")
|
||||
return provider if sep and name.strip() and provider in _SEARCHES else ""
|
||||
|
||||
|
||||
def run_search(target: str, params: dict[str, Any],
|
||||
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
||||
"""One search, outside any pull: for `app searches add`, which runs a
|
||||
search once before saving it."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
f"{', '.join(PROVIDERS)}")
|
||||
with httpx.Client() as client:
|
||||
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
|
||||
class SearchSource(Source):
|
||||
name = "search"
|
||||
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
|
||||
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
||||
subscription_field = "searches"
|
||||
|
||||
spec = PluginSpec(
|
||||
api_version=API_VERSION, id=name, version="1", label="Job searches",
|
||||
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
|
||||
profiles={"default": Profile(
|
||||
kind="role",
|
||||
tasks=Tasks(
|
||||
summary=SummaryPolicy(enabled=False),
|
||||
tagging=TaggingPolicy(enabled=False),
|
||||
),
|
||||
presentation=ARTICLE_READER,
|
||||
)},
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
seen_ids.append(source_id)
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
Reference in new issue
Block a user