A second pull (schedule, Run button, `app job pull` in another process) could start while one was fetching and overwrite the stored run marker, so the first run's boards failed their stamp check or a settle closed on the other run's id lists. `claim_run` sets `pull:run` in one conditional statement, only while it equals `pull:settled` or is older than an hour; a refused `pull` job queues no stage and says so in its result and a notice, and the CLI exits 1. Batches carry the marker in their payload, the plugins stamp with it, and the settle reads it off its own batches' payloads before releasing the run. Co-Authored-By: Claude Opus <noreply@anthropic.com>
386 lines
16 KiB
Python
386 lines
16 KiB
Python
"""Job searches: a query against a job board that spans companies.
|
|
|
|
`ats` reads the boards of companies someone already chose. This plugin finds
|
|
roles at companies nobody has listed yet, by asking a cross-company board a
|
|
saved question: "backend developer" in Region Hovedstaden, "python" in Berlin.
|
|
A subscription is one saved search; its `params` are the question.
|
|
|
|
Two boards, both checked live on 2026-09-23:
|
|
|
|
jobindex Denmark's largest job board. Every search result page has an RSS
|
|
twin at /jobsoegning.rss, no key needed. `q` takes Jobindex's
|
|
own syntax: words are OR, `"a b"` a phrase, `+word` required.
|
|
`area` is its `geoareaid`: 1 Region Hovedstaden, 4 Region
|
|
Sjælland, 2 Midtjylland, 5 Syddanmark, 3 Nordjylland, 8 Aarhus.
|
|
20 results a page.
|
|
adzuna An aggregator with a documented API for most of Europe, the UK
|
|
and the US, but not Denmark (adzuna.dk does not exist). Needs a
|
|
free key from developer.adzuna.com, as `sources.search.adzuna:
|
|
{app_id, app_key}` or `ADZUNA_APP_ID`/`ADZUNA_APP_KEY`. Built
|
|
against Adzuna's documented response and not yet run live.
|
|
|
|
Considered and left out: Jobnet (its search endpoint answers 401 outside the
|
|
browser session of its own site) and Jooble (a key needs a manual
|
|
application, and its API host answered a Cloudflare challenge).
|
|
|
|
A search result list is never a board's complete list, so a role missing from
|
|
one run is not closed here. A search that read everything that matches was
|
|
complete, and the pull closes what it stopped listing; the rest close once no
|
|
search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). Adzuna states
|
|
how many results match, so an Adzuna search is complete when it read that
|
|
many; Jobindex states no count, so a Jobindex search is complete when a page
|
|
within the page limit came back short.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, strip_html,
|
|
)
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
TIMEOUT = 15.0
|
|
|
|
#: Pages fetched per search unless its params say otherwise, and the most a
|
|
#: search may ask for. A search that needs more than ten pages is too broad to
|
|
#: read anyway.
|
|
DEFAULT_PAGES = 5
|
|
MAX_PAGES = 10
|
|
|
|
PROVIDERS = ("jobindex", "adzuna")
|
|
|
|
#: What every provider turns a result into.
|
|
Row = dict[str, Any]
|
|
|
|
|
|
def _web_url(url: str) -> bool:
|
|
parsed = httpx.URL((url or "").strip())
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
def _pages(params: dict[str, Any]) -> int:
|
|
try:
|
|
return max(1, min(int(params.get("pages") or DEFAULT_PAGES), MAX_PAGES))
|
|
except (TypeError, ValueError):
|
|
return DEFAULT_PAGES
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
|
|
response = client.get(url, params=params, timeout=TIMEOUT, follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT})
|
|
response.raise_for_status()
|
|
return response.content
|
|
|
|
|
|
# ---------------------------------------------------------------- jobindex
|
|
|
|
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
|
|
JOBINDEX_PAGE = 20
|
|
|
|
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
|
|
#: against the titles its feeds answer with. 0 is all of Denmark.
|
|
JOBINDEX_AREAS: dict[int, str] = {
|
|
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
|
|
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
|
|
8: "Aarhus", 7: "Aalborg",
|
|
}
|
|
|
|
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
|
|
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
|
|
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
|
|
_PARAGRAPHS = re.compile(r"<p>(.*?)</p>", re.IGNORECASE | re.DOTALL)
|
|
#: "Ledige job i Region Hovedstaden indeholdende ..." -> "Region Hovedstaden".
|
|
_SEARCH_AREA = re.compile(r"Ledige job i (.+?) indeholdende")
|
|
|
|
|
|
#: What a Danish ad's town line says about where the work is done:
|
|
#: "Birkerød, Hybrid position", "Nordhavn og mulighed for hjemmearbejde".
|
|
_HYBRID = re.compile(r"\bhybrid|hjemmearbejde", re.IGNORECASE)
|
|
_REMOTE = re.compile(r"\bremote\b|fjernarbejde|fuldt hjemmearbejde", re.IGNORECASE)
|
|
|
|
|
|
def _remote(text: str) -> str | None:
|
|
if _REMOTE.search(text):
|
|
return "remote"
|
|
return "hybrid" if _HYBRID.search(text) else None
|
|
|
|
|
|
def _text(fragment: str) -> str:
|
|
"""HTML to plain text. Jobindex breaks titles after a slash with a
|
|
zero-width space, which would otherwise end up in `title_key`."""
|
|
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
|
|
|
|
|
|
def parse_jobindex(body: bytes) -> list[Row]:
|
|
return _jobindex_page(body)[0]
|
|
|
|
|
|
def _jobindex_page(body: bytes) -> tuple[list[Row], int]:
|
|
"""One page of a Jobindex RSS search, and how many items it had before
|
|
any was dropped. An item is `<title>` "Role, Company",
|
|
a `<link>` to Jobindex's own page for it, and a `<description>` of HTML
|
|
whose `<h4>` is the title alone, whose logo's `alt` is the company on a
|
|
paid ad, and whose `jix_robotjob--area` span is the town. A crawled ad
|
|
("robot job") has no logo, so its company is what follows the title's
|
|
last comma, or '' when there is none. A body with no `<channel>` is not
|
|
a search result and raises, since an empty list would read as a search
|
|
that ran out and close everything it listed before."""
|
|
root = ET.fromstring(body)
|
|
channel = root.find("channel")
|
|
if channel is None:
|
|
raise ValueError("jobindex answered without an RSS channel")
|
|
area_match = _SEARCH_AREA.search(channel.findtext("title") or "")
|
|
search_area = area_match.group(1).strip() if area_match else ""
|
|
rows = []
|
|
items = channel.findall("item")
|
|
for item in items:
|
|
link = (item.findtext("link") or "").strip()
|
|
full_title = (item.findtext("title") or "").strip()
|
|
if not full_title or not _web_url(link):
|
|
continue
|
|
description = item.findtext("description") or ""
|
|
h4 = _H4.search(description)
|
|
title = _text(h4.group(1)) if h4 else full_title
|
|
alt = _ALT.search(description)
|
|
company = html.unescape(alt.group(1)).strip() if alt else ""
|
|
if not company and full_title.startswith(title + ", "):
|
|
company = full_title[len(title) + 2:].strip()
|
|
area = _AREA.search(description)
|
|
posted = None
|
|
if pub := item.findtext("pubDate"):
|
|
try:
|
|
posted = parsedate_to_datetime(pub)
|
|
except (TypeError, ValueError):
|
|
posted = None
|
|
place = _text(area.group(1)) if area else search_area
|
|
rows.append({
|
|
"id": link.rstrip("/").rsplit("/", 1)[-1],
|
|
"title": title or full_title,
|
|
"company": company,
|
|
"place": place,
|
|
"country": "DK",
|
|
"remote": _remote(f"{place} {title}"),
|
|
"url": link,
|
|
"posted": posted,
|
|
"description": " ".join(_text(p) for p in _PARAGRAPHS.findall(description)),
|
|
"salary_min": None, "salary_max": None, "currency": "",
|
|
})
|
|
return rows, len(items)
|
|
|
|
|
|
def search_jobindex(client: httpx.Client, params: dict[str, Any],
|
|
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
|
"""Every page of one search, newest first, until a short page or the page
|
|
limit. Returns the rows and whether the results ran out: a page with
|
|
fewer than `JOBINDEX_PAGE` items, counted before the parser drops any, so
|
|
a dropped item does not pass for the end. Jobindex states no total, so a
|
|
search that read the page limit full is never complete."""
|
|
query: dict[str, Any] = {"q": str(params.get("q") or ""), "sort": "date"}
|
|
if params.get("area"):
|
|
query["geoareaid"] = params["area"]
|
|
rows: list[Row] = []
|
|
for page in range(1, _pages(params) + 1):
|
|
got, items = _jobindex_page(_get(client, JOBINDEX_RSS, query | {"page": page}))
|
|
rows += got
|
|
if items < JOBINDEX_PAGE:
|
|
return rows, True
|
|
return rows, False
|
|
|
|
|
|
# ------------------------------------------------------------------ adzuna
|
|
|
|
ADZUNA_SEARCH = "https://api.adzuna.com/v1/api/jobs/{country}/search/{page}"
|
|
ADZUNA_PAGE = 50
|
|
|
|
#: Adzuna's countries in Europe, and the currency its salaries are in.
|
|
ADZUNA_COUNTRIES = {"gb": "GBP", "de": "EUR", "nl": "EUR", "fr": "EUR", "at": "EUR",
|
|
"be": "EUR", "es": "EUR", "it": "EUR", "pl": "PLN", "ch": "CHF"}
|
|
|
|
|
|
def parse_adzuna(body: bytes, country: str) -> list[Row]:
|
|
"""One page of Adzuna's `/search`: `{"count", "results": [...]}`. Titles
|
|
and snippets carry `<strong>` highlights. A salary Adzuna estimated
|
|
(`salary_is_predicted`) is not the posting's, so it is dropped."""
|
|
return _adzuna_rows(json.loads(body), country)
|
|
|
|
|
|
def _adzuna_rows(data: dict[str, Any], country: str) -> list[Row]:
|
|
rows = []
|
|
for job in data.get("results") or []:
|
|
title = _text(job.get("title") or "")
|
|
url = (job.get("redirect_url") or "").strip()
|
|
if not title or not _web_url(url):
|
|
continue
|
|
predicted = str(job.get("salary_is_predicted") or "0") == "1"
|
|
rows.append({
|
|
"id": str(job.get("id") or ""),
|
|
"title": title,
|
|
"company": ((job.get("company") or {}).get("display_name") or "").strip(),
|
|
"place": ((job.get("location") or {}).get("display_name") or "").strip(),
|
|
"country": country.upper(),
|
|
"remote": _remote(title),
|
|
"url": url,
|
|
"posted": _parse_iso(job.get("created")),
|
|
"description": _text(job.get("description") or ""),
|
|
"salary_min": None if predicted else job.get("salary_min"),
|
|
"salary_max": None if predicted else job.get("salary_max"),
|
|
"currency": "" if predicted else ADZUNA_COUNTRIES.get(country, ""),
|
|
})
|
|
return rows
|
|
|
|
|
|
def _parse_iso(value: Any) -> datetime | None:
|
|
try:
|
|
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
def search_adzuna(client: httpx.Client, params: dict[str, Any],
|
|
keys: dict[str, Any]) -> tuple[list[Row], bool]:
|
|
"""Every page of one search, until a short page, the stated `count` or
|
|
the page limit. Returns the rows and whether they are all `count` says
|
|
match, or, on an answer with no `count`, whether a page came back short."""
|
|
app_id = keys.get("app_id") or os.environ.get("ADZUNA_APP_ID", "")
|
|
app_key = keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY", "")
|
|
if not (app_id and app_key):
|
|
raise ValueError("adzuna needs sources.search.adzuna.app_id and app_key, "
|
|
"or ADZUNA_APP_ID and ADZUNA_APP_KEY")
|
|
country = str(params.get("country") or "").lower()
|
|
if country not in ADZUNA_COUNTRIES:
|
|
raise ValueError(f"adzuna country {country!r} is not one of "
|
|
f"{', '.join(sorted(ADZUNA_COUNTRIES))}")
|
|
query: dict[str, Any] = {"app_id": app_id, "app_key": app_key,
|
|
"results_per_page": ADZUNA_PAGE, "sort_by": "date",
|
|
"what": str(params.get("q") or "")}
|
|
if params.get("where"):
|
|
query["where"] = params["where"]
|
|
rows: list[Row] = []
|
|
seen: set[str] = set()
|
|
count: Any = None
|
|
ran_out = False
|
|
for page in range(1, _pages(params) + 1):
|
|
url = ADZUNA_SEARCH.format(country=country, page=page)
|
|
data = json.loads(_get(client, url, query))
|
|
results = data.get("results") or []
|
|
count = data.get("count", count)
|
|
seen |= {str(job.get("id") or "") for job in results}
|
|
rows += _adzuna_rows(data, country)
|
|
if len(results) < ADZUNA_PAGE:
|
|
ran_out = True
|
|
break
|
|
if isinstance(count, int) and len(seen) >= count:
|
|
break
|
|
# Adzuna states how many results match; the search is complete when it
|
|
# read that many distinct ones. Without a count, a short page is the end.
|
|
return rows, (len(seen) == count) if isinstance(count, int) else ran_out
|
|
|
|
|
|
_SEARCHES: dict[str, Callable[[httpx.Client, dict[str, Any], dict[str, Any]],
|
|
tuple[list[Row], bool]]] = {
|
|
"jobindex": search_jobindex,
|
|
"adzuna": search_adzuna,
|
|
}
|
|
|
|
|
|
def provider_of(target: str) -> str:
|
|
"""`jobindex:copenhagen-backend` -> `jobindex`, or '' for an unknown one."""
|
|
provider, sep, name = target.partition(":")
|
|
return provider if sep and name.strip() and provider in _SEARCHES else ""
|
|
|
|
|
|
def run_search(target: str, params: dict[str, Any],
|
|
keys: dict[str, Any] | None = None) -> tuple[list[Row], bool]:
|
|
"""One search, outside any pull: for `app searches add`, which runs a
|
|
search once before saving it."""
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
|
f"{', '.join(PROVIDERS)}")
|
|
with httpx.Client() as client:
|
|
return _SEARCHES[provider](client, params, (keys or {}).get(provider) or {})
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class SearchSource(Source):
|
|
name = "search"
|
|
description = "Saved searches on cross-company job boards: Jobindex, Adzuna."
|
|
#: Entries are `{"url": target, "label": name, "params": {...}}`.
|
|
subscription_field = "searches"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="Job searches",
|
|
targets=TargetSpec(mode="editable", kind="search", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# The pull run's marker, handed in by its batch as for `ats`.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
provider = provider_of(target)
|
|
if not provider:
|
|
log.warning("search: %r is not provider:name; skipped", target)
|
|
continue
|
|
try:
|
|
rows, complete = _SEARCHES[provider](
|
|
client, self.params_for(target), self.options.get(provider) or {})
|
|
except Exception as exc:
|
|
log.warning("search: %s failed (%s)", target, exc)
|
|
continue
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
source_id = f"{provider}:{row['id']}"
|
|
seen_ids.append(source_id)
|
|
yield Item(
|
|
source_id=source_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": row["company"],
|
|
"place": row["place"],
|
|
"country": row["country"],
|
|
"remote": row["remote"],
|
|
"salary_min": row["salary_min"],
|
|
"salary_max": row["salary_max"],
|
|
"currency": row["currency"],
|
|
"provider": provider,
|
|
"search": target,
|
|
},
|
|
)
|
|
self.state[f"search:ok:{target}"] = run_marker
|
|
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
|
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|