Find roles by search, not only by company: Jobindex and Adzuna searches

The ats boards only read companies someone listed, and the seed list is
US-centred: 31 of 29,000 open roles were in Denmark. A `search` plugin runs
saved queries on boards that span companies, pulled in the same task as the
boards: Jobindex (Denmark, RSS, no key) and Adzuna (UK and most of the EU,
free key, built to its documented API). Jobnet and Jooble were checked and
left out.

`country` is now filled from the board's location line by a lookup, with EU
for Europe-wide roles, so country gates work without extract. `app companies
prune --in` unsubscribes boards with no open role in a region. The starter
agents and searches are Copenhagen and Europe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-23 19:55:50 +02:00
1 parent bfab68c878
commit 047f7cd8ad
17 files changed
+1293 -72

No files matched your search

+42
View File
@@ -0,0 +1,42 @@
{
"__CLASS__": "Adzuna::API::Response::JobSearchResults",
"count": 3,
"mean": 65000,
"results": [
{
"__CLASS__": "Adzuna::API::Response::Job",
"id": "4412345678",
"title": "Senior <strong>Backend</strong> Engineer (m/w/d) - Remote",
"description": "Build our <strong>backend</strong> services in Python &amp; Go.",
"created": "2026-09-20T08:15:00Z",
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345678?se=x",
"company": {"__CLASS__": "Adzuna::API::Response::Company", "display_name": "Acme GmbH"},
"location": {"__CLASS__": "Adzuna::API::Response::Location",
"display_name": "Berlin, Deutschland", "area": ["Deutschland", "Berlin"]},
"category": {"__CLASS__": "Adzuna::API::Response::Category",
"label": "IT-Stellen", "tag": "it-jobs"},
"salary_min": 70000,
"salary_max": 90000,
"salary_is_predicted": "0",
"contract_time": "full_time"
},
{
"__CLASS__": "Adzuna::API::Response::Job",
"id": "4412345679",
"title": "Platform Engineer",
"description": "Kubernetes and Terraform.",
"created": "2026-09-19T10:00:00Z",
"redirect_url": "https://www.adzuna.de/jobs/land/ad/4412345679",
"company": {"display_name": "Beta AG"},
"location": {"display_name": "Hamburg"},
"salary_min": 64000,
"salary_max": 64000,
"salary_is_predicted": "1"
},
{
"id": "4412345680",
"title": "Refused",
"redirect_url": "javascript:alert(1)"
}
]
}
+125
View File
@@ -0,0 +1,125 @@
<?xml version="1.0" encoding="ISO-8859-1"?>
<rss version="2.0"
xmlns:blogChannel="http://backend.userland.com/blogChannelModule"
>
<channel>
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&#x26;sort=date</link>
<description>Danmarks st&#xF8;rste jobmarked</description>
<language>da</language>
<docs>http://blogs.law.harvard.edu/tech/rss</docs>
<webMaster>support@jobindex.dk (Jobindex)</webMaster>
<ttl>1440</ttl>
<image>
<title>Jobindex - Ledige job i Region Hovedstaden indeholdende mindst et af ordene backend eller developer</title>
<url>https://www.jobindex.dk/img/jobindex130x30gray.gif</url>
<link>https://www.jobindex.dk/jobsoegning/region-hovedstaden?q=backend+developer&#x26;sort=date</link>
</image>
<item>
<title>Dangerous Goods Compliance Manager &#x2013; ensure global safety, DSV</title>
<link>https://www.jobindex.dk/vis-job/h1701165</link>
<description>
&#x3C;div class=&#x22;jobsearch-result&#x22;&#x3E;
&#x3C;div class=&#x22;PaidJob&#x22;&#x3E;
&#x3C;div class=&#x22;PaidJob-inner&#x22;&#x3E;
&#x3C;CENTER style=&#x22;margin-bottom: 15px;&#x22;&#x3E;&#x3C;A HREF=&#x22;https://www.jobindex.dk/vis-job/h1701165&#x22; rel=&#x22;noopener&#x22; target=&#x22;_blank&#x22;&#x3E;&#x3C;img src=&#x22;https://www.jobindex.dk/img/brand/40_spo_top_dsv_1812_20250718.png&#x22; ALT=&#x22;DSV&#x22; style=&#x22;border: 0px; margin: 0;&#x22;&#x3E;&#x3C;/a&#x3E;&#x3C;/CENTER&#x3E;
&#x3C;h4&#x3E;&#x3C;A HREF=&#x22;https://www.jobindex.dk/vis-job/h1701165&#x22; rel=&#x22;noopener&#x22; target=&#x22;_blank&#x22;&#x3E;Dangerous Goods Compliance Manager &#x2013; ensure global safety&#x3C;/a&#x3E;&#x3C;/h4&#x3E;
&#x3C;div class=&#x22;jobad-element-area&#x22;&#x3E;
&#x3C;span class=&#x22;jix_robotjob--area&#x22;&#x3E;Hedehusene&#x3C;/span&#x3E;
&#x3C;/div&#x3E;
&#x3C;p&#x3E;Do you want to minimise workplace accidents and damage to property, while helping ensure that dangerous goods and hazardous materials are handled correctly at our nearly 3,000 locations worldwide?&#x3C;/p&#x3E;&#x3C;p&#x3E;&#x201C;One minute, you&#x2019;ll help a forwarding agent in Holland figure out what to do with a pallet. The next, you&#x2019;ll help shape our strategic decisions after the merger with DB Schenker,&#x201D; says Martin Guldhammer.&#x3C;/p&#x3E;
&#x3C;center&#x3E;&#x3C;A HREF=&#x22;https://www.jobindex.dk/vis-job/h1701165&#x22; rel=&#x22;noopener&#x22; target=&#x22;_blank&#x22;&#x3E;&#x3C;img src=&#x22;https://www.jobindex.dk/api/imagescaler/v1/brand/8_spo_bund_dsv_1812_20250718.png?max_width=525&#x22; alt=&#x22;DSV&#x22; style=&#x22;border: 0px; margin-top: 5px;&#x22;&#x3E;&#x3C;/a&#x3E;&#x3C;/center&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
</description>
<category>Logistik og spedition</category>
<guid isPermaLink="false">https://www.jobindex.dk/1701165</guid>
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
</item>
<item>
<title>Senior Account Executive</title>
<link>https://www.jobindex.dk/vis-job/r14011701</link>
<description>
&#x3C;div class=&#x22;&#x22;&#x3E;
&#x3C;div class=&#x22;jobsearch-result&#x22;&#x3E;
&#x3C;div class=&#x22;jix_robotjob&#x22; data-jobad-tid=&#x22;r14011701&#x22; data-robotjob-abtest-enabled=&#x22;&#x22;&#x3E;
&#x3C;div class=&#x22;jix_robotjob-inner&#x22;&#x3E;
&#x3C;h4&#x3E;&#x3C;A HREF=&#x22;https://www.jobindex.dk/vis-job/r14011701&#x22; rel=&#x22;noopener&#x22; target=&#x22;_blank&#x22;&#x3E;
Senior Account Executive
&#x3C;/a&#x3E;&#x3C;/h4&#x3E;
&#x3C;div class=&#x22;jobad-element-area&#x22;&#x3E;
&#x3C;span class=&#x22;jix_robotjob--area&#x22;&#x3E;K&#xF8;benhavn&#x3C;/span&#x3E;
&#x3C;/div&#x3E;
&#x3C;p&#x3E;All Gravy is the AI-native people operating platform for hospitality and retail.&#x3C;/p&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
</description>
<category>Salg</category>
<guid isPermaLink="false">https://www.jobindex.dk/14011701</guid>
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
</item>
<item>
<title>Marketing Content Creator, Clerk.IO ApS</title>
<link>https://www.jobindex.dk/vis-job/r14011706</link>
<description>
&#x3C;div class=&#x22;&#x22;&#x3E;
&#x3C;div class=&#x22;jobsearch-result&#x22;&#x3E;
&#x3C;div class=&#x22;jix_robotjob&#x22; data-jobad-tid=&#x22;r14011706&#x22; data-robotjob-abtest-enabled=&#x22;&#x22;&#x3E;
&#x3C;div class=&#x22;jix_robotjob-inner&#x22;&#x3E;
&#x3C;h4&#x3E;&#x3C;A HREF=&#x22;https://www.jobindex.dk/vis-job/r14011706&#x22; rel=&#x22;noopener&#x22; target=&#x22;_blank&#x22;&#x3E;
Marketing Content Creator
&#x3C;/a&#x3E;&#x3C;/h4&#x3E;
&#x3C;div class=&#x22;jobad-element-area&#x22;&#x3E;
&#x3C;span class=&#x22;jix_robotjob--area&#x22;&#x3E;K&#xF8;benhavn&#x3C;/span&#x3E;
&#x3C;/div&#x3E;
&#x3C;p&#x3E;Your marketing degree gave you the foundation.&#x3C;/p&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
&#x3C;/div&#x3E;
</description>
<category>Studiepraktik</category>
<category>Marketing</category>
<guid isPermaLink="false">https://www.jobindex.dk/14011706</guid>
<pubDate>Wed, 23 Sep 2026 00:00:00 +0200</pubDate>
</item>
</channel>
</rss>
+5 -2
View File
@@ -23,6 +23,7 @@ from app.storage import Store # noqa: E402
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
def _item(board: str, slug: str, job_id: str) -> Item:
@@ -102,7 +103,8 @@ def test_close_boards_closes_what_a_completed_board_no_longer_lists(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 1, "failed": 1, "roles": 1, "closed": 1}
assert result == {"boards": 1, "searches": 0, "failed": 1, "roles": 1,
"closed": 1}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
@@ -123,5 +125,6 @@ def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
assert result == {"boards": 0, "failed": 1, "roles": 0, "closed": 0}
assert result == {"boards": 0, "searches": 0, "failed": 1, "roles": 0,
"closed": 0}
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
+10 -1
View File
@@ -331,11 +331,20 @@ def test_the_shipped_starter_agents_validate() -> None:
from app import config as config_mod
agents = config_mod.load(ROOT / "config.yaml").agents
assert [a.name for a in agents] == ["Berlin backend", "Remote EU stretch"]
assert [a.name for a in agents] == ["Copenhagen engineering", "Remote from Europe",
"Europe hybrid"]
# Small enough that a prefer gate breaks ties rather than overrides fit.
assert {g.weight for a in agents for g in a.gates if g.mode == "prefer"} == {0.05}
def test_the_shipped_starter_searches_validate() -> None:
from app import config as config_mod
searches = config_mod.load(ROOT / "config.yaml").searches
assert searches and {s.provider for s in searches} == {"jobindex"}
assert all(s.params.get("q") for s in searches)
# ---------------------------------------------------------------- cascade
def test_the_pull_job_runs_pull_extract_embed_then_score(
+262
View File
@@ -0,0 +1,262 @@
"""The `search` plugin, `country_of`, and what the pull and the CLI do with
searches. No network: Jobindex's feed is a trimmed copy of a real one
(2026-09-23), Adzuna's is written from its documented response."""
from __future__ import annotations
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, ClassVar
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from plugins import search # noqa: E402
from plugins.search import SearchSource # noqa: E402
from app import cli # noqa: E402
from app.config import SearchSeed # noqa: E402
from app.jobs import pull # noqa: E402
from app.models import country_of # noqa: E402
from app.storage import Store # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "search"
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "roledesk.db")
# ----------------------------------------------------------------- country
@pytest.mark.parametrize(("place", "code"), [
("Berlin, Germany", "DE"),
("København K", "DK"),
("Kongens Lyngby", "DK"),
("Boston, MA", "US"),
("Berlin, DE", "DE"), # a state code only when nothing else matched
("Remote - US", "US"),
("London | Remote", "GB"),
("Northern Ireland", "GB"), # the longer name wins over "ireland"
("Remote - Europe", "EU"),
("Home based - EMEA", "EU"),
("London, EMEA", "GB"), # a country beats the region it is in
("London or New York", ""),
("Remote", ""),
("", ""),
])
def test_country_of(place: str, code: str) -> None:
assert country_of(place) == code
def test_upsert_fills_country_from_place_unless_the_plugin_says(store: Store) -> None:
def item(source_id: str, **raw: Any) -> Item:
it = Item(source_id=source_id, location="https://x.example/1", title="Engineer",
body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), raw=raw)
it.source = "ats"
return it
store.upsert_role(item("a", place="Copenhagen, Denmark"))
store.upsert_role(item("b", place="Remote", country="EU"))
store.upsert_role(item("c", place="Somewhere"))
assert {r.source_id: r.country for r in store.roles()} == {"a": "DK", "b": "EU", "c": ""}
# ----------------------------------------------------------------- parsers
def test_jobindex_reads_title_company_town_and_date() -> None:
rows = search.parse_jobindex((FIXTURES / "jobindex.xml").read_bytes())
assert [r["id"] for r in rows] == ["h1701165", "r14011701", "r14011706"]
paid, bare, robot = rows
# A paid ad: the company is its logo's alt text, the title its <h4>.
assert paid["company"] == "DSV"
assert paid["title"].startswith("Dangerous Goods Compliance Manager")
assert paid["place"] == "Hedehusene"
assert paid["country"] == "DK"
assert paid["posted"] is not None and paid["posted"].date().isoformat() == "2026-09-23"
assert paid["description"].startswith("Do you want to minimise")
# A crawled ad with no company after the title's comma has none.
assert bare["company"] == ""
assert bare["place"] == "København"
# A crawled ad names its company after the title.
assert robot["title"] == "Marketing Content Creator"
assert robot["company"] == "Clerk.IO ApS"
def test_jobindex_reads_hybrid_and_remote_from_the_town_line() -> None:
assert search._remote("Birkerød, Hybrid position") == "hybrid"
assert search._remote("Nordhavn og mulighed for hjemmearbejde") == "hybrid"
assert search._remote("Remote, Danmark") == "remote"
assert search._remote("Hedehusene") is None
def test_adzuna_strips_highlights_and_drops_predicted_salaries() -> None:
rows = search.parse_adzuna((FIXTURES / "adzuna.json").read_bytes(), "de")
assert [r["id"] for r in rows] == ["4412345678", "4412345679"] # javascript: refused
first, predicted = rows
assert first["title"] == "Senior Backend Engineer (m/w/d) - Remote"
assert first["description"] == "Build our backend services in Python & Go."
assert first["company"] == "Acme GmbH"
assert first["place"] == "Berlin, Deutschland"
assert first["country"] == "DE"
assert first["remote"] == "remote"
assert (first["salary_min"], first["salary_max"], first["currency"]) == (70000, 90000, "EUR")
assert (predicted["salary_min"], predicted["currency"]) == (None, "")
def test_adzuna_refuses_without_a_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("ADZUNA_APP_ID", raising=False)
monkeypatch.delenv("ADZUNA_APP_KEY", raising=False)
with pytest.raises(ValueError, match="app_id"):
search.search_adzuna(None, {"q": "x", "country": "de"}, {}) # type: ignore[arg-type]
def test_a_search_pages_until_a_short_page(monkeypatch: pytest.MonkeyPatch) -> None:
pages: list[int] = []
full = [{"id": str(n)} for n in range(search.JOBINDEX_PAGE)]
def fake_get(client: object, url: str, params: dict[str, Any]) -> bytes:
pages.append(params["page"])
return b"short" if params["page"] == 3 else b"full"
monkeypatch.setattr(search, "_get", fake_get)
monkeypatch.setattr(search, "parse_jobindex",
lambda body: full[:5] if body == b"short" else full)
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 5}, {}) # type: ignore[arg-type]
assert (pages, len(rows), complete) == ([1, 2, 3], 45, True)
pages.clear()
rows, complete = search.search_jobindex(None, {"q": "x", "pages": 2}, {}) # type: ignore[arg-type]
assert (pages, complete) == ([1, 2], False)
def test_provider_of() -> None:
assert search.provider_of("jobindex:copenhagen-backend") == "jobindex"
assert search.provider_of("adzuna:berlin") == "adzuna"
assert search.provider_of("indeed:x") == ""
assert search.provider_of("jobindex:") == ""
# ------------------------------------------------------------------- pull
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[SearchSeed]] = []
path = None
def _stub_search(monkeypatch: pytest.MonkeyPatch, ids: list[str], complete: bool) -> None:
def fake(client: object, params: dict[str, Any],
keys: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
return [{"id": i, "title": f"Engineer {i}", "company": "Acme", "place": "København",
"country": "DK", "remote": None, "url": f"https://x.example/{i}",
"posted": None, "description": "", "salary_min": None,
"salary_max": None, "currency": ""} for i in ids], complete
monkeypatch.setitem(search._SEARCHES, "jobindex", fake)
def _pull(store: Store) -> dict[str, Any]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
for batch in batches:
pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
return pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
def test_a_complete_search_closes_what_it_stopped_listing(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
_stub_search(monkeypatch, ["h1", "h2"], complete=True)
_pull(store)
roles = {r.source_id: r for r in store.roles()}
assert set(roles) == {"jobindex:h1", "jobindex:h2"}
assert roles["jobindex:h1"].country == "DK"
assert roles["jobindex:h1"].subscription_id is not None
_stub_search(monkeypatch, ["h2"], complete=True)
result = _pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
assert (result["searches"], result["closed"]) == (1, 1)
def test_a_search_cut_off_by_its_page_limit_closes_nothing_until_stale(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend"})
_stub_search(monkeypatch, ["h1", "h2"], complete=False)
_pull(store)
_stub_search(monkeypatch, ["h2"], complete=False)
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h1", "jobindex:h2"}
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat()
with store.transaction() as conn:
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = 'jobindex:h1'", (old,))
_pull(store)
assert {r.source_id for r in store.roles()} == {"jobindex:h2"}
def test_seed_searches_imports_once(store: Store) -> None:
config = FakeConfig()
config.searches = [SearchSeed(name="Copenhagen backend", provider="jobindex",
params={"q": "backend", "area": 1})]
assert pull.seed_searches(config, store) == ["Copenhagen backend"] # type: ignore[arg-type]
sub = store.subscriptions(source="search")[0]
assert (sub.target, sub.params) == ("jobindex:copenhagen-backend",
{"q": "backend", "area": 1})
store.unsubscribe(sub.id)
with store.transaction() as conn:
conn.execute("DELETE FROM subscriptions")
assert pull.seed_searches(config, store) == [] # type: ignore[arg-type]
# -------------------------------------------------------------------- cli
def test_prune_unsubscribes_boards_with_no_role_in_the_region(
store: Store, capsys: pytest.CaptureFixture[str]) -> None:
def board(slug: str, *places: str) -> int:
sub = store.add_subscription("ats", f"greenhouse:{slug}", label=slug, company=slug,
board="greenhouse")
for n, place in enumerate(places):
it = Item(source_id=f"greenhouse:{slug}:{n}", location="https://x.example/1",
title=f"Engineer {n}", body="", raw={"company": slug, "place": place})
it.source = "ats"
store.upsert_role(it, sub)
return sub.id
us = board("usonly", "Boston, MA", "Remote - US")
both = board("both", "San Francisco, CA", "London, UK")
vague = board("vague", "Remote", "Hybrid")
cli.prune_companies(store, "europe", apply=False)
assert "usonly" in capsys.readouterr().out
assert len(store.subscriptions(source="ats", fetchable=True)) == 3
cli.prune_companies(store, "europe", apply=True)
fetchable = {s.id for s in store.subscriptions(source="ats", fetchable=True)}
assert fetchable == {both, vague}
assert all(r.subscription_id != us for r in store.roles())
cli.prune_companies(store, "DK,EU", apply=True)
assert {s.id for s in store.subscriptions(source="ats", fetchable=True)} == {vague}
def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None:
_stub_search(monkeypatch, ["h1"], complete=True)
source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen",
"params": {"q": "x"}}]})
items = list(source.fetch())
assert [i.source_id for i in items] == ["jobindex:h1"]
assert items[0].raw["search"] == "jobindex:cph"
assert source.state["search:complete:jobindex:cph"] == "1"
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'