Wire freehire, Jobnet and Akademikernes Jobbank into the CLI, the Sources form, dupes and the docs

Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-10-03 12:24:34 +02:00
1 parent 018a1ad895
commit 6e6e655203
16 files changed
+422 -39

No files matched your search

+24 -4
View File
@@ -46,8 +46,19 @@ if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
def _ids(provider: str, table: str, lead: str) -> str:
"""`lead` and the `id label` pairs of a search reader's `REGIONS`, `MODES` or `REMOTE`
for a help text, or nothing when the reader is not loaded or has no such table."""
from app import jobs
pairs = ", ".join(f"{key or 'empty'} {label}".replace("%", "%%")
for key, label in jobs.sources.provider_table(provider, table).items())
return f"{lead}{pairs}" if pairs else ""
def build_parser() -> argparse.ArgumentParser:
from app.models import HAND_REASONS
from plugins import readers
parser = argparse.ArgumentParser(
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
@@ -128,17 +139,25 @@ def build_parser() -> argparse.ArgumentParser:
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull (or update it)")
search_add.add_argument("provider", choices=["jobindex", "adzuna", "platsbanken"])
search_add.add_argument("provider", choices=list(readers.PROVIDERS))
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
help="the query, in the provider's own syntax")
search_add.add_argument("--area", type=int, default=0,
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
search_add.add_argument("--country", default="",
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch; "
"freehire countries, comma-separated ISO codes: dk,se, "
"empty all")
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
search_add.add_argument("--region", default="",
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of Sweden")
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of "
"Sweden" + _ids("jobnet", "REGIONS", "; jobnet region")
+ _ids("jobbank", "REGIONS", "; jobbank amt"))
search_add.add_argument("--remote", default="",
help="freehire work mode" + _ids("freehire", "MODES", ": ")
+ _ids("jobbank", "REMOTE", "; jobbank remote work: ")
+ ", empty any")
search_add.add_argument("--pages", type=int, default=0,
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
@@ -608,7 +627,8 @@ def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace
try:
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
country=args.country, where=args.where,
region=args.region, pages=args.pages)
region=args.region, remote=args.remote,
pages=args.pages)
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
params)
except ValueError as exc:
+3 -1
View File
@@ -166,7 +166,9 @@ class SearchSeed(Section):
"""A saved search as `config.yaml` lists it: one `search` subscription,
imported once like the agents. `params` are the provider's own, see
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
`where` and `pages` for adzuna; `q` and `region` for platsbanken; `q`,
`country` and `remote` for freehire; `q` and `region` for jobnet; `q`,
`region` and `remote` for jobbank. The provider is any reader of
`scope="search"`, so a new one needs no edit here."""
name: str
+18 -5
View File
@@ -37,18 +37,31 @@ the strings boards really write and the storage layer only feeds it rows.
from __future__ import annotations
import re
import sys
from collections.abc import Iterable
from dataclasses import dataclass
from functools import lru_cache
from pathlib import Path
from typing import NamedTuple
from app import places
from app.models import country_of
#: Hosts that search across companies rather than run a company's own board.
#: A board closes a role reliably and carries the full text, so it is kept
#: over a copy found here.
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
def is_search_host(host: str) -> bool:
"""Whether `host` is a provider that searches across companies rather than
a company's own board: a reader of `scope="search"`. A board closes a role
reliably and carries the full text, so it is kept over a copy found here.
`plugins` is a top-level folder beside `app`, reached lazily, with the
repo root put on the path when it is not there yet, as
`app.models._declared_posted` does."""
try:
from plugins import readers
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import readers
return host in readers.PROVIDERS
@dataclass(frozen=True)
@@ -101,7 +114,7 @@ def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
board over a search provider, then one with a description, then the
earliest `posted_at` (none last), then the smallest `uid`, so the choice
is the same on every run."""
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
return (is_search_host(c.host), not c.has_description, not c.posted_at,
c.posted_at or "", c.uid)
+13 -8
View File
@@ -26,9 +26,6 @@ from plugins import readers
if TYPE_CHECKING:
from app.jobs.discover import Discovery
#: The providers, from the reader modules that declare `scope="search"`.
PROVIDERS = tuple(readers.PROVIDERS)
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
@@ -356,20 +353,28 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
"problems": problems}
def provider_table(provider: str, name: str) -> dict[str, str]:
"""The table `name` (`REGIONS`, `MODES` or `REMOTE`) of a search reader, as
the form and the CLI list its values; empty when the provider is not
loaded or declares none, so a page works before a reader is merged."""
return dict(getattr(readers.PROVIDERS.get(provider), name, None) or {})
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
where: str = "", region: str = "", remote: str = "",
pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads: those of its module's `PARAMS` whose filter dimension
`READS.filters` declares, cleaned by the module's `clean_params`, which
refuses a value the provider would not accept."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if provider not in readers.PROVIDERS:
raise ValueError(f"provider is one of {', '.join(readers.PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
module = readers.PROVIDERS[provider]
declared = module.READS.filters.declared()
given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country,
"where": where, "region": region}
"where": where, "region": region, "remote": remote}
wanted = {name: given[name] for name, dimension in module.PARAMS.items()
if dimension in declared}
params = module.clean_params(wanted)
@@ -379,7 +384,7 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
"""Whether `provider` can run: every provider but adzuna always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
+20 -7
View File
@@ -1,7 +1,8 @@
/* The Job boards section of Sources: one query on one board per row, on
* Jobindex, Adzuna or Platsbanken. The UI says "job board" for what the API and the CLI
* call a search (`/api/searches`, `app searches`); company boards are
* companies.js's section, since a board is a company's own, not a query's.
* Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank.
* The UI says "job board" for what the API and the CLI call a search
* (`/api/searches`, `app searches`); company boards are companies.js's
* section, since a board is a company's own, not a query's.
*
* Edit turns a row into the form from the page's `<template>`, in place, and
* New job board puts a blank form on top of the list, as agents.js does for
@@ -172,7 +173,13 @@
$("#s-area", f).value = String(p.area || (source ? 0 : 1));
$("#s-country", f).value = p.country || "de";
$("#s-where", f).value = p.where || "";
$("#s-region", f).value = p.region || "";
const of = (provider) => source && source.provider === provider;
$("#s-region", f).value = of("platsbanken") ? p.region || "" : "";
$("#s-countries", f).value = of("freehire") ? p.country || "" : "";
$("#s-mode", f).value = of("freehire") ? p.remote || "" : "";
$("#s-region-jobnet", f).value = of("jobnet") ? String(p.region || "") : "";
$("#s-region-jobbank", f).value = of("jobbank") ? String(p.region || "") : "";
$("#s-remote-jobbank", f).value = of("jobbank") ? p.remote || "" : "";
$("#s-pages", f).value = String(p.pages || 10);
showProvider(f);
f.addEventListener("input", () => { f.dataset.dirty = "true"; });
@@ -209,15 +216,21 @@
$("#s-q", editor).focus({ preventScroll: true });
}
// Region, country and remote word each have one control per provider that
// reads them; the body takes the shown provider's, and "" where it has none.
function searchBody(f) {
const provider = $("#s-provider", f).value;
const own = (controls) => (controls[provider] ? $(controls[provider], f).value.trim() : "");
return {
provider: $("#s-provider", f).value,
provider,
name: $("#s-name", f).value.trim(),
q: $("#s-q", f).value.trim(),
area: Number($("#s-area", f).value) || 0,
country: $("#s-country", f).value,
country: own({ adzuna: "#s-country", freehire: "#s-countries" }),
where: $("#s-where", f).value.trim(),
region: $("#s-region", f).value,
region: own({ platsbanken: "#s-region", jobnet: "#s-region-jobnet",
jobbank: "#s-region-jobbank" }),
remote: own({ freehire: "#s-mode", jobbank: "#s-remote-jobbank" }),
pages: Number($("#s-pages", f).value) || 0,
};
}
+28 -2
View File
@@ -19,7 +19,7 @@
top of the list on New, as agents.js does for the Search agents cards. #}
<section class="card" id="job-boards" data-section>
<h2>Job boards</h2>
<p class="lede">A query on Jobindex, Adzuna or Platsbanken finds roles at any company. Switching one off closes its roles until you switch it back on.</p>
<p class="lede">A query on Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank finds roles at any company. Switching one off closes its roles until you switch it back on.</p>
<div class="agents-head">
<p id="searches-count" class="count" role="status"></p>
@@ -38,6 +38,7 @@
<option value="jobindex">Jobindex (Denmark)</option>
<option value="adzuna">Adzuna (UK and EU)</option>
<option value="platsbanken">Platsbanken (Sweden)</option>
{% for value, label in providers %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label>Name <input type="text" id="s-name" required placeholder="Copenhagen backend"></label>
</div>
@@ -45,7 +46,10 @@
<input type="text" id="s-q" required spellcheck="false">
<span class="hint" data-provider="jobindex">Jobindex syntax: words are OR, <code>"a b"</code> is a phrase, <code>+word</code> is required.</span>
<span class="hint" data-provider="adzuna" hidden>Every word must appear.</span>
<span class="hint" data-provider="platsbanken" hidden>Free text; every word counts towards the match.</span></label>
<span class="hint" data-provider="platsbanken" hidden>Free text; every word counts towards the match.</span>
<span class="hint" data-provider="freehire" hidden>Words are OR over title, company, text and place, so a query is noisy; narrow it by country.</span>
<span class="hint" data-provider="jobnet" hidden>A loose match: words, no phrases.</span>
<span class="hint" data-provider="jobbank" hidden>Words are OR.</span></label>
<div class="form-row">
<label data-provider="jobindex">Area
<select id="s-area">
@@ -61,6 +65,28 @@
<option value="">All of Sweden</option>
{% for value, label in regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="freehire" hidden>Countries <input type="text" id="s-countries" placeholder="dk,se" spellcheck="false">
<span class="hint">ISO codes, comma-separated; empty for all</span></label>
<label data-provider="freehire" hidden>Work mode
<select id="s-mode">
<option value="">Any</option>
{% for value, label in modes %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="jobnet" hidden>Region
<select id="s-region-jobnet">
<option value="">All of Denmark</option>
{% for value, label in jobnet_regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="jobbank" hidden>Region
<select id="s-region-jobbank">
<option value="">All of Denmark</option>
{% for value, label in jobbank_regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="jobbank" hidden>Remote work
<select id="s-remote-jobbank">
<option value="">Any</option>
{% for value, label in jobbank_remote %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label>Pages
<input type="number" id="s-pages" min="1" max="10" value="10">
<span class="hint">per pull, up to 10</span></label>
+3 -1
View File
@@ -1043,6 +1043,7 @@ class SearchBody(BaseModel):
country: str = ""
where: str = ""
region: str = ""
remote: str = ""
pages: Annotated[int, Field(ge=0, le=10)] = 0
@@ -1050,7 +1051,8 @@ def _params(body: SearchBody) -> dict[str, Any]:
try:
return jobs.sources.search_params(body.provider, body.q, area=body.area,
country=body.country, where=body.where,
region=body.region, pages=body.pages)
region=body.region, remote=body.remote,
pages=body.pages)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
+28 -3
View File
@@ -27,8 +27,11 @@ keeps working.
functions are the classifier's vocabulary and the hosts are what it parses,
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
app-agnostic and never reaches back into `app`. The
Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page
come from the search readers in `plugins/readers/` for the same reason.
Jobindex areas, Adzuna countries, the regions of Platsbanken, Jobnet and
Jobbank, freehire's work modes and Jobbank's remote words on the Sources page
come from the search readers in `plugins/readers/` for the same reason. The
last four and the select's other providers are read from `readers.PROVIDERS`,
so the page works whether or not a provider's module is loaded.
"""
from __future__ import annotations
@@ -41,6 +44,7 @@ from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
from app import jobs
from app.models import LADDER, REMOTE, RUNGS, STATUSES
from app.web.deps import StoreDep
@@ -48,6 +52,7 @@ _ROOT = Path(__file__).resolve().parents[2]
if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
from plugins import readers # noqa: E402
from plugins.ats import FUNCTIONS, HOSTS # noqa: E402
from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402
from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402
@@ -110,6 +115,25 @@ HOST_OPTIONS = list(HOSTS.items())
LADDER_JSON = {"rungs": list(RUNGS), "levels": {k: list(v) for k, v in LADDER.items()}}
def search_options() -> dict[str, list[tuple[str, str]]]:
"""The `value, label` options the Sources form's providers add to its
selects, read from `readers.PROVIDERS` on each request: the providers the
template does not name itself, and the regions of Jobnet and Jobbank,
freehire's work modes and Jobbank's remote words, each in the reader's
order and without its empty value, which the form puts first as "all". All
empty for a provider whose module is not loaded."""
def options(provider: str, table: str) -> list[tuple[str, str]]:
return [(value, label) for value, label in jobs.sources.provider_table(provider, table).items()
if value]
return {"providers": [(n, m.LABEL) for n, m in readers.PROVIDERS.items()
if n not in ("jobindex", "adzuna", "platsbanken")],
"jobnet_regions": options("jobnet", "REGIONS"),
"jobbank_regions": options("jobbank", "REGIONS"),
"modes": options("freehire", "MODES"),
"jobbank_remote": options("jobbank", "REMOTE")}
def asset(path: str) -> str:
"""A static URL stamped with the file's modification time, so a changed
file is a different URL and a cached stale copy is never the answer."""
@@ -180,7 +204,8 @@ def agents(request: Request) -> HTMLResponse:
@router.get("/settings/sources", response_class=HTMLResponse)
def sources(request: Request) -> HTMLResponse:
return page(request, "settings/sources.html", areas=AREA_OPTIONS,
countries=COUNTRY_OPTIONS, regions=REGION_OPTIONS, hosts=HOST_OPTIONS)
countries=COUNTRY_OPTIONS, regions=REGION_OPTIONS, hosts=HOST_OPTIONS,
**search_options())
@router.get("/settings/account", response_class=HTMLResponse)