From 6e6e6552039c599d25187400a20877812119854b Mon Sep 17 00:00:00 2001 From: Wolf Date: Sat, 3 Oct 2026 12:24:34 +0200 Subject: [PATCH] Wire freehire, Jobnet and Akademikernes Jobbank into the CLI, the Sources form, dupes and the docs Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 9 ++-- app/cli.py | 28 +++++++++-- app/config.py | 4 +- app/dupes.py | 23 +++++++-- app/jobs/sources.py | 21 +++++---- app/static/sources.js | 27 ++++++++--- app/templates/settings/sources.html | 30 +++++++++++- app/web/api.py | 4 +- app/web/pages.py | 31 +++++++++++-- config.example.yaml | 10 +++- plugins/README.md | 52 ++++++++++++++++++++- tests/conftest.py | 72 +++++++++++++++++++++++++++++ tests/test_dupes.py | 12 +++++ tests/test_dupes_storage.py | 15 ++++++ tests/test_sources.py | 72 ++++++++++++++++++++++++++++- tests/test_web.py | 51 ++++++++++++++++++++ 16 files changed, 422 insertions(+), 39 deletions(-) create mode 100644 tests/conftest.py diff --git a/README.md b/README.md index 87a3fc4..162b6f3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # roledesk -A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and nine smaller hosts, one module each in `plugins/readers/`) and runs saved searches on boards that span companies (Jobindex, Adzuna, Platsbanken), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. +A job-search engine for one person. It reads the public job boards of the companies you follow (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor, Workday, SuccessFactors, Oracle, Phenom, Eightfold and nine smaller hosts, one module each in `plugins/readers/`) and runs saved searches on boards that span companies (Jobindex, Adzuna, Platsbanken, freehire, Jobnet, Akademikernes Jobbank), keeps every posting with when it was first seen and when it closed, scores the open roles for each of your search agents, and tracks the applications you make. Built on the [boilerplate](../boilerplate): its job queue and scheduler (`pipeline/`), password (`auth/`) and deploy kit (`deploy/`), plus the plugin contract and tools from news (`desk/`). FastAPI, Jinja2, SQLite, `uv`, no build step. @@ -22,6 +22,9 @@ The Sources sub-tab does all of this; the same actions exist on the command line ```bash uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1 uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin +uv run app searches add freehire "Head of data" --q "head of data" --country dk +uv run app searches add jobnet "Data science" --q "data science" --region HovedstadenOgBornholm +uv run app searches add jobbank "Data director" --q "data director" --region 2 uv run app searches list uv run app companies discover "Pleo" pleo.io --add # find its board on its own site, then follow it uv run app companies set-board lunar greenhouse:lunar # attach a board to a company, or replace the one it has @@ -41,7 +44,7 @@ The seed is a hand-picked list of European tech employers, not a list of boards: `app stages` shows or switches extract, summarise and enrich, `app seniority` shows per host how often the stated level equals the model's, `app dupes` recomputes which open roles are copies of one posting, and `app prep` maps the host words and folds the copies without a pull. -A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. A Platsbanken query is free text and `--region` the id of a län (`CaRE_1nn_cSU` is Skåne). An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles, and asks first when there are open roles to lose; switching it back on, or saving a search, queues a pull right after, so the ones it still lists reopen without waiting for the schedule. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. +A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. A Platsbanken query is free text and `--region` the id of a län (`CaRE_1nn_cSU` is Skåne). An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. A freehire query is free text over about 50 hosts whose words are OR over title, company, text and place, so it is noisy: `--country` takes comma-separated ISO codes (`dk,se`) and `--remote` a work mode. A Jobnet query is a loose match without phrases and `--region` one of its five regions (`HovedstadenOgBornholm`, `Midtjylland`, `Syddanmark`, `OevrigeSjaelland`, `Nordjylland`). An Akademikernes Jobbank query is OR words, `--region` its `amt` id (2 is Storkøbenhavn) and `--remote` `helt` or `delvist`. A freehire role whose employer is a board roledesk reads is a copy of that board's role and is hidden behind it. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles, and asks first when there are open roles to lose; switching it back on, or saving a search, queues a pull right after, so the ones it still lists reopen without waiting for the schedule. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. ## The tabs @@ -58,7 +61,7 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic | | | |---|---| | **Search agents** | a card per agent, newest first: name, an On/Off switch, its prompts, its rules in words, and its weights. Edit turns the card into the form in place (name, prompts as rows, a Where block for the location rules, the other require/any/prefer rules as rows, fetch terms, weights and limits); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it | -| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex, Adzuna or Platsbanken: new, try, save, edit, remove), company boards (every company, its board or the host it cannot be read from, its open roles and its affinity for the picked agent, what the pull asks its board for, as a reach line such as "follows agents: DK, SE, remote · all roles", the board's size with a Recheck, the All countries and All roles overrides, following a company, setting or changing a company's board), runs (Run the pull now, the switches of extract, summarise and enrich, what is running, the schedule, and the Problems log) and unmapped (the place texts no place resolves and the host words no table maps, to copy) | +| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank: new, try, save, edit, remove), company boards (every company, its board or the host it cannot be read from, its open roles and its affinity for the picked agent, what the pull asks its board for, as a reach line such as "follows agents: DK, SE, remote · all roles", the board's size with a Recheck, the All countries and All roles overrides, following a company, setting or changing a company's board), runs (Run the pull now, the switches of extract, summarise and enrich, what is running, the schedule, and the Problems log) and unmapped (the place texts no place resolves and the host words no table maps, to copy) | | **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else | Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed. diff --git a/app/cli.py b/app/cli.py index fc53f7b..b88d6f1 100644 --- a/app/cli.py +++ b/app/cli.py @@ -46,8 +46,19 @@ if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) +def _ids(provider: str, table: str, lead: str) -> str: + """`lead` and the `id label` pairs of a search reader's `REGIONS`, `MODES` or `REMOTE` + for a help text, or nothing when the reader is not loaded or has no such table.""" + from app import jobs + + pairs = ", ".join(f"{key or 'empty'} {label}".replace("%", "%%") + for key, label in jobs.sources.provider_table(provider, table).items()) + return f"{lead}{pairs}" if pairs else "" + + def build_parser() -> argparse.ArgumentParser: from app.models import HAND_REASONS + from plugins import readers parser = argparse.ArgumentParser( prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.") @@ -128,17 +139,25 @@ def build_parser() -> argparse.ArgumentParser: searches_sub = searches.add_subparsers(dest="action", required=True) search_add = searches_sub.add_parser( "add", help="run a search once, then save it for every pull (or update it)") - search_add.add_argument("provider", choices=["jobindex", "adzuna", "platsbanken"]) + search_add.add_argument("provider", choices=list(readers.PROVIDERS)) search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"') search_add.add_argument("--q", required=True, help="the query, in the provider's own syntax") search_add.add_argument("--area", type=int, default=0, help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus") search_add.add_argument("--country", default="", - help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch") + help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch; " + "freehire countries, comma-separated ISO codes: dk,se, " + "empty all") search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin") search_add.add_argument("--region", default="", - help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of Sweden") + help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of " + "Sweden" + _ids("jobnet", "REGIONS", "; jobnet region") + + _ids("jobbank", "REGIONS", "; jobbank amt")) + search_add.add_argument("--remote", default="", + help="freehire work mode" + _ids("freehire", "MODES", ": ") + + _ids("jobbank", "REMOTE", "; jobbank remote work: ") + + ", empty any") search_add.add_argument("--pages", type=int, default=0, help="result pages per pull (default 5, at most 10)") searches_sub.add_parser("list", help="every saved search and its open roles") @@ -608,7 +627,8 @@ def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace try: params = jobs.sources.search_params(args.provider, args.q, area=args.area, country=args.country, where=args.where, - region=args.region, pages=args.pages) + region=args.region, remote=args.remote, + pages=args.pages) sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name, params) except ValueError as exc: diff --git a/app/config.py b/app/config.py index bc98deb..03b4a99 100644 --- a/app/config.py +++ b/app/config.py @@ -166,7 +166,9 @@ class SearchSeed(Section): """A saved search as `config.yaml` lists it: one `search` subscription, imported once like the agents. `params` are the provider's own, see plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`, - `where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of + `where` and `pages` for adzuna; `q` and `region` for platsbanken; `q`, + `country` and `remote` for freehire; `q` and `region` for jobnet; `q`, + `region` and `remote` for jobbank. The provider is any reader of `scope="search"`, so a new one needs no edit here.""" name: str diff --git a/app/dupes.py b/app/dupes.py index 5d35e64..d828359 100644 --- a/app/dupes.py +++ b/app/dupes.py @@ -37,18 +37,31 @@ the strings boards really write and the storage layer only feeds it rows. from __future__ import annotations import re +import sys from collections.abc import Iterable from dataclasses import dataclass from functools import lru_cache +from pathlib import Path from typing import NamedTuple from app import places from app.models import country_of -#: Hosts that search across companies rather than run a company's own board. -#: A board closes a role reliably and carries the full text, so it is kept -#: over a copy found here. -SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"}) + +def is_search_host(host: str) -> bool: + """Whether `host` is a provider that searches across companies rather than + a company's own board: a reader of `scope="search"`. A board closes a role + reliably and carries the full text, so it is kept over a copy found here. + `plugins` is a top-level folder beside `app`, reached lazily, with the + repo root put on the path when it is not there yet, as + `app.models._declared_posted` does.""" + try: + from plugins import readers + except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from plugins import readers + + return host in readers.PROVIDERS @dataclass(frozen=True) @@ -101,7 +114,7 @@ def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]: board over a search provider, then one with a description, then the earliest `posted_at` (none last), then the smallest `uid`, so the choice is the same on every run.""" - return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at, + return (is_search_host(c.host), not c.has_description, not c.posted_at, c.posted_at or "", c.uid) diff --git a/app/jobs/sources.py b/app/jobs/sources.py index ef495b7..ec32aae 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -26,9 +26,6 @@ from plugins import readers if TYPE_CHECKING: from app.jobs.discover import Discovery -#: The providers, from the reader modules that declare `scope="search"`. -PROVIDERS = tuple(readers.PROVIDERS) - def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: """Switch one board or search on or off. Off closes its open roles and @@ -356,20 +353,28 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]: "problems": problems} +def provider_table(provider: str, name: str) -> dict[str, str]: + """The table `name` (`REGIONS`, `MODES` or `REMOTE`) of a search reader, as + the form and the CLI list its values; empty when the provider is not + loaded or declares none, so a page works before a reader is merged.""" + return dict(getattr(readers.PROVIDERS.get(provider), name, None) or {}) + + def search_params(provider: str, q: str, *, area: int = 0, country: str = "", - where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]: + where: str = "", region: str = "", remote: str = "", + pages: int = 0) -> dict[str, Any]: """The `params` a search subscription stores, with only the keys its provider reads: those of its module's `PARAMS` whose filter dimension `READS.filters` declares, cleaned by the module's `clean_params`, which refuses a value the provider would not accept.""" - if provider not in PROVIDERS: - raise ValueError(f"provider is one of {', '.join(PROVIDERS)}") + if provider not in readers.PROVIDERS: + raise ValueError(f"provider is one of {', '.join(readers.PROVIDERS)}") if not q.strip(): raise ValueError("a search needs a query") module = readers.PROVIDERS[provider] declared = module.READS.filters.declared() given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country, - "where": where, "region": region} + "where": where, "region": region, "remote": remote} wanted = {name: given[name] for name, dimension in module.PARAMS.items() if dimension in declared} params = module.clean_params(wanted) @@ -379,7 +384,7 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "", def has_key(config: Config, provider: str) -> bool: - """Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the + """Whether `provider` can run: every provider but adzuna always, adzuna with a key in the config or the environment.""" if provider != "adzuna": return True diff --git a/app/static/sources.js b/app/static/sources.js index b71756a..7172f87 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -1,7 +1,8 @@ /* The Job boards section of Sources: one query on one board per row, on - * Jobindex, Adzuna or Platsbanken. The UI says "job board" for what the API and the CLI - * call a search (`/api/searches`, `app searches`); company boards are - * companies.js's section, since a board is a company's own, not a query's. + * Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank. + * The UI says "job board" for what the API and the CLI call a search + * (`/api/searches`, `app searches`); company boards are companies.js's + * section, since a board is a company's own, not a query's. * * Edit turns a row into the form from the page's `