Wire freehire, Jobnet and Akademikernes Jobbank into the CLI, the Sources form, dupes and the docs
Provider names, the CLI's choices and the dupes rule's search hosts now come from readers.PROVIDERS, and the regions, work modes and remote words from the modules' REGIONS, MODES and REMOTE, read with getattr so main works before and after the readers merge. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
018a1ad895
commit
6e6e655203
16 files changed
+422
-39
No files matched your search
+24
-4
@@ -46,8 +46,19 @@ if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
|
||||
def _ids(provider: str, table: str, lead: str) -> str:
|
||||
"""`lead` and the `id label` pairs of a search reader's `REGIONS`, `MODES` or `REMOTE`
|
||||
for a help text, or nothing when the reader is not loaded or has no such table."""
|
||||
from app import jobs
|
||||
|
||||
pairs = ", ".join(f"{key or 'empty'} {label}".replace("%", "%%")
|
||||
for key, label in jobs.sources.provider_table(provider, table).items())
|
||||
return f"{lead}{pairs}" if pairs else ""
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
from app.models import HAND_REASONS
|
||||
from plugins import readers
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
|
||||
@@ -128,17 +139,25 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
search_add = searches_sub.add_parser(
|
||||
"add", help="run a search once, then save it for every pull (or update it)")
|
||||
search_add.add_argument("provider", choices=["jobindex", "adzuna", "platsbanken"])
|
||||
search_add.add_argument("provider", choices=list(readers.PROVIDERS))
|
||||
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
|
||||
search_add.add_argument("--q", required=True,
|
||||
help="the query, in the provider's own syntax")
|
||||
search_add.add_argument("--area", type=int, default=0,
|
||||
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
|
||||
search_add.add_argument("--country", default="",
|
||||
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
|
||||
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch; "
|
||||
"freehire countries, comma-separated ISO codes: dk,se, "
|
||||
"empty all")
|
||||
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
|
||||
search_add.add_argument("--region", default="",
|
||||
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of Sweden")
|
||||
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of "
|
||||
"Sweden" + _ids("jobnet", "REGIONS", "; jobnet region")
|
||||
+ _ids("jobbank", "REGIONS", "; jobbank amt"))
|
||||
search_add.add_argument("--remote", default="",
|
||||
help="freehire work mode" + _ids("freehire", "MODES", ": ")
|
||||
+ _ids("jobbank", "REMOTE", "; jobbank remote work: ")
|
||||
+ ", empty any")
|
||||
search_add.add_argument("--pages", type=int, default=0,
|
||||
help="result pages per pull (default 5, at most 10)")
|
||||
searches_sub.add_parser("list", help="every saved search and its open roles")
|
||||
@@ -608,7 +627,8 @@ def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace
|
||||
try:
|
||||
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
|
||||
country=args.country, where=args.where,
|
||||
region=args.region, pages=args.pages)
|
||||
region=args.region, remote=args.remote,
|
||||
pages=args.pages)
|
||||
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
|
||||
params)
|
||||
except ValueError as exc:
|
||||
|
||||
+3
-1
@@ -166,7 +166,9 @@ class SearchSeed(Section):
|
||||
"""A saved search as `config.yaml` lists it: one `search` subscription,
|
||||
imported once like the agents. `params` are the provider's own, see
|
||||
plugins/search.py: `q`, `area` and `pages` for jobindex; `q`, `country`,
|
||||
`where` and `pages` for adzuna; `q` and `region` for platsbanken. The provider is any reader of
|
||||
`where` and `pages` for adzuna; `q` and `region` for platsbanken; `q`,
|
||||
`country` and `remote` for freehire; `q` and `region` for jobnet; `q`,
|
||||
`region` and `remote` for jobbank. The provider is any reader of
|
||||
`scope="search"`, so a new one needs no edit here."""
|
||||
|
||||
name: str
|
||||
|
||||
+18
-5
@@ -37,18 +37,31 @@ the strings boards really write and the storage layer only feeds it rows.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
from app import places
|
||||
from app.models import country_of
|
||||
|
||||
#: Hosts that search across companies rather than run a company's own board.
|
||||
#: A board closes a role reliably and carries the full text, so it is kept
|
||||
#: over a copy found here.
|
||||
SEARCH_HOSTS: frozenset[str] = frozenset({"jobindex", "adzuna", "platsbanken"})
|
||||
|
||||
def is_search_host(host: str) -> bool:
|
||||
"""Whether `host` is a provider that searches across companies rather than
|
||||
a company's own board: a reader of `scope="search"`. A board closes a role
|
||||
reliably and carries the full text, so it is kept over a copy found here.
|
||||
`plugins` is a top-level folder beside `app`, reached lazily, with the
|
||||
repo root put on the path when it is not there yet, as
|
||||
`app.models._declared_posted` does."""
|
||||
try:
|
||||
from plugins import readers
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import readers
|
||||
|
||||
return host in readers.PROVIDERS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -101,7 +114,7 @@ def keep_order(c: Copy) -> tuple[bool, bool, bool, str, str]:
|
||||
board over a search provider, then one with a description, then the
|
||||
earliest `posted_at` (none last), then the smallest `uid`, so the choice
|
||||
is the same on every run."""
|
||||
return (c.host in SEARCH_HOSTS, not c.has_description, not c.posted_at,
|
||||
return (is_search_host(c.host), not c.has_description, not c.posted_at,
|
||||
c.posted_at or "", c.uid)
|
||||
|
||||
|
||||
|
||||
+13
-8
@@ -26,9 +26,6 @@ from plugins import readers
|
||||
if TYPE_CHECKING:
|
||||
from app.jobs.discover import Discovery
|
||||
|
||||
#: The providers, from the reader modules that declare `scope="search"`.
|
||||
PROVIDERS = tuple(readers.PROVIDERS)
|
||||
|
||||
|
||||
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
|
||||
"""Switch one board or search on or off. Off closes its open roles and
|
||||
@@ -356,20 +353,28 @@ def seed_companies(store: Store, path: str | Path) -> dict[str, Any]:
|
||||
"problems": problems}
|
||||
|
||||
|
||||
def provider_table(provider: str, name: str) -> dict[str, str]:
|
||||
"""The table `name` (`REGIONS`, `MODES` or `REMOTE`) of a search reader, as
|
||||
the form and the CLI list its values; empty when the provider is not
|
||||
loaded or declares none, so a page works before a reader is merged."""
|
||||
return dict(getattr(readers.PROVIDERS.get(provider), name, None) or {})
|
||||
|
||||
|
||||
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
where: str = "", region: str = "", pages: int = 0) -> dict[str, Any]:
|
||||
where: str = "", region: str = "", remote: str = "",
|
||||
pages: int = 0) -> dict[str, Any]:
|
||||
"""The `params` a search subscription stores, with only the keys its
|
||||
provider reads: those of its module's `PARAMS` whose filter dimension
|
||||
`READS.filters` declares, cleaned by the module's `clean_params`, which
|
||||
refuses a value the provider would not accept."""
|
||||
if provider not in PROVIDERS:
|
||||
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
|
||||
if provider not in readers.PROVIDERS:
|
||||
raise ValueError(f"provider is one of {', '.join(readers.PROVIDERS)}")
|
||||
if not q.strip():
|
||||
raise ValueError("a search needs a query")
|
||||
module = readers.PROVIDERS[provider]
|
||||
declared = module.READS.filters.declared()
|
||||
given: dict[str, Any] = {"q": q.strip(), "area": area, "country": country,
|
||||
"where": where, "region": region}
|
||||
"where": where, "region": region, "remote": remote}
|
||||
wanted = {name: given[name] for name, dimension in module.PARAMS.items()
|
||||
if dimension in declared}
|
||||
params = module.clean_params(wanted)
|
||||
@@ -379,7 +384,7 @@ def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
|
||||
|
||||
|
||||
def has_key(config: Config, provider: str) -> bool:
|
||||
"""Whether `provider` can run: jobindex and platsbanken always, adzuna with a key in the
|
||||
"""Whether `provider` can run: every provider but adzuna always, adzuna with a key in the
|
||||
config or the environment."""
|
||||
if provider != "adzuna":
|
||||
return True
|
||||
|
||||
+20
-7
@@ -1,7 +1,8 @@
|
||||
/* The Job boards section of Sources: one query on one board per row, on
|
||||
* Jobindex, Adzuna or Platsbanken. The UI says "job board" for what the API and the CLI
|
||||
* call a search (`/api/searches`, `app searches`); company boards are
|
||||
* companies.js's section, since a board is a company's own, not a query's.
|
||||
* Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank.
|
||||
* The UI says "job board" for what the API and the CLI call a search
|
||||
* (`/api/searches`, `app searches`); company boards are companies.js's
|
||||
* section, since a board is a company's own, not a query's.
|
||||
*
|
||||
* Edit turns a row into the form from the page's `<template>`, in place, and
|
||||
* New job board puts a blank form on top of the list, as agents.js does for
|
||||
@@ -172,7 +173,13 @@
|
||||
$("#s-area", f).value = String(p.area || (source ? 0 : 1));
|
||||
$("#s-country", f).value = p.country || "de";
|
||||
$("#s-where", f).value = p.where || "";
|
||||
$("#s-region", f).value = p.region || "";
|
||||
const of = (provider) => source && source.provider === provider;
|
||||
$("#s-region", f).value = of("platsbanken") ? p.region || "" : "";
|
||||
$("#s-countries", f).value = of("freehire") ? p.country || "" : "";
|
||||
$("#s-mode", f).value = of("freehire") ? p.remote || "" : "";
|
||||
$("#s-region-jobnet", f).value = of("jobnet") ? String(p.region || "") : "";
|
||||
$("#s-region-jobbank", f).value = of("jobbank") ? String(p.region || "") : "";
|
||||
$("#s-remote-jobbank", f).value = of("jobbank") ? p.remote || "" : "";
|
||||
$("#s-pages", f).value = String(p.pages || 10);
|
||||
showProvider(f);
|
||||
f.addEventListener("input", () => { f.dataset.dirty = "true"; });
|
||||
@@ -209,15 +216,21 @@
|
||||
$("#s-q", editor).focus({ preventScroll: true });
|
||||
}
|
||||
|
||||
// Region, country and remote word each have one control per provider that
|
||||
// reads them; the body takes the shown provider's, and "" where it has none.
|
||||
function searchBody(f) {
|
||||
const provider = $("#s-provider", f).value;
|
||||
const own = (controls) => (controls[provider] ? $(controls[provider], f).value.trim() : "");
|
||||
return {
|
||||
provider: $("#s-provider", f).value,
|
||||
provider,
|
||||
name: $("#s-name", f).value.trim(),
|
||||
q: $("#s-q", f).value.trim(),
|
||||
area: Number($("#s-area", f).value) || 0,
|
||||
country: $("#s-country", f).value,
|
||||
country: own({ adzuna: "#s-country", freehire: "#s-countries" }),
|
||||
where: $("#s-where", f).value.trim(),
|
||||
region: $("#s-region", f).value,
|
||||
region: own({ platsbanken: "#s-region", jobnet: "#s-region-jobnet",
|
||||
jobbank: "#s-region-jobbank" }),
|
||||
remote: own({ freehire: "#s-mode", jobbank: "#s-remote-jobbank" }),
|
||||
pages: Number($("#s-pages", f).value) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
top of the list on New, as agents.js does for the Search agents cards. #}
|
||||
<section class="card" id="job-boards" data-section>
|
||||
<h2>Job boards</h2>
|
||||
<p class="lede">A query on Jobindex, Adzuna or Platsbanken finds roles at any company. Switching one off closes its roles until you switch it back on.</p>
|
||||
<p class="lede">A query on Jobindex, Adzuna, Platsbanken, freehire, Jobnet or Akademikernes Jobbank finds roles at any company. Switching one off closes its roles until you switch it back on.</p>
|
||||
|
||||
<div class="agents-head">
|
||||
<p id="searches-count" class="count" role="status"></p>
|
||||
@@ -38,6 +38,7 @@
|
||||
<option value="jobindex">Jobindex (Denmark)</option>
|
||||
<option value="adzuna">Adzuna (UK and EU)</option>
|
||||
<option value="platsbanken">Platsbanken (Sweden)</option>
|
||||
{% for value, label in providers %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label>Name <input type="text" id="s-name" required placeholder="Copenhagen backend"></label>
|
||||
</div>
|
||||
@@ -45,7 +46,10 @@
|
||||
<input type="text" id="s-q" required spellcheck="false">
|
||||
<span class="hint" data-provider="jobindex">Jobindex syntax: words are OR, <code>"a b"</code> is a phrase, <code>+word</code> is required.</span>
|
||||
<span class="hint" data-provider="adzuna" hidden>Every word must appear.</span>
|
||||
<span class="hint" data-provider="platsbanken" hidden>Free text; every word counts towards the match.</span></label>
|
||||
<span class="hint" data-provider="platsbanken" hidden>Free text; every word counts towards the match.</span>
|
||||
<span class="hint" data-provider="freehire" hidden>Words are OR over title, company, text and place, so a query is noisy; narrow it by country.</span>
|
||||
<span class="hint" data-provider="jobnet" hidden>A loose match: words, no phrases.</span>
|
||||
<span class="hint" data-provider="jobbank" hidden>Words are OR.</span></label>
|
||||
<div class="form-row">
|
||||
<label data-provider="jobindex">Area
|
||||
<select id="s-area">
|
||||
@@ -61,6 +65,28 @@
|
||||
<option value="">All of Sweden</option>
|
||||
{% for value, label in regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label data-provider="freehire" hidden>Countries <input type="text" id="s-countries" placeholder="dk,se" spellcheck="false">
|
||||
<span class="hint">ISO codes, comma-separated; empty for all</span></label>
|
||||
<label data-provider="freehire" hidden>Work mode
|
||||
<select id="s-mode">
|
||||
<option value="">Any</option>
|
||||
{% for value, label in modes %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label data-provider="jobnet" hidden>Region
|
||||
<select id="s-region-jobnet">
|
||||
<option value="">All of Denmark</option>
|
||||
{% for value, label in jobnet_regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label data-provider="jobbank" hidden>Region
|
||||
<select id="s-region-jobbank">
|
||||
<option value="">All of Denmark</option>
|
||||
{% for value, label in jobbank_regions %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label data-provider="jobbank" hidden>Remote work
|
||||
<select id="s-remote-jobbank">
|
||||
<option value="">Any</option>
|
||||
{% for value, label in jobbank_remote %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
|
||||
</select></label>
|
||||
<label>Pages
|
||||
<input type="number" id="s-pages" min="1" max="10" value="10">
|
||||
<span class="hint">per pull, up to 10</span></label>
|
||||
|
||||
+3
-1
@@ -1043,6 +1043,7 @@ class SearchBody(BaseModel):
|
||||
country: str = ""
|
||||
where: str = ""
|
||||
region: str = ""
|
||||
remote: str = ""
|
||||
pages: Annotated[int, Field(ge=0, le=10)] = 0
|
||||
|
||||
|
||||
@@ -1050,7 +1051,8 @@ def _params(body: SearchBody) -> dict[str, Any]:
|
||||
try:
|
||||
return jobs.sources.search_params(body.provider, body.q, area=body.area,
|
||||
country=body.country, where=body.where,
|
||||
region=body.region, pages=body.pages)
|
||||
region=body.region, remote=body.remote,
|
||||
pages=body.pages)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
|
||||
|
||||
+28
-3
@@ -27,8 +27,11 @@ keeps working.
|
||||
functions are the classifier's vocabulary and the hosts are what it parses,
|
||||
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
|
||||
app-agnostic and never reaches back into `app`. The
|
||||
Jobindex areas, Adzuna countries and Platsbanken regions on the Sources page
|
||||
come from the search readers in `plugins/readers/` for the same reason.
|
||||
Jobindex areas, Adzuna countries, the regions of Platsbanken, Jobnet and
|
||||
Jobbank, freehire's work modes and Jobbank's remote words on the Sources page
|
||||
come from the search readers in `plugins/readers/` for the same reason. The
|
||||
last four and the select's other providers are read from `readers.PROVIDERS`,
|
||||
so the page works whether or not a provider's module is loaded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -41,6 +44,7 @@ from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import HTMLResponse, RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from app import jobs
|
||||
from app.models import LADDER, REMOTE, RUNGS, STATUSES
|
||||
from app.web.deps import StoreDep
|
||||
|
||||
@@ -48,6 +52,7 @@ _ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
from plugins import readers # noqa: E402
|
||||
from plugins.ats import FUNCTIONS, HOSTS # noqa: E402
|
||||
from plugins.readers.adzuna import COUNTRIES as ADZUNA_COUNTRIES # noqa: E402
|
||||
from plugins.readers.jobindex import AREAS as JOBINDEX_AREAS # noqa: E402
|
||||
@@ -110,6 +115,25 @@ HOST_OPTIONS = list(HOSTS.items())
|
||||
LADDER_JSON = {"rungs": list(RUNGS), "levels": {k: list(v) for k, v in LADDER.items()}}
|
||||
|
||||
|
||||
def search_options() -> dict[str, list[tuple[str, str]]]:
|
||||
"""The `value, label` options the Sources form's providers add to its
|
||||
selects, read from `readers.PROVIDERS` on each request: the providers the
|
||||
template does not name itself, and the regions of Jobnet and Jobbank,
|
||||
freehire's work modes and Jobbank's remote words, each in the reader's
|
||||
order and without its empty value, which the form puts first as "all". All
|
||||
empty for a provider whose module is not loaded."""
|
||||
def options(provider: str, table: str) -> list[tuple[str, str]]:
|
||||
return [(value, label) for value, label in jobs.sources.provider_table(provider, table).items()
|
||||
if value]
|
||||
|
||||
return {"providers": [(n, m.LABEL) for n, m in readers.PROVIDERS.items()
|
||||
if n not in ("jobindex", "adzuna", "platsbanken")],
|
||||
"jobnet_regions": options("jobnet", "REGIONS"),
|
||||
"jobbank_regions": options("jobbank", "REGIONS"),
|
||||
"modes": options("freehire", "MODES"),
|
||||
"jobbank_remote": options("jobbank", "REMOTE")}
|
||||
|
||||
|
||||
def asset(path: str) -> str:
|
||||
"""A static URL stamped with the file's modification time, so a changed
|
||||
file is a different URL and a cached stale copy is never the answer."""
|
||||
@@ -180,7 +204,8 @@ def agents(request: Request) -> HTMLResponse:
|
||||
@router.get("/settings/sources", response_class=HTMLResponse)
|
||||
def sources(request: Request) -> HTMLResponse:
|
||||
return page(request, "settings/sources.html", areas=AREA_OPTIONS,
|
||||
countries=COUNTRY_OPTIONS, regions=REGION_OPTIONS, hosts=HOST_OPTIONS)
|
||||
countries=COUNTRY_OPTIONS, regions=REGION_OPTIONS, hosts=HOST_OPTIONS,
|
||||
**search_options())
|
||||
|
||||
|
||||
@router.get("/settings/account", response_class=HTMLResponse)
|
||||
|
||||
Reference in new issue
Block a user